You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2015/11/13 03:37:31 UTC
[01/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Repository: drill
Updated Branches:
refs/heads/master 9b88d5914 -> 7a151d691
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
new file mode 100644
index 0000000..2d351f2
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
@@ -0,0 +1,100 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector.complex.impl;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.holders.UnionHolder;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.ListVector;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
+
+public class UnionListReader extends AbstractFieldReader {
+
+ private ListVector vector;
+ private ValueVector data;
+ private UInt4Vector offsets;
+
+ public UnionListReader(ListVector vector) {
+ this.vector = vector;
+ this.data = vector.getDataVector();
+ this.offsets = vector.getOffsetVector();
+ }
+
+ @Override
+ public boolean isSet() {
+ return true;
+ }
+
+ MajorType type = Types.optional(MinorType.LIST);
+
+ public MajorType getType() {
+ return type;
+ }
+
+ private int currentOffset;
+ private int maxOffset;
+
+ @Override
+ public void setPosition(int index) {
+ super.setPosition(index);
+ currentOffset = offsets.getAccessor().get(index) - 1;
+ maxOffset = offsets.getAccessor().get(index + 1);
+ }
+
+ @Override
+ public FieldReader reader() {
+ return data.getReader();
+ }
+
+ @Override
+ public Object readObject() {
+ return vector.getAccessor().getObject(idx());
+ }
+
+ @Override
+ public void read(int index, UnionHolder holder) {
+ setPosition(idx());
+ for (int i = -1; i < index; i++) {
+ next();
+ }
+ holder.reader = data.getReader();
+ holder.isSet = data.getReader().isSet() ? 1 : 0;
+ }
+
+ @Override
+ public boolean next() {
+ if (currentOffset + 1 < maxOffset) {
+ data.getReader().setPosition(++currentOffset);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public void copyAsValue(ListWriter writer) {
+ ComplexCopier.copy(this, (FieldWriter) writer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java
new file mode 100644
index 0000000..caa3aa6
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.reader;
+
+import org.apache.drill.exec.vector.complex.reader.BaseReader.ListReader;
+import org.apache.drill.exec.vector.complex.reader.BaseReader.MapReader;
+import org.apache.drill.exec.vector.complex.reader.BaseReader.RepeatedListReader;
+import org.apache.drill.exec.vector.complex.reader.BaseReader.RepeatedMapReader;
+import org.apache.drill.exec.vector.complex.reader.BaseReader.ScalarReader;
+
+
+
+public interface FieldReader extends MapReader, ListReader, ScalarReader, RepeatedMapReader, RepeatedListReader {
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/writer/FieldWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/writer/FieldWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/writer/FieldWriter.java
new file mode 100644
index 0000000..1a64978
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/writer/FieldWriter.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.writer;
+
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ScalarWriter;
+
+public interface FieldWriter extends MapWriter, ListWriter, ScalarWriter {
+ void allocate();
+ void clear();
+}
[30/45] drill git commit: DRILL-3987: (REFACTOR) Remove any parent
Drill dependencies for drill-memory-base.
Posted by ja...@apache.org.
DRILL-3987: (REFACTOR) Remove any parent Drill dependencies for drill-memory-base.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a289fd94
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a289fd94
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a289fd94
Branch: refs/heads/master
Commit: a289fd941a33833524516dea1a3f75ea853aa055
Parents: 363bf33
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Nov 8 09:09:05 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 17:00:48 2015 -0800
----------------------------------------------------------------------
exec/memory/base/pom.xml | 23 --------------------
.../netty/buffer/PooledByteBufAllocatorL.java | 15 ++++++++-----
.../netty/buffer/UnsafeDirectLittleEndian.java | 11 +++++++---
.../exec/exception/OutOfMemoryException.java | 3 +--
4 files changed, 19 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a289fd94/exec/memory/base/pom.xml
----------------------------------------------------------------------
diff --git a/exec/memory/base/pom.xml b/exec/memory/base/pom.xml
index 6bfd29b..59f6a29 100644
--- a/exec/memory/base/pom.xml
+++ b/exec/memory/base/pom.xml
@@ -23,29 +23,6 @@
<dependencies>
<dependency>
- <groupId>org.apache.drill</groupId>
- <artifactId>drill-protocol</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.drill</groupId>
- <artifactId>drill-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.drill</groupId>
- <artifactId>drill-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>2.5.0</version>
- </dependency>
- <dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>3.0.1</version>
http://git-wip-us.apache.org/repos/asf/drill/blob/a289fd94/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index 5809df5..2fc1bd0 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -23,9 +23,6 @@ import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.util.AssertionUtil;
-
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Metric;
@@ -142,7 +139,7 @@ public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
}
normalBuffersHist.update(buf.capacity());
- if (AssertionUtil.ASSERT_ENABLED) {
+ if (ASSERT_ENABLED) {
normalBufferSize.addAndGet(buf.capacity());
normalBufferCount.incrementAndGet();
}
@@ -228,7 +225,7 @@ public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
normalBufferCount.set(0);
hugeBufferSize.set(0);
normalBufferSize.set(0);
- throw new DrillRuntimeException(buf.toString());
+ throw new IllegalStateException(buf.toString());
}
}
@@ -254,4 +251,12 @@ public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
buf.append(" bytes.");
return buf.toString();
}
+
+ public static final boolean ASSERT_ENABLED;
+
+ static {
+ boolean isAssertEnabled = false;
+ assert isAssertEnabled = true;
+ ASSERT_ENABLED = isAssertEnabled;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a289fd94/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
index 50a9a01..559f06d 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -23,8 +23,6 @@ import io.netty.util.internal.PlatformDependent;
import java.nio.ByteOrder;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.drill.exec.util.AssertionUtil;
-
public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
private final AbstractByteBuf wrapped;
@@ -43,7 +41,7 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
this.bufferSize = bufferSize;
// initCap is used if we're tracking memory release. If we're in non-debug mode, we'll skip this.
- this.initCap = AssertionUtil.ASSERT_ENABLED ? capacity() : -1;
+ this.initCap = ASSERT_ENABLED ? capacity() : -1;
}
private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake) {
@@ -243,5 +241,12 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
return released;
}
+ public static final boolean ASSERT_ENABLED;
+
+ static {
+ boolean isAssertEnabled = false;
+ assert isAssertEnabled = true;
+ ASSERT_ENABLED = isAssertEnabled;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a289fd94/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java b/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
index 17d099d..3fbae6d 100644
--- a/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
@@ -17,9 +17,8 @@
*/
package org.apache.drill.exec.exception;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-public class OutOfMemoryException extends DrillRuntimeException {
+public class OutOfMemoryException extends RuntimeException {
private static final long serialVersionUID = -6858052345185793382L;
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutOfMemoryException.class);
[11/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/io/netty/buffer/LargeBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/LargeBuffer.java b/exec/java-exec/src/main/java/io/netty/buffer/LargeBuffer.java
deleted file mode 100644
index f1d4842..0000000
--- a/exec/java-exec/src/main/java/io/netty/buffer/LargeBuffer.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This is basically a complete copy of DuplicatedByteBuf. We copy because we can't override the release methods to keep
- * global track of created Large Buffers.
- */
-public class LargeBuffer extends AbstractByteBuf {
-
- private final AtomicLong hugeBufferSize;
- private final AtomicLong hugeBufferCount;
-
- @Override
- public ByteBuffer nioBuffer(int index, int length) {
- return unwrap().nioBuffer(index, length);
- }
-
- private final ByteBuf buffer;
- private final int initCap;
-
- public LargeBuffer(ByteBuf buffer, AtomicLong hugeBufferSize, AtomicLong hugeBufferCount) {
- super(buffer.maxCapacity());
- initCap = buffer.capacity();
- this.hugeBufferCount = hugeBufferCount;
- this.hugeBufferSize = hugeBufferSize;
-
- if (buffer instanceof LargeBuffer) {
- this.buffer = ((LargeBuffer) buffer).buffer;
- } else {
- this.buffer = buffer;
- }
-
- setIndex(buffer.readerIndex(), buffer.writerIndex());
- }
-
- @Override
- public ByteBuf unwrap() {
- return buffer;
- }
-
- @Override
- public ByteBufAllocator alloc() {
- return buffer.alloc();
- }
-
- @Override
- public ByteOrder order() {
- return buffer.order();
- }
-
- @Override
- public boolean isDirect() {
- return buffer.isDirect();
- }
-
- @Override
- public int capacity() {
- return buffer.capacity();
- }
-
- @Override
- public ByteBuf capacity(int newCapacity) {
- buffer.capacity(newCapacity);
- return this;
- }
-
- @Override
- public boolean hasArray() {
- return buffer.hasArray();
- }
-
- @Override
- public byte[] array() {
- return buffer.array();
- }
-
- @Override
- public int arrayOffset() {
- return buffer.arrayOffset();
- }
-
- @Override
- public boolean hasMemoryAddress() {
- return buffer.hasMemoryAddress();
- }
-
- @Override
- public long memoryAddress() {
- return buffer.memoryAddress();
- }
-
- @Override
- public byte getByte(int index) {
- return _getByte(index);
- }
-
- @Override
- protected byte _getByte(int index) {
- return buffer.getByte(index);
- }
-
- @Override
- public short getShort(int index) {
- return _getShort(index);
- }
-
- @Override
- protected short _getShort(int index) {
- return buffer.getShort(index);
- }
-
- @Override
- public int getUnsignedMedium(int index) {
- return _getUnsignedMedium(index);
- }
-
- @Override
- protected int _getUnsignedMedium(int index) {
- return buffer.getUnsignedMedium(index);
- }
-
- @Override
- public int getInt(int index) {
- return _getInt(index);
- }
-
- @Override
- protected int _getInt(int index) {
- return buffer.getInt(index);
- }
-
- @Override
- public long getLong(int index) {
- return _getLong(index);
- }
-
- @Override
- protected long _getLong(int index) {
- return buffer.getLong(index);
- }
-
- @Override
- public ByteBuf copy(int index, int length) {
- return new LargeBuffer(buffer.copy(index, length), hugeBufferSize, hugeBufferCount);
- }
-
- @Override
- public ByteBuf slice(int index, int length) {
- return new SlicedByteBuf(this, index, length);
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
- buffer.getBytes(index, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
- buffer.getBytes(index, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuffer dst) {
- buffer.getBytes(index, dst);
- return this;
- }
-
- @Override
- public ByteBuf setByte(int index, int value) {
- _setByte(index, value);
- return this;
- }
-
- @Override
- protected void _setByte(int index, int value) {
- buffer.setByte(index, value);
- }
-
- @Override
- public ByteBuf setShort(int index, int value) {
- _setShort(index, value);
- return this;
- }
-
- @Override
- protected void _setShort(int index, int value) {
- buffer.setShort(index, value);
- }
-
- @Override
- public ByteBuf setMedium(int index, int value) {
- _setMedium(index, value);
- return this;
- }
-
- @Override
- protected void _setMedium(int index, int value) {
- buffer.setMedium(index, value);
- }
-
- @Override
- public ByteBuf setInt(int index, int value) {
- _setInt(index, value);
- return this;
- }
-
- @Override
- protected void _setInt(int index, int value) {
- buffer.setInt(index, value);
- }
-
- @Override
- public ByteBuf setLong(int index, long value) {
- _setLong(index, value);
- return this;
- }
-
- @Override
- protected void _setLong(int index, long value) {
- buffer.setLong(index, value);
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
- buffer.setBytes(index, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
- buffer.setBytes(index, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuffer src) {
- buffer.setBytes(index, src);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, OutputStream out, int length)
- throws IOException {
- buffer.getBytes(index, out, length);
- return this;
- }
-
- @Override
- public int getBytes(int index, GatheringByteChannel out, int length)
- throws IOException {
- return buffer.getBytes(index, out, length);
- }
-
- @Override
- public int setBytes(int index, InputStream in, int length)
- throws IOException {
- return buffer.setBytes(index, in, length);
- }
-
- @Override
- public int setBytes(int index, ScatteringByteChannel in, int length)
- throws IOException {
- return buffer.setBytes(index, in, length);
- }
-
- @Override
- public int nioBufferCount() {
- return buffer.nioBufferCount();
- }
-
- @Override
- public ByteBuffer[] nioBuffers(int index, int length) {
- return buffer.nioBuffers(index, length);
- }
-
- @Override
- public ByteBuffer internalNioBuffer(int index, int length) {
- return nioBuffer(index, length);
- }
-
- @Override
- public int forEachByte(int index, int length, ByteBufProcessor processor) {
- return buffer.forEachByte(index, length, processor);
- }
-
- @Override
- public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
- return buffer.forEachByteDesc(index, length, processor);
- }
-
- @Override
- public final int refCnt() {
- return unwrap().refCnt();
- }
-
- @Override
- public final ByteBuf retain() {
- unwrap().retain();
- return this;
- }
-
- @Override
- public final ByteBuf retain(int increment) {
- unwrap().retain(increment);
- return this;
- }
-
- @Override
- public boolean release() {
- return release(1);
- }
-
- @Override
- public boolean release(int decrement) {
- boolean released = unwrap().release(decrement);
- if (released) {
- hugeBufferSize.addAndGet(-initCap);
- hugeBufferCount.decrementAndGet();
- }
- return released;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
deleted file mode 100644
index 2ca79f0..0000000
--- a/exec/java-exec/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.util.internal.StringUtil;
-
-import java.lang.reflect.Field;
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.metrics.DrillMetrics;
-import org.apache.drill.exec.util.AssertionUtil;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
-
-public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PooledByteBufAllocatorL.class);
-
- private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
- private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
-
- private static final String METRIC_PREFIX = "drill.allocator.";
- public static final PooledByteBufAllocatorL DEFAULT = new PooledByteBufAllocatorL();
-
- private final MetricRegistry registry = DrillMetrics.getInstance();
- private final AtomicLong hugeBufferSize = new AtomicLong(0);
- private final AtomicLong hugeBufferCount = new AtomicLong(0);
- private final AtomicLong normalBufferSize = new AtomicLong(0);
- private final AtomicLong normalBufferCount = new AtomicLong(0);
-
- private final PoolArena<ByteBuffer>[] directArenas;
- private final MemoryStatusThread statusThread;
- private final Histogram largeBuffersHist;
- private final Histogram normalBuffersHist;
-
- private PooledByteBufAllocatorL() {
- super(true);
- try {
- Field f = PooledByteBufAllocator.class.getDeclaredField("directArenas");
- f.setAccessible(true);
- this.directArenas = (PoolArena<ByteBuffer>[]) f.get(this);
- } catch (Exception e) {
- throw new RuntimeException("Failure while initializing allocator. Unable to retrieve direct arenas field.", e);
- }
-
- if (memoryLogger.isTraceEnabled()) {
- statusThread = new MemoryStatusThread();
- statusThread.start();
- } else {
- statusThread = null;
- }
- removeOldMetrics();
-
- registry.register(METRIC_PREFIX + "normal.size", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return normalBufferSize.get();
- }
- });
-
- registry.register(METRIC_PREFIX + "normal.count", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return normalBufferCount.get();
- }
- });
-
- registry.register(METRIC_PREFIX + "huge.size", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return hugeBufferSize.get();
- }
- });
-
- registry.register(METRIC_PREFIX + "huge.count", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return hugeBufferCount.get();
- }
- });
-
- largeBuffersHist = registry.histogram(METRIC_PREFIX + "huge.hist");
- normalBuffersHist = registry.histogram(METRIC_PREFIX + "normal.hist");
-
- }
-
- private synchronized void removeOldMetrics() {
- registry.removeMatching(new MetricFilter() {
- @Override
- public boolean matches(String name, Metric metric) {
- return name.startsWith("drill.allocator.");
- }
- });
- }
-
- @Override
- protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
- throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
- }
-
- @Override
- protected UnsafeDirectLittleEndian newDirectBuffer(int initialCapacity, int maxCapacity) {
- PoolThreadCache cache = threadCache.get();
- PoolArena<ByteBuffer> directArena = cache.directArena;
-
- if (directArena != null) {
-
- if (initialCapacity > directArena.chunkSize) {
- // This is beyond chunk size so we'll allocate separately.
- ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity);
-
- hugeBufferCount.incrementAndGet();
- hugeBufferSize.addAndGet(buf.capacity());
- largeBuffersHist.update(buf.capacity());
- // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
- return new UnsafeDirectLittleEndian(new LargeBuffer(buf, hugeBufferSize, hugeBufferCount));
-
- } else {
- // within chunk, use arena.
- ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity);
- if (!(buf instanceof PooledUnsafeDirectByteBuf)) {
- fail();
- }
-
- normalBuffersHist.update(buf.capacity());
- if (AssertionUtil.ASSERT_ENABLED) {
- normalBufferSize.addAndGet(buf.capacity());
- normalBufferCount.incrementAndGet();
- }
-
- return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount, normalBufferSize);
- }
-
- } else {
- throw fail();
- }
- }
-
- private UnsupportedOperationException fail() {
- return new UnsupportedOperationException(
- "Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
- }
-
-
- @Override
- public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
- if (initialCapacity == 0 && maxCapacity == 0) {
- newDirectBuffer(initialCapacity, maxCapacity);
- }
- validate(initialCapacity, maxCapacity);
- return newDirectBuffer(initialCapacity, maxCapacity);
- }
-
- @Override
- public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
- throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
- }
-
-
- private static void validate(int initialCapacity, int maxCapacity) {
- if (initialCapacity < 0) {
- throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)");
- }
- if (initialCapacity > maxCapacity) {
- throw new IllegalArgumentException(String.format(
- "initialCapacity: %d (expected: not greater than maxCapacity(%d)",
- initialCapacity, maxCapacity));
- }
- }
-
- private class MemoryStatusThread extends Thread {
-
- public MemoryStatusThread() {
- super("memory-status-logger");
- this.setDaemon(true);
- this.setName("allocation.logger");
- }
-
- @Override
- public void run() {
- while (true) {
- memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString());
- try {
- Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000);
- } catch (InterruptedException e) {
- return;
- }
-
- }
- }
-
- }
-
- public void checkAndReset() {
- if (hugeBufferCount.get() != 0 || normalBufferCount.get() != 0) {
- StringBuilder buf = new StringBuilder();
- buf.append("Large buffers outstanding: ");
- buf.append(hugeBufferCount.get());
- buf.append(" totaling ");
- buf.append(hugeBufferSize.get());
- buf.append(" bytes.");
- buf.append('\n');
- buf.append("Normal buffers outstanding: ");
- buf.append(normalBufferCount.get());
- buf.append(" totaling ");
- buf.append(normalBufferSize.get());
- buf.append(" bytes.");
- hugeBufferCount.set(0);
- normalBufferCount.set(0);
- hugeBufferSize.set(0);
- normalBufferSize.set(0);
- throw new DrillRuntimeException(buf.toString());
- }
- }
-
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append(directArenas.length);
- buf.append(" direct arena(s):");
- buf.append(StringUtil.NEWLINE);
- for (PoolArena<ByteBuffer> a : directArenas) {
- buf.append(a);
- }
-
- buf.append("Large buffers outstanding: ");
- buf.append(this.hugeBufferCount.get());
- buf.append(" totaling ");
- buf.append(this.hugeBufferSize.get());
- buf.append(" bytes.");
- buf.append('\n');
- buf.append("Normal buffers outstanding: ");
- buf.append(this.normalBufferCount.get());
- buf.append(" totaling ");
- buf.append(this.normalBufferSize.get());
- buf.append(" bytes.");
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
deleted file mode 100644
index e332b13..0000000
--- a/exec/java-exec/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.netty.buffer;
-
-import io.netty.util.internal.PlatformDependent;
-
-import java.nio.ByteOrder;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.drill.exec.util.AssertionUtil;
-
-public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
- private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
- private final AbstractByteBuf wrapped;
- private final long memoryAddress;
- private AtomicLong bufferCount;
- private AtomicLong bufferSize;
- private long initCap = -1;
-
- UnsafeDirectLittleEndian(LargeBuffer buf) {
- this(buf, true);
- }
-
- UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong bufferCount, AtomicLong bufferSize) {
- this(buf, true);
- this.bufferCount = bufferCount;
- this.bufferSize = bufferSize;
-
- // initCap is used if we're tracking memory release. If we're in non-debug mode, we'll skip this.
- this.initCap = AssertionUtil.ASSERT_ENABLED ? capacity() : -1;
- }
-
- private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake) {
- super(buf);
- if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
- throw new IllegalStateException("Drill only runs on LittleEndian systems.");
- }
- wrapped = buf;
- this.memoryAddress = buf.memoryAddress();
- }
- private long addr(int index) {
- return memoryAddress + index;
- }
-
- @Override
- public long getLong(int index) {
-// wrapped.checkIndex(index, 8);
- long v = PlatformDependent.getLong(addr(index));
- return v;
- }
-
- @Override
- public float getFloat(int index) {
- return Float.intBitsToFloat(getInt(index));
- }
-
- @Override
- public ByteBuf slice() {
- return slice(this.readerIndex(), readableBytes());
- }
-
- @Override
- public ByteBuf slice(int index, int length) {
- return new SlicedByteBuf(this, index, length);
- }
-
- @Override
- public ByteOrder order() {
- return ByteOrder.LITTLE_ENDIAN;
- }
-
- @Override
- public ByteBuf order(ByteOrder endianness) {
- return this;
- }
-
- @Override
- public double getDouble(int index) {
- return Double.longBitsToDouble(getLong(index));
- }
-
- @Override
- public char getChar(int index) {
- return (char) getShort(index);
- }
-
- @Override
- public long getUnsignedInt(int index) {
- return getInt(index) & 0xFFFFFFFFL;
- }
-
- @Override
- public int getInt(int index) {
-// wrapped.checkIndex(index, 4);
- int v = PlatformDependent.getInt(addr(index));
- return v;
- }
-
- @Override
- public int getUnsignedShort(int index) {
- return getShort(index) & 0xFFFF;
- }
-
- @Override
- public short getShort(int index) {
-// wrapped.checkIndex(index, 2);
- short v = PlatformDependent.getShort(addr(index));
- return v;
- }
-
- @Override
- public ByteBuf setShort(int index, int value) {
- wrapped.checkIndex(index, 2);
- _setShort(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setInt(int index, int value) {
- wrapped.checkIndex(index, 4);
- _setInt(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setLong(int index, long value) {
- wrapped.checkIndex(index, 8);
- _setLong(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setChar(int index, int value) {
- setShort(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setFloat(int index, float value) {
- setInt(index, Float.floatToRawIntBits(value));
- return this;
- }
-
- @Override
- public ByteBuf setDouble(int index, double value) {
- setLong(index, Double.doubleToRawLongBits(value));
- return this;
- }
-
- @Override
- public ByteBuf writeShort(int value) {
- wrapped.ensureWritable(2);
- _setShort(wrapped.writerIndex, value);
- wrapped.writerIndex += 2;
- return this;
- }
-
- @Override
- public ByteBuf writeInt(int value) {
- wrapped.ensureWritable(4);
- _setInt(wrapped.writerIndex, value);
- wrapped.writerIndex += 4;
- return this;
- }
-
- @Override
- public ByteBuf writeLong(long value) {
- wrapped.ensureWritable(8);
- _setLong(wrapped.writerIndex, value);
- wrapped.writerIndex += 8;
- return this;
- }
-
- @Override
- public ByteBuf writeChar(int value) {
- writeShort(value);
- return this;
- }
-
- @Override
- public ByteBuf writeFloat(float value) {
- writeInt(Float.floatToRawIntBits(value));
- return this;
- }
-
- @Override
- public ByteBuf writeDouble(double value) {
- writeLong(Double.doubleToRawLongBits(value));
- return this;
- }
-
- private void _setShort(int index, int value) {
- PlatformDependent.putShort(addr(index), (short) value);
- }
-
- private void _setInt(int index, int value) {
- PlatformDependent.putInt(addr(index), value);
- }
-
- private void _setLong(int index, long value) {
- PlatformDependent.putLong(addr(index), value);
- }
-
- @Override
- public byte getByte(int index) {
- return PlatformDependent.getByte(addr(index));
- }
-
- @Override
- public ByteBuf setByte(int index, int value) {
- PlatformDependent.putByte(addr(index), (byte) value);
- return this;
- }
-
- @Override
- public boolean release() {
- return release(1);
- }
-
- @Override
- public boolean release(int decrement) {
- boolean released = super.release(decrement);
- if (released && initCap != -1) {
- bufferCount.decrementAndGet();
- bufferSize.addAndGet(-initCap);
- }
- return released;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
deleted file mode 100644
index f5ae70c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.exception;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.record.RecordBatch;
-
-/**
- * An exception that is used to signal that allocation request in bytes is greater than the maximum allowed by
- * {@link org.apache.drill.exec.memory.BufferAllocator#buffer(int) allocator}.
- *
- * <p>Operators should handle this exception to split the batch and later resume the execution on the next
- * {@link RecordBatch#next() iteration}.</p>
- *
- */
-public class OversizedAllocationException extends DrillRuntimeException {
- public OversizedAllocationException() {
- super();
- }
-
- public OversizedAllocationException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public OversizedAllocationException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public OversizedAllocationException(String message) {
- super(message);
- }
-
- public OversizedAllocationException(Throwable cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/exception/SchemaChangeRuntimeException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/exception/SchemaChangeRuntimeException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/exception/SchemaChangeRuntimeException.java
deleted file mode 100644
index f2a7e63..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/exception/SchemaChangeRuntimeException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.exception;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-
-public class SchemaChangeRuntimeException extends DrillRuntimeException {
- public SchemaChangeRuntimeException() {
- super();
- }
-
- public SchemaChangeRuntimeException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public SchemaChangeRuntimeException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public SchemaChangeRuntimeException(String message) {
- super(message);
- }
-
- public SchemaChangeRuntimeException(Throwable cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
deleted file mode 100644
index e1025df..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.holders;
-
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-public class ComplexHolder implements ValueHolder {
- public FieldReader reader;
- public int isSet;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
deleted file mode 100644
index 391a795..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.expr.holders;
-
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.Types;
-
-/*
- * Holder class for the vector ObjectVector. This holder internally stores a
- * reference to an object. The ObjectVector maintains an array of these objects.
- * This holder can be used only as workspace variables in aggregate functions.
- * Using this holder should be avoided and we should stick to native holder types.
- */
-@Deprecated
-public class ObjectHolder implements ValueHolder {
- public static final TypeProtos.MajorType TYPE = Types.required(TypeProtos.MinorType.GENERIC_OBJECT);
-
- public TypeProtos.MajorType getType() {return TYPE;}
-
- public Object obj;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
deleted file mode 100644
index 09746da..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.holders;
-
-public final class RepeatedListHolder implements ValueHolder{
- public int start;
- public int end;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
deleted file mode 100644
index 247f75e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.holders;
-
-public final class RepeatedMapHolder implements ValueHolder{
- public int start;
- public int end;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
deleted file mode 100644
index 84cdefb..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.holders;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-public class UnionHolder implements ValueHolder {
- public static final MajorType TYPE = Types.optional(MinorType.UNION);
- public FieldReader reader;
- public int isSet;
-
- public MajorType getType() {
- return reader.getType();
- }
-
- public boolean isSet() {
- return isSet == 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
deleted file mode 100644
index 2431dcb..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.holders;
-
-/**
- * Wrapper object for an individual value in Drill.
- *
- * ValueHolders are designed to be mutable wrapper objects for defining clean
- * APIs that access data in Drill. For performance, object creation is avoided
- * at all costs throughout execution. For this reason, ValueHolders are
- * disallowed from implementing any methods, this allows for them to be
- * replaced by their java primitive inner members during optimization of
- * run-time generated code.
- */
-public interface ValueHolder {
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
deleted file mode 100644
index b01534e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.DrillBuf;
-import io.netty.buffer.UnsafeDirectLittleEndian;
-
-import java.io.Closeable;
-
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.util.Pointer;
-
-/**
- * Wrapper class to deal with byte buffer allocation. Ensures users only use designated methods. Also allows inser
- */
-public interface BufferAllocator extends Closeable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BufferAllocator.class);
-
- /**
- * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
- * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
- *
- * @param size
- * The size in bytes.
- * @return A new ByteBuf.
- * @throws OutOfMemoryRuntimeException if buffer cannot be allocated
- */
- public abstract DrillBuf buffer(int size);
-
- /**
- * Allocate a new or reused buffer within provided range. Note that the buffer may technically be larger than the
- * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
- *
- * @param minSize The minimum size in bytes.
- * @param maxSize The maximum size in bytes.
- * @return A new ByteBuf.
- * @throws OutOfMemoryRuntimeException if buffer cannot be allocated
- */
- public abstract DrillBuf buffer(int minSize, int maxSize);
-
- public abstract ByteBufAllocator getUnderlyingAllocator();
-
- /**
- * Create a child allocator nested below this one.
- *
- * @param context - owning fragment for this allocator
- * @param initialReservation - specified in bytes
- * @param maximumReservation - specified in bytes
- * @param applyFragmentLimit - flag to conditionally enable fragment memory limits
- * @return - a new buffer allocator owned by the parent it was spawned from
- * @throws OutOfMemoryException - when off-heap memory has been exhausted
- */
- public abstract BufferAllocator getChildAllocator(FragmentContext context, long initialReservation,
- long maximumReservation, boolean applyFragmentLimit) throws OutOfMemoryException;
-
- /**
- * Take over ownership of fragment accounting. Always takes over ownership.
- * @param buf
- * @return false if over allocation.
- */
- public boolean takeOwnership(DrillBuf buf) ;
-
- /**
- * Take over ownership of fragment accounting. Always takes over ownership.
- * @param buf
- * @return false if over allocation.
- */
- public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut);
-
- public PreAllocator getNewPreAllocator();
-
- //public void addFragmentContext(FragmentContext c);
-
- /**
- * For Top Level Allocators. Reset the fragment limits for all allocators
- */
- public void resetFragmentLimits();
-
- /**
- * For Child allocators to set the Fragment limit for the corresponding fragment allocator.
- * @param l the new fragment limit
- */
- public void setFragmentLimit(long l);
-
- public long getFragmentLimit();
-
-
- /**
- * Not thread safe.
- *
- * WARNING: unclaimed pre-allocations leak memory. If you call preAllocate(), you must
- * make sure to ultimately try to get the buffer and release it.
- */
- public interface PreAllocator {
- public boolean preAllocate(int bytes);
-
- public DrillBuf getAllocation();
- }
-
- /**
- * @param bytes
- * @return
- */
-
- /**
- *
- */
-
- /**
- * Close and release all buffers generated from this buffer pool.
- */
- @Override
- public abstract void close();
-
- public abstract long getAllocatedMemory();
-
- public abstract long getPeakMemoryAllocation();
-
- public DrillBuf getEmpty();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
deleted file mode 100644
index 305eabd..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-
-public class OutOfMemoryRuntimeException extends DrillRuntimeException{
-
- public OutOfMemoryRuntimeException() {
- super();
-
- }
-
- public OutOfMemoryRuntimeException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
-
- }
-
- public OutOfMemoryRuntimeException(String message, Throwable cause) {
- super(message, cause);
-
- }
-
- public OutOfMemoryRuntimeException(String message) {
- super(message);
-
- }
-
- public OutOfMemoryRuntimeException(Throwable cause) {
- super(cause);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
deleted file mode 100644
index 26e1257..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.record;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Map;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-
-
-public class MaterializedField {
- private final Key key;
- // use an ordered set as existing code relies on order (e,g. parquet writer)
- private final LinkedHashSet<MaterializedField> children;
-
- private MaterializedField(SchemaPath path, MajorType type) {
- this(path, type, new LinkedHashSet<MaterializedField>());
- }
-
- private MaterializedField(SchemaPath path, MajorType type, LinkedHashSet<MaterializedField> children) {
- this.key = new Key(path, type);
- this.children = children;
- }
-
- public static MaterializedField create(SerializedField serField){
- MaterializedField field = new MaterializedField(SchemaPath.create(serField.getNamePart()), serField.getMajorType());
- for (SerializedField sf:serField.getChildList()) {
- field.addChild(MaterializedField.create(sf));
- }
- return field;
- }
-
- /**
- * Create and return a serialized field based on the current state.
- */
- public SerializedField getSerializedField() {
- SerializedField.Builder serializedFieldBuilder = getAsBuilder();
- for(MaterializedField childMaterializedField : getChildren()) {
- serializedFieldBuilder.addChild(childMaterializedField.getSerializedField());
- }
- return serializedFieldBuilder.build();
- }
-
-
- public SerializedField.Builder getAsBuilder(){
- return SerializedField.newBuilder() //
- .setMajorType(key.type) //
- .setNamePart(key.path.getAsNamePart());
- }
-
- public Collection<MaterializedField> getChildren() {
- return children;
- }
-
- public void addChild(MaterializedField field){
- children.add(field);
- }
-
- public MaterializedField clone() {
- return withPathAndType(getPath(), getType());
- }
-
- public MaterializedField withType(MajorType type) {
- return withPathAndType(getPath(), type);
- }
-
- public MaterializedField withPath(SchemaPath path) {
- return withPathAndType(path, getType());
- }
-
- public MaterializedField withPathAndType(final SchemaPath path, final MajorType type) {
- final LinkedHashSet<MaterializedField> newChildren = new LinkedHashSet<>(children.size());
- final MaterializedField clone = new MaterializedField(path, type, newChildren);
- for (final MaterializedField child:children) {
- newChildren.add(child.clone());
- }
- return clone;
- }
-
- public String getLastName(){
- PathSegment seg = key.path.getRootSegment();
- while (seg.getChild() != null) {
- seg = seg.getChild();
- }
- return seg.getNameSegment().getPath();
- }
-
-
- // TODO: rewrite without as direct match rather than conversion then match.
- public boolean matches(SerializedField field){
- MaterializedField f = create(field);
- return f.equals(this);
- }
-
- public static MaterializedField create(String path, MajorType type){
- SchemaPath p = SchemaPath.getSimplePath(path);
- return create(p, type);
- }
-
- public static MaterializedField create(SchemaPath path, MajorType type) {
- return new MaterializedField(path, type);
- }
-
- public SchemaPath getPath(){
- return key.path;
- }
-
- /**
- * Get the schema path. Deprecated, use getPath() instead.
- * @return the SchemaPath of this field.
- */
- @Deprecated
- public SchemaPath getAsSchemaPath(){
- return getPath();
- }
-
-// public String getName(){
-// StringBuilder sb = new StringBuilder();
-// boolean first = true;
-// for(NamePart np : def.getNameList()){
-// if(np.getType() == Type.ARRAY){
-// sb.append("[]");
-// }else{
-// if(first){
-// first = false;
-// }else{
-// sb.append(".");
-// }
-// sb.append('`');
-// sb.append(np.getName());
-// sb.append('`');
-//
-// }
-// }
-// return sb.toString();
-// }
-
- public int getWidth() {
- return key.type.getWidth();
- }
-
- public MajorType getType() {
- return key.type;
- }
-
- public int getScale() {
- return key.type.getScale();
- }
- public int getPrecision() {
- return key.type.getPrecision();
- }
- public boolean isNullable() {
- return key.type.getMode() == DataMode.OPTIONAL;
- }
-
- public DataMode getDataMode() {
- return key.type.getMode();
- }
-
- public MaterializedField getOtherNullableVersion(){
- MajorType mt = key.type;
- DataMode newDataMode = null;
- switch (mt.getMode()){
- case OPTIONAL:
- newDataMode = DataMode.REQUIRED;
- break;
- case REQUIRED:
- newDataMode = DataMode.OPTIONAL;
- break;
- default:
- throw new UnsupportedOperationException();
- }
- return new MaterializedField(key.path, mt.toBuilder().setMode(newDataMode).build());
- }
-
- public Class<?> getValueClass() {
- return TypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
- }
-
- public boolean matches(SchemaPath path) {
- if (!path.isSimplePath()) {
- return false;
- }
-
- return key.path.equals(path);
- }
-
-
- @Override
- public int hashCode() {
- int result = 1;
- // DRILL-1872: Compute hashCode only on key. See also the comment
- // in MapVector$MapTransferPair
- result = ((key == null) ? 0 : key.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- MaterializedField other = (MaterializedField) obj;
- // DRILL-1872: Compute equals only on key. See also the comment
- // in MapVector$MapTransferPair
-
- if (key == null) {
- if (other.key != null) {
- return false;
- }
- } else if (!key.equals(other.key)) {
- return false;
- }
- return true;
- }
-
-
- @Override
- public String toString() {
- final int maxLen = 10;
- String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : "";
- return key.path + "(" + key.type.getMinorType().name() + ":" + key.type.getMode().name() + ")" + childStr;
- }
-
-
- private String toString(Collection<?> collection, int maxLen) {
- StringBuilder builder = new StringBuilder();
- builder.append("[");
- int i = 0;
- for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
- if (i > 0){
- builder.append(", ");
- }
- builder.append(iterator.next());
- }
- builder.append("]");
- return builder.toString();
- }
-
- public Key key() {
- return key;
- }
-
- public String toExpr(){
- return key.path.toExpr();
- }
-
- /**
- * Since the {@code MaterializedField) itself is mutable, in certain cases, it is not suitable
- * as a key of a {@link Map}. This inner class allows the {@link MaterializedField} object to be
- * used for this purpose.
- */
- public class Key {
-
- private final SchemaPath path;
- private final MajorType type;
-
- private Key(SchemaPath path, MajorType type) {
- this.path = path;
- this.type = type;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((path == null) ? 0 : path.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- Key other = (Key) obj;
- if (path == null) {
- if (other.path != null) {
- return false;
- }
- } else if (!path.equals(other.path)) {
- return false;
- }
-
- return true;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/record/TransferPair.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TransferPair.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/TransferPair.java
deleted file mode 100644
index 3585a8c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TransferPair.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.record;
-
-import org.apache.drill.exec.vector.ValueVector;
-
-public interface TransferPair {
- public void transfer();
- public void splitAndTransfer(int startIndex, int length);
- public ValueVector getTo();
- public void copyValueSafe(int from, int to);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
deleted file mode 100644
index 931f981..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.record;
-
-import java.util.Arrays;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.vector.ValueVector;
-
-import com.carrotsearch.hppc.IntArrayList;
-import com.google.common.base.Preconditions;
-
-public class TypedFieldId {
- final MajorType finalType;
- final MajorType secondaryFinal;
- final MajorType intermediateType;
- final int[] fieldIds;
- final boolean isHyperReader;
- final boolean isListVector;
- final PathSegment remainder;
-
- public TypedFieldId(MajorType type, int... fieldIds) {
- this(type, type, type, false, null, fieldIds);
- }
-
- public TypedFieldId(MajorType type, IntArrayList breadCrumb, PathSegment remainder) {
- this(type, type, type, false, remainder, breadCrumb.toArray());
- }
-
- public TypedFieldId(MajorType type, boolean isHyper, int... fieldIds) {
- this(type, type, type, isHyper, null, fieldIds);
- }
-
- public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, PathSegment remainder, int... fieldIds) {
- this(intermediateType, secondaryFinal, finalType, isHyper, false, remainder, fieldIds);
- }
-
- public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, boolean isListVector, PathSegment remainder, int... fieldIds) {
- super();
- this.intermediateType = intermediateType;
- this.finalType = finalType;
- this.secondaryFinal = secondaryFinal;
- this.fieldIds = fieldIds;
- this.isHyperReader = isHyper;
- this.isListVector = isListVector;
- this.remainder = remainder;
- }
-
- public TypedFieldId cloneWithChild(int id) {
- int[] fieldIds = ArrayUtils.add(this.fieldIds, id);
- return new TypedFieldId(intermediateType, secondaryFinal, finalType, isHyperReader, remainder, fieldIds);
- }
-
- public PathSegment getLastSegment() {
- if (remainder == null) {
- return null;
- }
- PathSegment seg = remainder;
- while (seg.getChild() != null) {
- seg = seg.getChild();
- }
- return seg;
- }
-
- public TypedFieldId cloneWithRemainder(PathSegment remainder) {
- return new TypedFieldId(intermediateType, secondaryFinal, finalType, isHyperReader, remainder, fieldIds);
- }
-
- public boolean hasRemainder() {
- return remainder != null;
- }
-
- public PathSegment getRemainder() {
- return remainder;
- }
-
- public boolean isHyperReader() {
- return isHyperReader;
- }
-
- public boolean isListVector() {
- return isListVector;
- }
-
- public MajorType getIntermediateType() {
- return intermediateType;
- }
-
- public Class<? extends ValueVector> getIntermediateClass() {
- return (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(intermediateType.getMinorType(), intermediateType.getMode());
- }
-
- public MajorType getFinalType() {
- return finalType;
- }
-
- public int[] getFieldIds() {
- return fieldIds;
- }
-
- public MajorType getSecondaryFinal() {
- return secondaryFinal;
- }
-
- public static Builder newBuilder() {
- return new Builder();
- }
-
- public static class Builder{
- final IntArrayList ids = new IntArrayList();
- MajorType finalType;
- MajorType intermediateType;
- MajorType secondaryFinal;
- PathSegment remainder;
- boolean hyperReader = false;
- boolean withIndex = false;
- boolean isListVector = false;
-
- public Builder addId(int id) {
- ids.add(id);
- return this;
- }
-
- public Builder withIndex() {
- withIndex = true;
- return this;
- }
-
- public Builder remainder(PathSegment remainder) {
- this.remainder = remainder;
- return this;
- }
-
- public Builder hyper() {
- this.hyperReader = true;
- return this;
- }
-
- public Builder listVector() {
- this.isListVector = true;
- return this;
- }
-
- public Builder finalType(MajorType finalType) {
- this.finalType = finalType;
- return this;
- }
-
- public Builder secondaryFinal(MajorType secondaryFinal) {
- this.secondaryFinal = secondaryFinal;
- return this;
- }
-
- public Builder intermediateType(MajorType intermediateType) {
- this.intermediateType = intermediateType;
- return this;
- }
-
- public TypedFieldId build() {
- Preconditions.checkNotNull(intermediateType);
- Preconditions.checkNotNull(finalType);
-
- if (intermediateType == null) {
- intermediateType = finalType;
- }
- if (secondaryFinal == null) {
- secondaryFinal = finalType;
- }
-
- MajorType actualFinalType = finalType;
- //MajorType secondaryFinal = finalType;
-
- // if this has an index, switch to required type for output
- //if(withIndex && intermediateType == finalType) actualFinalType = finalType.toBuilder().setMode(DataMode.REQUIRED).build();
-
- // if this isn't a direct access, switch the final type to nullable as offsets may be null.
- // TODO: there is a bug here with some things.
- //if(intermediateType != finalType) actualFinalType = finalType.toBuilder().setMode(DataMode.OPTIONAL).build();
-
- return new TypedFieldId(intermediateType, secondaryFinal, actualFinalType, hyperReader, isListVector, remainder, ids.toArray());
- }
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(fieldIds);
- result = prime * result + ((finalType == null) ? 0 : finalType.hashCode());
- result = prime * result + ((intermediateType == null) ? 0 : intermediateType.hashCode());
- result = prime * result + (isHyperReader ? 1231 : 1237);
- result = prime * result + ((remainder == null) ? 0 : remainder.hashCode());
- result = prime * result + ((secondaryFinal == null) ? 0 : secondaryFinal.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- TypedFieldId other = (TypedFieldId) obj;
- if (!Arrays.equals(fieldIds, other.fieldIds)) {
- return false;
- }
- if (finalType == null) {
- if (other.finalType != null) {
- return false;
- }
- } else if (!finalType.equals(other.finalType)) {
- return false;
- }
- if (intermediateType == null) {
- if (other.intermediateType != null) {
- return false;
- }
- } else if (!intermediateType.equals(other.intermediateType)) {
- return false;
- }
- if (isHyperReader != other.isHyperReader) {
- return false;
- }
- if (remainder == null) {
- if (other.remainder != null) {
- return false;
- }
- } else if (!remainder.equals(other.remainder)) {
- return false;
- }
- if (secondaryFinal == null) {
- if (other.secondaryFinal != null) {
- return false;
- }
- } else if (!secondaryFinal.equals(other.secondaryFinal)) {
- return false;
- }
- return true;
- }
-
- @Override
- public String toString() {
- final int maxLen = 10;
- return "TypedFieldId [fieldIds="
- + (fieldIds != null ? Arrays.toString(Arrays.copyOf(fieldIds, Math.min(fieldIds.length, maxLen))) : null)
- + ", remainder=" + remainder + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/util/CallBack.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/CallBack.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/CallBack.java
deleted file mode 100644
index 0243f8a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/CallBack.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-
-public interface CallBack {
- public void doWork();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
deleted file mode 100644
index dd5145d..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.Text;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-
-public class JsonStringArrayList<E> extends ArrayList<E> {
-
- private static ObjectMapper mapper;
-
- static {
- mapper = new ObjectMapper();
- SimpleModule serializer = new SimpleModule("TextSerializer")
- .addSerializer(Text.class, new TextSerializer());
- mapper.registerModule(serializer);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof List)) {
- return false;
- }
- List other = (List) obj;
- return this.size() == other.size() && this.containsAll(other);
- }
-
- @Override
- public final String toString() {
- try {
- return mapper.writeValueAsString(this);
- } catch(JsonProcessingException e) {
- throw new IllegalStateException("Cannot serialize array list to JSON string", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
deleted file mode 100644
index ac980eb..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.apache.hadoop.io.Text;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-
-/*
- * Simple class that extends the regular java.util.HashMap but overrides the
- * toString() method of the HashMap class to produce a JSON string instead
- */
-public class JsonStringHashMap<K, V> extends LinkedHashMap<K, V> {
-
- private static ObjectMapper mapper;
-
- static {
- mapper = new ObjectMapper();
- SimpleModule serializer = new SimpleModule("TextSerializer")
- .addSerializer(Text.class, new TextSerializer());
- mapper.registerModule(serializer);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof Map)) {
- return false;
- }
- Map other = (Map) obj;
- if (this.size() != other.size()) {
- return false;
- }
- for (K key : this.keySet()) {
- if (this.get(key) == null ) {
- if (other.get(key) == null) {
- continue;
- } else {
- return false;
- }
- }
- if ( ! this.get(key).equals(other.get(key))) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public final String toString() {
- try {
- return mapper.writeValueAsString(this);
- } catch(JsonProcessingException e) {
- throw new IllegalStateException("Cannot serialize hash map to JSON string", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/util/Pointer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/Pointer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/Pointer.java
deleted file mode 100644
index 074ed76..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/Pointer.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-public class Pointer<T> {
- public T value;
-
- public Pointer(){}
-
- public Pointer(T value){
- this.value = value;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
deleted file mode 100644
index 87df102..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-
-public class TextSerializer extends StdSerializer<Text> {
-
- public TextSerializer() {
- super(Text.class);
- }
-
- @Override
- public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
- jsonGenerator.writeString(text.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
deleted file mode 100644
index 7d1f08d..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import com.google.common.base.Preconditions;
-
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
-
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
- }
-
- public V getVector() {
- return vector;
- }
-
- public boolean isCreated() {
- return created;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
deleted file mode 100644
index 622e2d2..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import org.apache.drill.exec.vector.complex.RepeatedFixedWidthVectorLike;
-import org.apache.drill.exec.vector.complex.RepeatedVariableWidthVectorLike;
-
-public class AllocationHelper {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class);
-
- public static void allocate(ValueVector v, int valueCount, int bytesPerValue) {
- allocate(v, valueCount, bytesPerValue, 5);
- }
-
- public static void allocatePrecomputedChildCount(ValueVector v, int valueCount, int bytesPerValue, int childValCount){
- if(v instanceof FixedWidthVector) {
- ((FixedWidthVector) v).allocateNew(valueCount);
- } else if (v instanceof VariableWidthVector) {
- ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount);
- } else if(v instanceof RepeatedFixedWidthVectorLike) {
- ((RepeatedFixedWidthVectorLike) v).allocateNew(valueCount, childValCount);
- } else if(v instanceof RepeatedVariableWidthVectorLike) {
- ((RepeatedVariableWidthVectorLike) v).allocateNew(childValCount * bytesPerValue, valueCount, childValCount);
- } else {
- v.allocateNew();
- }
- }
-
- public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){
- allocatePrecomputedChildCount(v, valueCount, bytesPerValue, repeatedPerTop * valueCount);
- }
-
- /**
- * Allocates the exact amount if v is fixed width, otherwise falls back to dynamic allocation
- * @param v value vector we are trying to allocate
- * @param valueCount size we are trying to allocate
- * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the memory
- */
- public static void allocateNew(ValueVector v, int valueCount) {
- if (v instanceof FixedWidthVector) {
- ((FixedWidthVector) v).allocateNew(valueCount);
- } else {
- v.allocateNew();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
deleted file mode 100644
index 2fc741c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.DrillBuf;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-
-public abstract class BaseDataValueVector extends BaseValueVector {
-
- protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
-
- protected DrillBuf data;
-
- public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- data = allocator.getEmpty();
- }
-
- @Override
- public void clear() {
- if (data != null) {
- data.release();
- }
- data = allocator.getEmpty();
- super.clear();
- }
-
- @Override
- public void close() {
- clear();
- if (data != null) {
- data.release();
- data = null;
- }
- super.close();
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- DrillBuf[] out;
- if (getBufferSize() == 0) {
- out = new DrillBuf[0];
- } else {
- out = new DrillBuf[]{data};
- if (clear) {
- data.readerIndex(0);
- data.retain(1);
- }
- }
- if (clear) {
- clear();
- }
- return out;
- }
-
- @Override
- public int getBufferSize() {
- if (getAccessor().getValueCount() == 0) {
- return 0;
- }
- return data.writerIndex();
- }
-
- public DrillBuf getBuffer() {
- return data;
- }
-
- /**
- * This method has a similar effect of allocateNew() without actually clearing and reallocating
- * the value vector. The purpose is to move the value vector to a "mutate" state
- */
- public void reset() {}
-}
[22/45] drill git commit: DRILL-3987: (MOVE) Extract RPC,
memory-base and memory-impl as separate modules.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
new file mode 100644
index 0000000..dcf28ff
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+
+public class RpcExceptionHandler<C extends RemoteConnection> implements ChannelHandler{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcExceptionHandler.class);
+
+ private final C connection;
+
+ public RpcExceptionHandler(C connection){
+ this.connection = connection;
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+ if(!ctx.channel().isOpen() || cause.getMessage().equals("Connection reset by peer")){
+ logger.warn("Exception occurred with closed channel. Connection: {}", connection.getName(), cause);
+ return;
+ }else{
+ logger.error("Exception in RPC communication. Connection: {}. Closing connection.", connection.getName(), cause);
+ ctx.close();
+ }
+ }
+
+ @Override
+ public void handlerAdded(ChannelHandlerContext ctx) throws Exception {
+ }
+
+
+ @Override
+ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMessage.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMessage.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMessage.java
new file mode 100644
index 0000000..9712c9a
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMessage.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
+
+public abstract class RpcMessage {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcMessage.class);
+
+ public RpcMode mode;
+ public int rpcType;
+ public int coordinationId;
+
+ public RpcMessage(RpcMode mode, int rpcType, int coordinationId) {
+ this.mode = mode;
+ this.rpcType = rpcType;
+ this.coordinationId = coordinationId;
+ }
+
+ public abstract int getBodySize();
+ abstract void release();
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcOutcome.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcOutcome.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcOutcome.java
new file mode 100644
index 0000000..af9aa01
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcOutcome.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+public interface RpcOutcome<T> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcOutcome.class);
+
+ public void set(Object value, ByteBuf buffer);
+ public void setException(Throwable t);
+ public Class<T> getOutcomeType();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcOutcomeListener.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcOutcomeListener.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcOutcomeListener.java
new file mode 100644
index 0000000..4485cf9
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcOutcomeListener.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+public interface RpcOutcomeListener<V> {
+
+ /**
+ * Called when an error occurred while waiting for the RPC outcome.
+ * @param ex
+ */
+ void failed(RpcException ex);
+
+
+ void success(V value, ByteBuf buffer);
+
+ /**
+ * Called when the sending thread is interrupted. Possible when the fragment is cancelled due to query cancellations or
+ * failures.
+ */
+ void interrupted(final InterruptedException e);
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
new file mode 100644
index 0000000..cdc441d
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollEventLoopGroup;
+import io.netty.channel.epoll.EpollServerSocketChannel;
+import io.netty.channel.epoll.EpollSocketChannel;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.ServerSocketChannel;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.util.internal.SystemPropertyUtil;
+
+import java.util.Locale;
+
+
+/**
+ * TransportCheck decides whether or not to use the native EPOLL mechanism for communication.
+ */
+public class TransportCheck {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TransportCheck.class);
+
+ private static final String USE_LINUX_EPOLL = "drill.exec.enable-epoll";
+
+ public static final boolean SUPPORTS_EPOLL;
+
+ static{
+
+ String name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.US).trim();
+
+ if (name.startsWith("linux") && SystemPropertyUtil.getBoolean(USE_LINUX_EPOLL, false)) {
+ SUPPORTS_EPOLL = true;
+ } else {
+ SUPPORTS_EPOLL = false;
+ }
+ }
+
+ public static Class<? extends ServerSocketChannel> getServerSocketChannel(){
+ if(SUPPORTS_EPOLL){
+ return EpollServerSocketChannel.class;
+ }else{
+ return NioServerSocketChannel.class;
+ }
+ }
+
+ public static Class<? extends SocketChannel> getClientSocketChannel(){
+ if(SUPPORTS_EPOLL){
+ return EpollSocketChannel.class;
+ }else{
+ return NioSocketChannel.class;
+ }
+ }
+
+ public static EventLoopGroup createEventLoopGroup(int nThreads, String prefix) {
+ if(SUPPORTS_EPOLL){
+ return new EpollEventLoopGroup(nThreads, new NamedThreadFactory(prefix));
+ }else{
+ return new NioEventLoopGroup(nThreads, new NamedThreadFactory(prefix));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/UserRpcException.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/UserRpcException.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/UserRpcException.java
new file mode 100644
index 0000000..1d2cc1a
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/UserRpcException.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+
+public class UserRpcException extends RpcException {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UserRpcException.class);
+
+ private final String message;
+ private final DrillbitEndpoint endpoint;
+ private final Throwable t;
+
+ public UserRpcException(DrillbitEndpoint endpoint, String message, Throwable t) {
+ super(t);
+ this.message = message;
+ this.endpoint = endpoint;
+ this.t = t;
+ }
+
+ public String getUserMessage() {
+ return message;
+ }
+
+ public DrillbitEndpoint getEndpoint() {
+ return endpoint;
+ }
+
+ public Throwable getUserException() {
+ return t;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/pom.xml
----------------------------------------------------------------------
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 5534ad4..1639912 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -33,6 +33,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.drill.memory</groupId>
+ <artifactId>drill-memory-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.drill</groupId>
<artifactId>drill-common</artifactId>
<version>${project.version}</version>
@@ -56,11 +61,6 @@
<artifactId>hppc</artifactId>
<version>0.4.2</version>
</dependency>
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>3.0.1</version>
- </dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java b/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
deleted file mode 100644
index 38eb622..0000000
--- a/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
+++ /dev/null
@@ -1,749 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.util.internal.PlatformDependent;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-import java.nio.charset.Charset;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.drill.exec.memory.Accountor;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.ops.BufferManager;
-import org.apache.drill.exec.util.AssertionUtil;
-
-import com.google.common.base.Preconditions;
-
-public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillBuf.class);
-
- private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
-
- private final ByteBuf b;
- private final long addr;
- private final int offset;
- private final boolean rootBuffer;
- private final AtomicLong rootRefCnt = new AtomicLong(1);
- private volatile BufferAllocator allocator;
- private volatile Accountor acct;
- private volatile int length;
-
- // TODO - cleanup
- // The code is partly shared and partly copy-pasted between
- // these three types. They should be unified under one interface
- // to share code and to remove the hacky code here to use only
- // one of these types at a time and use null checks to find out
- // which.
- private BufferManager bufManager;
-
- public DrillBuf(BufferAllocator allocator, Accountor a, UnsafeDirectLittleEndian b) {
- super(b.maxCapacity());
- this.b = b;
- this.addr = b.memoryAddress();
- this.acct = a;
- this.length = b.capacity();
- this.offset = 0;
- this.rootBuffer = true;
- this.allocator = allocator;
- }
-
- private DrillBuf(BufferAllocator allocator, Accountor a) {
- super(0);
- this.b = new EmptyByteBuf(allocator.getUnderlyingAllocator()).order(ByteOrder.LITTLE_ENDIAN);
- this.allocator = allocator;
- this.acct = a;
- this.length = 0;
- this.addr = 0;
- this.rootBuffer = false;
- this.offset = 0;
- }
-
- /**
- * Special constructor used for RPC ownership transfer. Takes a snapshot slice of the current buf
- * but points directly to the underlying UnsafeLittleEndian buffer. Does this by calling unwrap()
- * twice on the provided DrillBuf and expecting an UnsafeDirectLittleEndian buffer. This operation
- * includes taking a new reference count on the underlying buffer and maintaining returning with a
- * current reference count for itself (masking the underlying reference count).
- * @param allocator
- * @param a Allocator used when users try to receive allocator from buffer.
- * @param b Accountor used for accounting purposes.
- */
- public DrillBuf(BufferAllocator allocator, Accountor a, DrillBuf b) {
- this(allocator, a, getUnderlying(b), b, 0, b.length, true);
- assert b.unwrap().unwrap() instanceof UnsafeDirectLittleEndian;
- b.unwrap().unwrap().retain();
- }
-
-
- private DrillBuf(DrillBuf buffer, int index, int length) {
- this(buffer.allocator, null, buffer, buffer, index, length, false);
- }
-
- private static ByteBuf getUnderlying(DrillBuf b){
- ByteBuf underlying = b.unwrap().unwrap();
- return underlying.slice((int) (b.memoryAddress() - underlying.memoryAddress()), b.length);
- }
- private DrillBuf(BufferAllocator allocator, Accountor a, ByteBuf replacement, DrillBuf buffer, int index, int length, boolean root) {
- super(length);
- if (index < 0 || index > buffer.capacity() - length) {
- throw new IndexOutOfBoundsException(buffer.toString() + ".slice(" + index + ", " + length + ')');
- }
-
- this.length = length;
- writerIndex(length);
-
- this.b = replacement;
- this.addr = buffer.memoryAddress() + index;
- this.offset = index;
- this.acct = a;
- this.length = length;
- this.rootBuffer = root;
- this.allocator = allocator;
- }
-
- public void setBufferManager(BufferManager bufManager) {
- this.bufManager = bufManager;
- }
-
- public BufferAllocator getAllocator() {
- return allocator;
- }
-
- public DrillBuf reallocIfNeeded(int size) {
- if (this.capacity() >= size) {
- return this;
- }
-
- if (bufManager != null) {
- return bufManager.replace(this, size);
- } else {
- throw new UnsupportedOperationException("Realloc is only available in the context of an operator's UDFs");
- }
-
- }
-
- @Override
- public int refCnt() {
- if(rootBuffer){
- return (int) this.rootRefCnt.get();
- }else{
- return b.refCnt();
- }
-
- }
-
- private long addr(int index) {
- return addr + index;
- }
-
- private final void checkIndexD(int index) {
- ensureAccessible();
- if (index < 0 || index >= capacity()) {
- throw new IndexOutOfBoundsException(String.format(
- "index: %d (expected: range(0, %d))", index, capacity()));
- }
- }
-
- private final void checkIndexD(int index, int fieldLength) {
- ensureAccessible();
- if (fieldLength < 0) {
- throw new IllegalArgumentException("length: " + fieldLength + " (expected: >= 0)");
- }
- if (index < 0 || index > capacity() - fieldLength) {
- throw new IndexOutOfBoundsException(String.format(
- "index: %d, length: %d (expected: range(0, %d))", index, fieldLength, capacity()));
- }
- }
-
- /**
- * Allows a function to determine whether not reading a particular string of bytes is valid.
- *
- * Will throw an exception if the memory is not readable for some reason. Only doesn't something in the
- * case that AssertionUtil.BOUNDS_CHECKING_ENABLED is true.
- *
- * @param start The starting position of the bytes to be read.
- * @param end The exclusive endpoint of the bytes to be read.
- */
- public void checkBytes(int start, int end){
- if (BOUNDS_CHECKING_ENABLED) {
- checkIndexD(start, end - start);
- }
- }
-
- private void chk(int index, int width) {
- if (BOUNDS_CHECKING_ENABLED) {
- checkIndexD(index, width);
- }
- }
-
- private void chk(int index) {
- if (BOUNDS_CHECKING_ENABLED) {
- checkIndexD(index);
- }
- }
-
- private void ensure(int width) {
- if (BOUNDS_CHECKING_ENABLED) {
- ensureWritable(width);
- }
- }
-
- public boolean transferAccounting(Accountor target) {
- if (rootBuffer) {
- boolean outcome = acct.transferTo(target, this, length);
- acct = target;
- return outcome;
- } else {
- throw new UnsupportedOperationException();
- }
- }
-
- @Override
- public synchronized boolean release() {
- return release(1);
- }
-
- /**
- * Release the provided number of reference counts. If this is a root buffer, will decrease accounting if the local reference count returns to zero.
- */
- @Override
- public synchronized boolean release(int decrement) {
-
- if(rootBuffer){
- final long newRefCnt = this.rootRefCnt.addAndGet(-decrement);
- Preconditions.checkArgument(newRefCnt > -1, "Buffer has negative reference count.");
- if (newRefCnt == 0) {
- b.release(decrement);
- acct.release(this, length);
- return true;
- }else{
- return false;
- }
- }else{
- return b.release(decrement);
- }
- }
-
- @Override
- public int capacity() {
- return length;
- }
-
- @Override
- public synchronized ByteBuf capacity(int newCapacity) {
- if (rootBuffer) {
- if (newCapacity == length) {
- return this;
- } else if (newCapacity < length) {
- b.capacity(newCapacity);
- int diff = length - b.capacity();
- acct.releasePartial(this, diff);
- this.length = length - diff;
- return this;
- } else {
- throw new UnsupportedOperationException("Accounting byte buf doesn't support increasing allocations.");
- }
- } else {
- throw new UnsupportedOperationException("Non root bufs doen't support changing allocations.");
- }
- }
-
- @Override
- public int maxCapacity() {
- return length;
- }
-
- @Override
- public ByteBufAllocator alloc() {
- return b.alloc();
- }
-
- @Override
- public ByteOrder order() {
- return ByteOrder.LITTLE_ENDIAN;
- }
-
- @Override
- public ByteBuf order(ByteOrder endianness) {
- // if(endianness != ByteOrder.LITTLE_ENDIAN) throw new
- // UnsupportedOperationException("Drill buffers only support little endian.");
- return this;
- }
-
- @Override
- public ByteBuf unwrap() {
- return b;
- }
-
- @Override
- public boolean isDirect() {
- return true;
- }
-
- @Override
- public ByteBuf readBytes(int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf readSlice(int length) {
- ByteBuf slice = slice(readerIndex(), length);
- readerIndex(readerIndex() + length);
- return slice;
- }
-
- @Override
- public ByteBuf copy() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf copy(int index, int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf slice() {
- return slice(readerIndex(), readableBytes());
- }
-
- @Override
- public DrillBuf slice(int index, int length) {
- DrillBuf buf = new DrillBuf(this, index, length);
- buf.writerIndex = length;
- return buf;
- }
-
- @Override
- public DrillBuf duplicate() {
- return new DrillBuf(this, 0, length);
- }
-
- @Override
- public int nioBufferCount() {
- return 1;
- }
-
- @Override
- public ByteBuffer nioBuffer() {
- return nioBuffer(readerIndex(), readableBytes());
- }
-
- @Override
- public ByteBuffer nioBuffer(int index, int length) {
- return b.nioBuffer(offset + index, length);
- }
-
- @Override
- public ByteBuffer internalNioBuffer(int index, int length) {
- return b.internalNioBuffer(offset + index, length);
- }
-
- @Override
- public ByteBuffer[] nioBuffers() {
- return new ByteBuffer[]{nioBuffer()};
- }
-
- @Override
- public ByteBuffer[] nioBuffers(int index, int length) {
- return new ByteBuffer[]{nioBuffer(index, length)};
- }
-
- @Override
- public boolean hasArray() {
- return b.hasArray();
- }
-
- @Override
- public byte[] array() {
- return b.array();
- }
-
- @Override
- public int arrayOffset() {
- return b.arrayOffset();
- }
-
- @Override
- public boolean hasMemoryAddress() {
- return true;
- }
-
- @Override
- public long memoryAddress() {
- return this.addr;
- }
-
- @Override
- public String toString(Charset charset) {
- return toString(readerIndex, readableBytes(), charset);
- }
-
- @Override
- public String toString(int index, int length, Charset charset) {
- if (length == 0) {
- return "";
- }
-
- ByteBuffer nioBuffer;
- if (nioBufferCount() == 1) {
- nioBuffer = nioBuffer(index, length);
- } else {
- nioBuffer = ByteBuffer.allocate(length);
- getBytes(index, nioBuffer);
- nioBuffer.flip();
- }
-
- return ByteBufUtil.decodeString(nioBuffer, charset);
- }
-
- @Override
- public int hashCode() {
- return System.identityHashCode(this);
- }
-
- @Override
- public boolean equals(Object obj) {
- // identity equals only.
- return this == obj;
- }
-
- @Override
- public synchronized ByteBuf retain(int increment) {
- if(rootBuffer){
- this.rootRefCnt.addAndGet(increment);
- }else{
- b.retain(increment);
- }
- return this;
- }
-
- @Override
- public ByteBuf retain() {
- return retain(1);
- }
-
- @Override
- public long getLong(int index) {
- chk(index, 8);
- long v = PlatformDependent.getLong(addr(index));
- return v;
- }
-
- @Override
- public float getFloat(int index) {
- return Float.intBitsToFloat(getInt(index));
- }
-
- @Override
- public double getDouble(int index) {
- return Double.longBitsToDouble(getLong(index));
- }
-
- @Override
- public char getChar(int index) {
- return (char) getShort(index);
- }
-
- @Override
- public long getUnsignedInt(int index) {
- return getInt(index) & 0xFFFFFFFFL;
- }
-
- @Override
- public int getInt(int index) {
- chk(index, 4);
- int v = PlatformDependent.getInt(addr(index));
- return v;
- }
-
- @Override
- public int getUnsignedShort(int index) {
- return getShort(index) & 0xFFFF;
- }
-
- @Override
- public short getShort(int index) {
- chk(index, 2);
- short v = PlatformDependent.getShort(addr(index));
- return v;
- }
-
- @Override
- public ByteBuf setShort(int index, int value) {
- chk(index, 2);
- PlatformDependent.putShort(addr(index), (short) value);
- return this;
- }
-
- @Override
- public ByteBuf setInt(int index, int value) {
- chk(index, 4);
- PlatformDependent.putInt(addr(index), value);
- return this;
- }
-
- @Override
- public ByteBuf setLong(int index, long value) {
- chk(index, 8);
- PlatformDependent.putLong(addr(index), value);
- return this;
- }
-
- @Override
- public ByteBuf setChar(int index, int value) {
- chk(index, 2);
- PlatformDependent.putShort(addr(index), (short) value);
- return this;
- }
-
- @Override
- public ByteBuf setFloat(int index, float value) {
- chk(index, 4);
- PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value));
- return this;
- }
-
- @Override
- public ByteBuf setDouble(int index, double value) {
- chk(index, 8);
- PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value));
- return this;
- }
-
- @Override
- public ByteBuf writeShort(int value) {
- ensure(2);
- PlatformDependent.putShort(addr(writerIndex), (short) value);
- writerIndex += 2;
- return this;
- }
-
- @Override
- public ByteBuf writeInt(int value) {
- ensure(4);
- PlatformDependent.putInt(addr(writerIndex), value);
- writerIndex += 4;
- return this;
- }
-
- @Override
- public ByteBuf writeLong(long value) {
- ensure(8);
- PlatformDependent.putLong(addr(writerIndex), value);
- writerIndex += 8;
- return this;
- }
-
- @Override
- public ByteBuf writeChar(int value) {
- ensure(2);
- PlatformDependent.putShort(addr(writerIndex), (short) value);
- writerIndex += 2;
- return this;
- }
-
- @Override
- public ByteBuf writeFloat(float value) {
- ensure(4);
- PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value));
- writerIndex += 4;
- return this;
- }
-
- @Override
- public ByteBuf writeDouble(double value) {
- ensure(8);
- PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value));
- writerIndex += 8;
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
- b.getBytes(index + offset, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuffer dst) {
- b.getBytes(index + offset, dst);
- return this;
- }
-
- @Override
- public ByteBuf setByte(int index, int value) {
- chk(index, 1);
- PlatformDependent.putByte(addr(index), (byte) value);
- return this;
- }
-
- public void setByte(int index, byte b){
- chk(index, 1);
- PlatformDependent.putByte(addr(index), b);
- }
-
- public void writeByteUnsafe(byte b){
- PlatformDependent.putByte(addr(readerIndex), b);
- readerIndex++;
- }
-
- @Override
- protected byte _getByte(int index) {
- return getByte(index);
- }
-
- @Override
- protected short _getShort(int index) {
- return getShort(index);
- }
-
- @Override
- protected int _getInt(int index) {
- return getInt(index);
- }
-
- @Override
- protected long _getLong(int index) {
- return getLong(index);
- }
-
- @Override
- protected void _setByte(int index, int value) {
- setByte(index, value);
- }
-
- @Override
- protected void _setShort(int index, int value) {
- setShort(index, value);
- }
-
- @Override
- protected void _setMedium(int index, int value) {
- setMedium(index, value);
- }
-
- @Override
- protected void _setInt(int index, int value) {
- setInt(index, value);
- }
-
- @Override
- protected void _setLong(int index, long value) {
- setLong(index, value);
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
- b.getBytes(index + offset, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
- b.getBytes(index + offset, out, length);
- return this;
- }
-
- @Override
- protected int _getUnsignedMedium(int index) {
- long addr = addr(index);
- return (PlatformDependent.getByte(addr) & 0xff) << 16 |
- (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
- PlatformDependent.getByte(addr + 2) & 0xff;
- }
-
- @Override
- public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
- return b.getBytes(index + offset, out, length);
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
- b.setBytes(index + offset, src, srcIndex, length);
- return this;
- }
-
- public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
- if (src.isDirect()) {
- checkIndex(index, length);
- PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this.memoryAddress() + index,
- length);
- } else {
- if (srcIndex == 0 && src.capacity() == length) {
- b.setBytes(index + offset, src);
- } else {
- ByteBuffer newBuf = src.duplicate();
- newBuf.position(srcIndex);
- newBuf.limit(srcIndex + length);
- b.setBytes(index + offset, src);
- }
- }
-
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
- b.setBytes(index + offset, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuffer src) {
- b.setBytes(index + offset, src);
- return this;
- }
-
- @Override
- public int setBytes(int index, InputStream in, int length) throws IOException {
- return b.setBytes(index + offset, in, length);
- }
-
- @Override
- public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
- return b.setBytes(index + offset, in, length);
- }
-
- @Override
- public byte getByte(int index) {
- chk(index, 1);
- return PlatformDependent.getByte(addr(index));
- }
-
- public static DrillBuf getEmpty(BufferAllocator allocator, Accountor a) {
- return new DrillBuf(allocator, a);
- }
-
- public boolean isRootBuffer() {
- return rootBuffer;
- }
-
- @Override
- public void close() {
- release();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java b/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
deleted file mode 100644
index 53ca91c..0000000
--- a/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import org.apache.drill.exec.exception.OutOfMemoryException;
-import org.apache.drill.exec.memory.Accountor;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.LimitConsumer;
-import org.apache.drill.exec.util.Pointer;
-
-class FakeAllocator implements BufferAllocator {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FakeAllocator.class);
-
-
- public static final Accountor FAKE_ACCOUNTOR = new FakeAccountor();
- public static final BufferAllocator FAKE_ALLOCATOR = new FakeAllocator();
-
- @Override
- public DrillBuf buffer(int size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DrillBuf buffer(int minSize, int maxSize) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBufAllocator getUnderlyingAllocator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public BufferAllocator getChildAllocator(LimitConsumer consumer, long initialReservation, long maximumReservation,
- boolean applyFragmentLimit)
- throws OutOfMemoryException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DrillBuf getEmpty() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean takeOwnership(DrillBuf buf) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public PreAllocator getNewPreAllocator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void resetLimits() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setLimit(long l) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getLimit(){
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public long getAllocatedMemory() {
- return 0;
- }
-
- @Override
- public long getPeakMemoryAllocation() {
- return 0;
- }
-
- static class FakeAccountor implements Accountor {
-
- public FakeAccountor() {
- }
-
- @Override
- public boolean transferTo(Accountor target, DrillBuf buf, long size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getAvailable() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getCapacity() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getAllocation() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean reserve(long size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean forceAdditionalReservation(long size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void reserved(long expected, DrillBuf buf) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void releasePartial(DrillBuf buf, long size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void release(DrillBuf buf, long size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean transferIn(DrillBuf buf, long size) {
- return false;
- }
-
- @Override
- public long getPeakMemoryAllocation() {
- return 0;
- }
-
- @Override
- public long resetFragmentLimits() {
- return 0;
- }
-
- @Override
- public void setFragmentLimit(long add) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getFragmentLimit() {
- return 0;
- }
-
-
- }
-
- @Override
- public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut) {
- throw new UnsupportedOperationException();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/io/netty/buffer/LargeBuffer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/LargeBuffer.java b/exec/vector/src/main/java/io/netty/buffer/LargeBuffer.java
deleted file mode 100644
index f1d4842..0000000
--- a/exec/vector/src/main/java/io/netty/buffer/LargeBuffer.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This is basically a complete copy of DuplicatedByteBuf. We copy because we can't override the release methods to keep
- * global track of created Large Buffers.
- */
-public class LargeBuffer extends AbstractByteBuf {
-
- private final AtomicLong hugeBufferSize;
- private final AtomicLong hugeBufferCount;
-
- @Override
- public ByteBuffer nioBuffer(int index, int length) {
- return unwrap().nioBuffer(index, length);
- }
-
- private final ByteBuf buffer;
- private final int initCap;
-
- public LargeBuffer(ByteBuf buffer, AtomicLong hugeBufferSize, AtomicLong hugeBufferCount) {
- super(buffer.maxCapacity());
- initCap = buffer.capacity();
- this.hugeBufferCount = hugeBufferCount;
- this.hugeBufferSize = hugeBufferSize;
-
- if (buffer instanceof LargeBuffer) {
- this.buffer = ((LargeBuffer) buffer).buffer;
- } else {
- this.buffer = buffer;
- }
-
- setIndex(buffer.readerIndex(), buffer.writerIndex());
- }
-
- @Override
- public ByteBuf unwrap() {
- return buffer;
- }
-
- @Override
- public ByteBufAllocator alloc() {
- return buffer.alloc();
- }
-
- @Override
- public ByteOrder order() {
- return buffer.order();
- }
-
- @Override
- public boolean isDirect() {
- return buffer.isDirect();
- }
-
- @Override
- public int capacity() {
- return buffer.capacity();
- }
-
- @Override
- public ByteBuf capacity(int newCapacity) {
- buffer.capacity(newCapacity);
- return this;
- }
-
- @Override
- public boolean hasArray() {
- return buffer.hasArray();
- }
-
- @Override
- public byte[] array() {
- return buffer.array();
- }
-
- @Override
- public int arrayOffset() {
- return buffer.arrayOffset();
- }
-
- @Override
- public boolean hasMemoryAddress() {
- return buffer.hasMemoryAddress();
- }
-
- @Override
- public long memoryAddress() {
- return buffer.memoryAddress();
- }
-
- @Override
- public byte getByte(int index) {
- return _getByte(index);
- }
-
- @Override
- protected byte _getByte(int index) {
- return buffer.getByte(index);
- }
-
- @Override
- public short getShort(int index) {
- return _getShort(index);
- }
-
- @Override
- protected short _getShort(int index) {
- return buffer.getShort(index);
- }
-
- @Override
- public int getUnsignedMedium(int index) {
- return _getUnsignedMedium(index);
- }
-
- @Override
- protected int _getUnsignedMedium(int index) {
- return buffer.getUnsignedMedium(index);
- }
-
- @Override
- public int getInt(int index) {
- return _getInt(index);
- }
-
- @Override
- protected int _getInt(int index) {
- return buffer.getInt(index);
- }
-
- @Override
- public long getLong(int index) {
- return _getLong(index);
- }
-
- @Override
- protected long _getLong(int index) {
- return buffer.getLong(index);
- }
-
- @Override
- public ByteBuf copy(int index, int length) {
- return new LargeBuffer(buffer.copy(index, length), hugeBufferSize, hugeBufferCount);
- }
-
- @Override
- public ByteBuf slice(int index, int length) {
- return new SlicedByteBuf(this, index, length);
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
- buffer.getBytes(index, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
- buffer.getBytes(index, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuffer dst) {
- buffer.getBytes(index, dst);
- return this;
- }
-
- @Override
- public ByteBuf setByte(int index, int value) {
- _setByte(index, value);
- return this;
- }
-
- @Override
- protected void _setByte(int index, int value) {
- buffer.setByte(index, value);
- }
-
- @Override
- public ByteBuf setShort(int index, int value) {
- _setShort(index, value);
- return this;
- }
-
- @Override
- protected void _setShort(int index, int value) {
- buffer.setShort(index, value);
- }
-
- @Override
- public ByteBuf setMedium(int index, int value) {
- _setMedium(index, value);
- return this;
- }
-
- @Override
- protected void _setMedium(int index, int value) {
- buffer.setMedium(index, value);
- }
-
- @Override
- public ByteBuf setInt(int index, int value) {
- _setInt(index, value);
- return this;
- }
-
- @Override
- protected void _setInt(int index, int value) {
- buffer.setInt(index, value);
- }
-
- @Override
- public ByteBuf setLong(int index, long value) {
- _setLong(index, value);
- return this;
- }
-
- @Override
- protected void _setLong(int index, long value) {
- buffer.setLong(index, value);
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
- buffer.setBytes(index, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
- buffer.setBytes(index, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuffer src) {
- buffer.setBytes(index, src);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, OutputStream out, int length)
- throws IOException {
- buffer.getBytes(index, out, length);
- return this;
- }
-
- @Override
- public int getBytes(int index, GatheringByteChannel out, int length)
- throws IOException {
- return buffer.getBytes(index, out, length);
- }
-
- @Override
- public int setBytes(int index, InputStream in, int length)
- throws IOException {
- return buffer.setBytes(index, in, length);
- }
-
- @Override
- public int setBytes(int index, ScatteringByteChannel in, int length)
- throws IOException {
- return buffer.setBytes(index, in, length);
- }
-
- @Override
- public int nioBufferCount() {
- return buffer.nioBufferCount();
- }
-
- @Override
- public ByteBuffer[] nioBuffers(int index, int length) {
- return buffer.nioBuffers(index, length);
- }
-
- @Override
- public ByteBuffer internalNioBuffer(int index, int length) {
- return nioBuffer(index, length);
- }
-
- @Override
- public int forEachByte(int index, int length, ByteBufProcessor processor) {
- return buffer.forEachByte(index, length, processor);
- }
-
- @Override
- public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
- return buffer.forEachByteDesc(index, length, processor);
- }
-
- @Override
- public final int refCnt() {
- return unwrap().refCnt();
- }
-
- @Override
- public final ByteBuf retain() {
- unwrap().retain();
- return this;
- }
-
- @Override
- public final ByteBuf retain(int increment) {
- unwrap().retain(increment);
- return this;
- }
-
- @Override
- public boolean release() {
- return release(1);
- }
-
- @Override
- public boolean release(int decrement) {
- boolean released = unwrap().release(decrement);
- if (released) {
- hugeBufferSize.addAndGet(-initCap);
- hugeBufferCount.decrementAndGet();
- }
- return released;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
deleted file mode 100644
index 5809df5..0000000
--- a/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.util.internal.StringUtil;
-
-import java.lang.reflect.Field;
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.util.AssertionUtil;
-
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
-
-public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PooledByteBufAllocatorL.class);
-
- private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
- private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
-
- private static final String METRIC_PREFIX = "drill.allocator.";
- private final MetricRegistry registry;
- private final AtomicLong hugeBufferSize = new AtomicLong(0);
- private final AtomicLong hugeBufferCount = new AtomicLong(0);
- private final AtomicLong normalBufferSize = new AtomicLong(0);
- private final AtomicLong normalBufferCount = new AtomicLong(0);
-
- private final PoolArena<ByteBuffer>[] directArenas;
- private final MemoryStatusThread statusThread;
- private final Histogram largeBuffersHist;
- private final Histogram normalBuffersHist;
-
- public PooledByteBufAllocatorL(MetricRegistry registry) {
- super(true);
- this.registry = registry;
- try {
- Field f = PooledByteBufAllocator.class.getDeclaredField("directArenas");
- f.setAccessible(true);
- this.directArenas = (PoolArena<ByteBuffer>[]) f.get(this);
- } catch (Exception e) {
- throw new RuntimeException("Failure while initializing allocator. Unable to retrieve direct arenas field.", e);
- }
-
- if (memoryLogger.isTraceEnabled()) {
- statusThread = new MemoryStatusThread();
- statusThread.start();
- } else {
- statusThread = null;
- }
- removeOldMetrics();
-
- registry.register(METRIC_PREFIX + "normal.size", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return normalBufferSize.get();
- }
- });
-
- registry.register(METRIC_PREFIX + "normal.count", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return normalBufferCount.get();
- }
- });
-
- registry.register(METRIC_PREFIX + "huge.size", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return hugeBufferSize.get();
- }
- });
-
- registry.register(METRIC_PREFIX + "huge.count", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return hugeBufferCount.get();
- }
- });
-
- largeBuffersHist = registry.histogram(METRIC_PREFIX + "huge.hist");
- normalBuffersHist = registry.histogram(METRIC_PREFIX + "normal.hist");
-
- }
-
- private synchronized void removeOldMetrics() {
- registry.removeMatching(new MetricFilter() {
- @Override
- public boolean matches(String name, Metric metric) {
- return name.startsWith("drill.allocator.");
- }
-
- });
- }
-
- @Override
- protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
- throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
- }
-
- @Override
- protected UnsafeDirectLittleEndian newDirectBuffer(int initialCapacity, int maxCapacity) {
- PoolThreadCache cache = threadCache.get();
- PoolArena<ByteBuffer> directArena = cache.directArena;
-
- if (directArena != null) {
-
- if (initialCapacity > directArena.chunkSize) {
- // This is beyond chunk size so we'll allocate separately.
- ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity);
-
- hugeBufferCount.incrementAndGet();
- hugeBufferSize.addAndGet(buf.capacity());
- largeBuffersHist.update(buf.capacity());
- // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
- return new UnsafeDirectLittleEndian(new LargeBuffer(buf, hugeBufferSize, hugeBufferCount));
-
- } else {
- // within chunk, use arena.
- ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity);
- if (!(buf instanceof PooledUnsafeDirectByteBuf)) {
- fail();
- }
-
- normalBuffersHist.update(buf.capacity());
- if (AssertionUtil.ASSERT_ENABLED) {
- normalBufferSize.addAndGet(buf.capacity());
- normalBufferCount.incrementAndGet();
- }
-
- return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount, normalBufferSize);
- }
-
- } else {
- throw fail();
- }
- }
-
- private UnsupportedOperationException fail() {
- return new UnsupportedOperationException(
- "Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
- }
-
-
- @Override
- public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
- if (initialCapacity == 0 && maxCapacity == 0) {
- newDirectBuffer(initialCapacity, maxCapacity);
- }
- validate(initialCapacity, maxCapacity);
- return newDirectBuffer(initialCapacity, maxCapacity);
- }
-
- @Override
- public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
- throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
- }
-
-
- private static void validate(int initialCapacity, int maxCapacity) {
- if (initialCapacity < 0) {
- throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)");
- }
- if (initialCapacity > maxCapacity) {
- throw new IllegalArgumentException(String.format(
- "initialCapacity: %d (expected: not greater than maxCapacity(%d)",
- initialCapacity, maxCapacity));
- }
- }
-
- private class MemoryStatusThread extends Thread {
-
- public MemoryStatusThread() {
- super("memory-status-logger");
- this.setDaemon(true);
- this.setName("allocation.logger");
- }
-
- @Override
- public void run() {
- while (true) {
- memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString());
- try {
- Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000);
- } catch (InterruptedException e) {
- return;
- }
-
- }
- }
-
- }
-
- public void checkAndReset() {
- if (hugeBufferCount.get() != 0 || normalBufferCount.get() != 0) {
- StringBuilder buf = new StringBuilder();
- buf.append("Large buffers outstanding: ");
- buf.append(hugeBufferCount.get());
- buf.append(" totaling ");
- buf.append(hugeBufferSize.get());
- buf.append(" bytes.");
- buf.append('\n');
- buf.append("Normal buffers outstanding: ");
- buf.append(normalBufferCount.get());
- buf.append(" totaling ");
- buf.append(normalBufferSize.get());
- buf.append(" bytes.");
- hugeBufferCount.set(0);
- normalBufferCount.set(0);
- hugeBufferSize.set(0);
- normalBufferSize.set(0);
- throw new DrillRuntimeException(buf.toString());
- }
- }
-
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append(directArenas.length);
- buf.append(" direct arena(s):");
- buf.append(StringUtil.NEWLINE);
- for (PoolArena<ByteBuffer> a : directArenas) {
- buf.append(a);
- }
-
- buf.append("Large buffers outstanding: ");
- buf.append(this.hugeBufferCount.get());
- buf.append(" totaling ");
- buf.append(this.hugeBufferSize.get());
- buf.append(" bytes.");
- buf.append('\n');
- buf.append("Normal buffers outstanding: ");
- buf.append(this.normalBufferCount.get());
- buf.append(" totaling ");
- buf.append(this.normalBufferSize.get());
- buf.append(" bytes.");
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/vector/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
deleted file mode 100644
index e332b13..0000000
--- a/exec/vector/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.netty.buffer;
-
-import io.netty.util.internal.PlatformDependent;
-
-import java.nio.ByteOrder;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.drill.exec.util.AssertionUtil;
-
-public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
- private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
- private final AbstractByteBuf wrapped;
- private final long memoryAddress;
- private AtomicLong bufferCount;
- private AtomicLong bufferSize;
- private long initCap = -1;
-
- UnsafeDirectLittleEndian(LargeBuffer buf) {
- this(buf, true);
- }
-
- UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong bufferCount, AtomicLong bufferSize) {
- this(buf, true);
- this.bufferCount = bufferCount;
- this.bufferSize = bufferSize;
-
- // initCap is used if we're tracking memory release. If we're in non-debug mode, we'll skip this.
- this.initCap = AssertionUtil.ASSERT_ENABLED ? capacity() : -1;
- }
-
- private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake) {
- super(buf);
- if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
- throw new IllegalStateException("Drill only runs on LittleEndian systems.");
- }
- wrapped = buf;
- this.memoryAddress = buf.memoryAddress();
- }
- private long addr(int index) {
- return memoryAddress + index;
- }
-
- @Override
- public long getLong(int index) {
-// wrapped.checkIndex(index, 8);
- long v = PlatformDependent.getLong(addr(index));
- return v;
- }
-
- @Override
- public float getFloat(int index) {
- return Float.intBitsToFloat(getInt(index));
- }
-
- @Override
- public ByteBuf slice() {
- return slice(this.readerIndex(), readableBytes());
- }
-
- @Override
- public ByteBuf slice(int index, int length) {
- return new SlicedByteBuf(this, index, length);
- }
-
- @Override
- public ByteOrder order() {
- return ByteOrder.LITTLE_ENDIAN;
- }
-
- @Override
- public ByteBuf order(ByteOrder endianness) {
- return this;
- }
-
- @Override
- public double getDouble(int index) {
- return Double.longBitsToDouble(getLong(index));
- }
-
- @Override
- public char getChar(int index) {
- return (char) getShort(index);
- }
-
- @Override
- public long getUnsignedInt(int index) {
- return getInt(index) & 0xFFFFFFFFL;
- }
-
- @Override
- public int getInt(int index) {
-// wrapped.checkIndex(index, 4);
- int v = PlatformDependent.getInt(addr(index));
- return v;
- }
-
- @Override
- public int getUnsignedShort(int index) {
- return getShort(index) & 0xFFFF;
- }
-
- @Override
- public short getShort(int index) {
-// wrapped.checkIndex(index, 2);
- short v = PlatformDependent.getShort(addr(index));
- return v;
- }
-
- @Override
- public ByteBuf setShort(int index, int value) {
- wrapped.checkIndex(index, 2);
- _setShort(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setInt(int index, int value) {
- wrapped.checkIndex(index, 4);
- _setInt(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setLong(int index, long value) {
- wrapped.checkIndex(index, 8);
- _setLong(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setChar(int index, int value) {
- setShort(index, value);
- return this;
- }
-
- @Override
- public ByteBuf setFloat(int index, float value) {
- setInt(index, Float.floatToRawIntBits(value));
- return this;
- }
-
- @Override
- public ByteBuf setDouble(int index, double value) {
- setLong(index, Double.doubleToRawLongBits(value));
- return this;
- }
-
- @Override
- public ByteBuf writeShort(int value) {
- wrapped.ensureWritable(2);
- _setShort(wrapped.writerIndex, value);
- wrapped.writerIndex += 2;
- return this;
- }
-
- @Override
- public ByteBuf writeInt(int value) {
- wrapped.ensureWritable(4);
- _setInt(wrapped.writerIndex, value);
- wrapped.writerIndex += 4;
- return this;
- }
-
- @Override
- public ByteBuf writeLong(long value) {
- wrapped.ensureWritable(8);
- _setLong(wrapped.writerIndex, value);
- wrapped.writerIndex += 8;
- return this;
- }
-
- @Override
- public ByteBuf writeChar(int value) {
- writeShort(value);
- return this;
- }
-
- @Override
- public ByteBuf writeFloat(float value) {
- writeInt(Float.floatToRawIntBits(value));
- return this;
- }
-
- @Override
- public ByteBuf writeDouble(double value) {
- writeLong(Double.doubleToRawLongBits(value));
- return this;
- }
-
- private void _setShort(int index, int value) {
- PlatformDependent.putShort(addr(index), (short) value);
- }
-
- private void _setInt(int index, int value) {
- PlatformDependent.putInt(addr(index), value);
- }
-
- private void _setLong(int index, long value) {
- PlatformDependent.putLong(addr(index), value);
- }
-
- @Override
- public byte getByte(int index) {
- return PlatformDependent.getByte(addr(index));
- }
-
- @Override
- public ByteBuf setByte(int index, int value) {
- PlatformDependent.putByte(addr(index), (byte) value);
- return this;
- }
-
- @Override
- public boolean release() {
- return release(1);
- }
-
- @Override
- public boolean release(int decrement) {
- boolean released = super.release(decrement);
- if (released && initCap != -1) {
- bufferCount.decrementAndGet();
- bufferSize.addAndGet(-initCap);
- }
- return released;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java b/exec/vector/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
deleted file mode 100644
index 17d099d..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.exception;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-
-public class OutOfMemoryException extends DrillRuntimeException {
- private static final long serialVersionUID = -6858052345185793382L;
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutOfMemoryException.class);
-
- public OutOfMemoryException() {
- super();
- }
-
- public OutOfMemoryException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public OutOfMemoryException(String message, Throwable cause) {
- super(message, cause);
-
- }
-
- public OutOfMemoryException(String message) {
- super(message);
-
- }
-
- public OutOfMemoryException(Throwable cause) {
- super(cause);
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/org/apache/drill/exec/memory/Accountor.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/Accountor.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/Accountor.java
deleted file mode 100644
index 7014a0b..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/memory/Accountor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import io.netty.buffer.DrillBuf;
-
-public interface Accountor extends AutoCloseable{
-
- public boolean transferTo(Accountor target, DrillBuf buf, long size);
- public boolean transferIn(DrillBuf buf, long size);
- public long getAvailable();
- public long getCapacity();
- public long getAllocation();
- public long getPeakMemoryAllocation();
-
- public boolean reserve(long size);
- public boolean forceAdditionalReservation(long size);
-
- public void reserved(long expected, DrillBuf buf);
-
- public void release(DrillBuf buf, long size);
- public void releasePartial(DrillBuf buf, long size);
- public long resetFragmentLimits();
- public void close();
-
- public void setFragmentLimit(long add);
- public long getFragmentLimit();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
deleted file mode 100644
index a708e92..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.DrillBuf;
-
-import java.io.Closeable;
-
-import org.apache.drill.exec.exception.OutOfMemoryException;
-import org.apache.drill.exec.util.Pointer;
-
-/**
- * Wrapper class to deal with byte buffer allocation. Ensures users only use designated methods. Also allows inser
- */
-public interface BufferAllocator extends Closeable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BufferAllocator.class);
-
- /**
- * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
- * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
- *
- * @param size
- * The size in bytes.
- * @return A new ByteBuf.
- * @throws OutOfMemoryException if buffer cannot be allocated
- */
- public abstract DrillBuf buffer(int size);
-
- /**
- * Allocate a new or reused buffer within provided range. Note that the buffer may technically be larger than the
- * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
- *
- * @param minSize The minimum size in bytes.
- * @param maxSize The maximum size in bytes.
- * @return A new ByteBuf.
- * @throws OutOfMemoryException if buffer cannot be allocated
- */
- public abstract DrillBuf buffer(int minSize, int maxSize);
-
- public abstract ByteBufAllocator getUnderlyingAllocator();
-
- /**
- * Create a child allocator nested below this one.
- *
- * @param context
- * - BufferManager associated with the new child allocator
- * @param initialReservation
- * - specified in bytes
- * @param maximumReservation
- * - specified in bytes
- * @param applyFragmentLimit
- * - flag to conditionally enable fragment memory limits
- * @return - a new buffer allocator owned by the parent it was spawned from
- * @throws OutOfMemoryException
- * - when off-heap memory has been exhausted
- */
- public abstract BufferAllocator getChildAllocator(LimitConsumer limitListener, long initialReservation,
- long maximumReservation, boolean applyFragmentLimit) throws OutOfMemoryException;
-
- /**
- * Take over ownership of fragment accounting. Always takes over ownership.
- * @param buf
- * @return false if over allocation.
- */
- public boolean takeOwnership(DrillBuf buf) ;
-
- /**
- * Take over ownership of fragment accounting. Always takes over ownership.
- * @param buf
- * @return false if over allocation.
- */
- public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut);
-
- public PreAllocator getNewPreAllocator();
-
- //public void addFragmentContext(FragmentContext c);
-
- /**
- * For Top Level Allocators. Reset the fragment limits for all allocators
- */
- public void resetLimits();
-
- /**
- * For Child allocators to set the Fragment limit for the corresponding fragment allocator.
- * @param l the new fragment limit
- */
- public void setLimit(long l);
-
- public long getLimit();
-
-
- /**
- * Not thread safe.
- *
- * WARNING: unclaimed pre-allocations leak memory. If you call preAllocate(), you must
- * make sure to ultimately try to get the buffer and release it.
- */
- public interface PreAllocator {
- public boolean preAllocate(int bytes);
-
- public DrillBuf getAllocation();
- }
-
- /**
- * @param bytes
- * @return
- */
-
- /**
- *
- */
-
- /**
- * Close and release all buffers generated from this buffer pool.
- */
- @Override
- public abstract void close();
-
- public abstract long getAllocatedMemory();
-
- public abstract long getPeakMemoryAllocation();
-
- public DrillBuf getEmpty();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
deleted file mode 100644
index 777d9d2..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-public interface LimitConsumer {
-
- public String getIdentifier();
- public long getAllocated();
-
- public long getLimit();
-
- public void setLimit(long limit);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/org/apache/drill/exec/ops/BufferManager.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/ops/BufferManager.java b/exec/vector/src/main/java/org/apache/drill/exec/ops/BufferManager.java
deleted file mode 100644
index 4df8f0e..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/ops/BufferManager.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.ops;
-
-import io.netty.buffer.DrillBuf;
-
-/**
- * Manages a list of {@link DrillBuf}s that can be reallocated as needed. Upon
- * re-allocation the old buffer will be freed. Managing a list of these buffers
- * prevents some parts of the system from needing to define a correct location
- * to place the final call to free them.
- *
- * The current uses of these types of buffers are within the pluggable components of Drill.
- * In UDFs, memory management should not be a concern. We provide access to re-allocatable
- * DrillBufs to give UDF writers general purpose buffers we can account for. To prevent the need
- * for UDFs to contain boilerplate to close all of the buffers they request, this list
- * is tracked at a higher level and all of the buffers are freed once we are sure that
- * the code depending on them is done executing (currently {@link FragmentContext}
- * and {@link QueryContext}.
- */
-public interface BufferManager extends AutoCloseable {
-
- /**
- * Replace an old buffer with a new version at least of the provided size. Does not copy data.
- *
- * @param old
- * Old Buffer that the user is no longer going to use.
- * @param newSize
- * Size of new replacement buffer.
- * @return
- */
- public DrillBuf replace(DrillBuf old, int newSize);
-
- /**
- * Get a managed buffer of indeterminate size.
- *
- * @return A buffer.
- */
- public DrillBuf getManagedBuffer();
-
- /**
- * Get a managed buffer of at least a certain size.
- *
- * @param size
- * The desired size
- * @return A buffer
- */
- public DrillBuf getManagedBuffer(int size);
-
- public void close();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/AssertionUtil.java b/exec/vector/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
deleted file mode 100644
index 2348e62..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-public class AssertionUtil {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
-
-
- public static final boolean ASSERT_ENABLED;
- public static final boolean BOUNDS_CHECKING_ENABLED;
-
- static{
- boolean isAssertEnabled = false;
- assert isAssertEnabled = true;
- ASSERT_ENABLED = isAssertEnabled;
- BOUNDS_CHECKING_ENABLED = ASSERT_ENABLED || !"true".equals(System.getProperty("drill.enable_unsafe_memory_access"));
- }
-
- public static boolean isAssertionsEnabled(){
- return ASSERT_ENABLED;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java b/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
deleted file mode 100644
index 074ed76..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-public class Pointer<T> {
- public T value;
-
- public Pointer(){}
-
- public Pointer(T value){
- this.value = value;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
index 290b237..caedb96 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -17,29 +17,26 @@
*/
package org.apache.drill.exec.vector.complex;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import javax.annotation.Nullable;
-import javax.validation.constraints.DecimalMin.List;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.util.CallBack;
import org.apache.drill.exec.vector.ValueVector;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
/**
* Base class for composite vectors.
*
[41/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
DRILL-3987: (MOVE) Move logical expressions and operators out of common. Move to new drill-logical model.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/44dea433
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/44dea433
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/44dea433
Branch: refs/heads/master
Commit: 44dea433395f00126021e72eb581e0f07e481d0a
Parents: a289fd9
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Nov 8 09:35:45 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 17:00:49 2015 -0800
----------------------------------------------------------------------
.../drill/common/expression/parser/ExprLexer.g | 154 -----
.../drill/common/expression/parser/ExprParser.g | 323 ---------
.../org/apache/drill/common/JSONOptions.java | 184 -----
.../apache/drill/common/SerializedExecutor.java | 144 ----
.../common/config/LogicalPlanPersistence.java | 65 --
.../exceptions/ExecutionSetupException.java | 56 --
.../exceptions/ExpressionParsingException.java | 45 --
.../LogicalOperatorValidationException.java | 44 --
.../exceptions/LogicalPlanParsingException.java | 48 --
.../common/expression/BooleanOperator.java | 66 --
.../drill/common/expression/CastExpression.java | 67 --
.../common/expression/ConvertExpression.java | 85 ---
.../drill/common/expression/ErrorCollector.java | 45 --
.../common/expression/ErrorCollectorImpl.java | 115 ----
.../common/expression/ExpressionFunction.java | 96 ---
.../common/expression/ExpressionPosition.java | 50 --
.../expression/ExpressionStringBuilder.java | 332 ----------
.../drill/common/expression/FieldReference.java | 133 ----
.../drill/common/expression/FunctionCall.java | 81 ---
.../common/expression/FunctionCallFactory.java | 132 ----
.../expression/FunctionHolderExpression.java | 83 ---
.../drill/common/expression/FunctionName.java | 29 -
.../drill/common/expression/IfExpression.java | 163 -----
.../common/expression/LogicalExpression.java | 110 ---
.../expression/LogicalExpressionBase.java | 81 ---
.../drill/common/expression/NullExpression.java | 60 --
.../common/expression/OutputTypeDeterminer.java | 97 ---
.../drill/common/expression/PathSegment.java | 309 ---------
.../drill/common/expression/SchemaPath.java | 290 --------
.../common/expression/TypedNullConstant.java | 54 --
.../common/expression/ValidationError.java | 22 -
.../common/expression/ValueExpressions.java | 664 -------------------
.../common/expression/fn/CastFunctions.java | 151 -----
.../drill/common/expression/fn/FuncHolder.java | 23 -
.../common/expression/fn/package-info.java | 21 -
.../drill/common/expression/package-info.java | 29 -
.../drill/common/expression/types/AtomType.java | 68 --
.../drill/common/expression/types/DataType.java | 147 ----
.../expression/types/DataTypeFactory.java | 25 -
.../common/expression/types/LateBindType.java | 53 --
.../common/expression/types/package-info.java | 21 -
.../visitors/AbstractExprVisitor.java | 176 -----
.../expression/visitors/AggregateChecker.java | 206 ------
.../visitors/ConditionalExprOptimizer.java | 112 ----
.../expression/visitors/ConstantChecker.java | 210 ------
.../common/expression/visitors/ExprVisitor.java | 72 --
.../visitors/ExpressionValidationError.java | 31 -
.../visitors/ExpressionValidationException.java | 38 --
.../visitors/ExpressionValidator.java | 234 -------
.../common/expression/visitors/OpVisitor.java | 24 -
.../expression/visitors/SimpleExprVisitor.java | 158 -----
.../expression/visitors/package-info.java | 24 -
.../drill/common/graph/AdjacencyList.java | 190 ------
.../common/graph/AdjacencyListBuilder.java | 75 ---
.../org/apache/drill/common/graph/Edge.java | 42 --
.../org/apache/drill/common/graph/Graph.java | 68 --
.../apache/drill/common/graph/GraphAlgos.java | 159 -----
.../apache/drill/common/graph/GraphValue.java | 26 -
.../apache/drill/common/graph/GraphVisitor.java | 25 -
.../apache/drill/common/graph/Visitable.java | 22 -
.../apache/drill/common/graph/package-info.java | 21 -
.../common/logical/FormatPluginConfig.java | 40 --
.../common/logical/FormatPluginConfigBase.java | 44 --
.../drill/common/logical/LogicalPlan.java | 122 ----
.../common/logical/LogicalPlanBuilder.java | 56 --
.../drill/common/logical/PlanProperties.java | 121 ----
.../common/logical/StoragePluginConfig.java | 43 --
.../common/logical/StoragePluginConfigBase.java | 37 --
.../common/logical/UnexpectedOperatorType.java | 35 -
.../drill/common/logical/ValidationError.java | 43 --
.../common/logical/data/AbstractBuilder.java | 46 --
.../logical/data/AbstractSingleBuilder.java | 42 --
.../drill/common/logical/data/Filter.java | 53 --
.../drill/common/logical/data/Flatten.java | 67 --
.../common/logical/data/GroupingAggregate.java | 101 ---
.../apache/drill/common/logical/data/Join.java | 140 ----
.../common/logical/data/JoinCondition.java | 55 --
.../apache/drill/common/logical/data/Limit.java | 79 ---
.../common/logical/data/LogicalOperator.java | 56 --
.../logical/data/LogicalOperatorBase.java | 92 ---
.../common/logical/data/NamedExpression.java | 52 --
.../apache/drill/common/logical/data/Order.java | 252 -------
.../drill/common/logical/data/Project.java | 92 ---
.../common/logical/data/RunningAggregate.java | 63 --
.../apache/drill/common/logical/data/Scan.java | 78 ---
.../drill/common/logical/data/Sequence.java | 187 ------
.../logical/data/SingleInputOperator.java | 53 --
.../drill/common/logical/data/SinkOperator.java | 30 -
.../common/logical/data/SourceOperator.java | 35 -
.../apache/drill/common/logical/data/Store.java | 85 ---
.../drill/common/logical/data/Transform.java | 54 --
.../apache/drill/common/logical/data/Union.java | 94 ---
.../drill/common/logical/data/Values.java | 69 --
.../drill/common/logical/data/Window.java | 120 ----
.../drill/common/logical/data/Writer.java | 61 --
.../drill/common/logical/data/package-info.java | 21 -
.../data/visitors/AbstractLogicalVisitor.java | 126 ----
.../logical/data/visitors/LogicalVisitor.java | 66 --
.../logical/data/visitors/package-info.java | 21 -
.../drill/common/logical/package-info.java | 25 -
.../drill/common/util/AbstractDynamicBean.java | 81 ---
.../java/org/apache/drill/common/util/Hook.java | 79 ---
.../common/expression/PathSegmentTests.java | 46 --
.../common/expression/parser/TreeTest.java | 118 ----
.../drill/common/logical/data/OrderTest.java | 93 ---
.../drill/storage/CheckStorageConfig.java | 51 --
exec/java-exec/pom.xml | 5 +
.../apache/drill/common/SerializedExecutor.java | 144 ++++
exec/vector/pom.xml | 24 +-
logical/pom.xml | 163 +++++
.../drill/common/expression/parser/ExprLexer.g | 154 +++++
.../drill/common/expression/parser/ExprParser.g | 323 +++++++++
.../org/apache/drill/common/JSONOptions.java | 184 +++++
.../common/config/LogicalPlanPersistence.java | 65 ++
.../exceptions/ExecutionSetupException.java | 56 ++
.../exceptions/ExpressionParsingException.java | 45 ++
.../LogicalOperatorValidationException.java | 44 ++
.../exceptions/LogicalPlanParsingException.java | 48 ++
.../common/expression/BooleanOperator.java | 66 ++
.../drill/common/expression/CastExpression.java | 67 ++
.../common/expression/ConvertExpression.java | 85 +++
.../drill/common/expression/ErrorCollector.java | 45 ++
.../common/expression/ErrorCollectorImpl.java | 115 ++++
.../common/expression/ExpressionFunction.java | 96 +++
.../common/expression/ExpressionPosition.java | 50 ++
.../expression/ExpressionStringBuilder.java | 332 ++++++++++
.../drill/common/expression/FieldReference.java | 133 ++++
.../drill/common/expression/FunctionCall.java | 81 +++
.../common/expression/FunctionCallFactory.java | 132 ++++
.../expression/FunctionHolderExpression.java | 83 +++
.../drill/common/expression/FunctionName.java | 29 +
.../drill/common/expression/IfExpression.java | 163 +++++
.../common/expression/LogicalExpression.java | 110 +++
.../expression/LogicalExpressionBase.java | 81 +++
.../drill/common/expression/NullExpression.java | 60 ++
.../common/expression/OutputTypeDeterminer.java | 97 +++
.../drill/common/expression/PathSegment.java | 309 +++++++++
.../drill/common/expression/SchemaPath.java | 290 ++++++++
.../common/expression/TypedNullConstant.java | 54 ++
.../common/expression/ValidationError.java | 22 +
.../common/expression/ValueExpressions.java | 664 +++++++++++++++++++
.../common/expression/fn/CastFunctions.java | 151 +++++
.../drill/common/expression/fn/FuncHolder.java | 23 +
.../common/expression/fn/package-info.java | 21 +
.../drill/common/expression/package-info.java | 29 +
.../drill/common/expression/types/AtomType.java | 68 ++
.../drill/common/expression/types/DataType.java | 147 ++++
.../expression/types/DataTypeFactory.java | 25 +
.../common/expression/types/LateBindType.java | 53 ++
.../common/expression/types/package-info.java | 21 +
.../visitors/AbstractExprVisitor.java | 176 +++++
.../expression/visitors/AggregateChecker.java | 206 ++++++
.../visitors/ConditionalExprOptimizer.java | 112 ++++
.../expression/visitors/ConstantChecker.java | 210 ++++++
.../common/expression/visitors/ExprVisitor.java | 72 ++
.../visitors/ExpressionValidationError.java | 31 +
.../visitors/ExpressionValidationException.java | 38 ++
.../visitors/ExpressionValidator.java | 234 +++++++
.../common/expression/visitors/OpVisitor.java | 24 +
.../expression/visitors/SimpleExprVisitor.java | 158 +++++
.../expression/visitors/package-info.java | 24 +
.../drill/common/graph/AdjacencyList.java | 190 ++++++
.../common/graph/AdjacencyListBuilder.java | 75 +++
.../org/apache/drill/common/graph/Edge.java | 42 ++
.../org/apache/drill/common/graph/Graph.java | 68 ++
.../apache/drill/common/graph/GraphAlgos.java | 159 +++++
.../apache/drill/common/graph/GraphValue.java | 26 +
.../apache/drill/common/graph/GraphVisitor.java | 25 +
.../apache/drill/common/graph/Visitable.java | 22 +
.../apache/drill/common/graph/package-info.java | 21 +
.../common/logical/FormatPluginConfig.java | 40 ++
.../common/logical/FormatPluginConfigBase.java | 44 ++
.../drill/common/logical/LogicalPlan.java | 122 ++++
.../common/logical/LogicalPlanBuilder.java | 56 ++
.../drill/common/logical/PlanProperties.java | 121 ++++
.../common/logical/StoragePluginConfig.java | 43 ++
.../common/logical/StoragePluginConfigBase.java | 37 ++
.../common/logical/UnexpectedOperatorType.java | 35 +
.../drill/common/logical/ValidationError.java | 43 ++
.../common/logical/data/AbstractBuilder.java | 46 ++
.../logical/data/AbstractSingleBuilder.java | 42 ++
.../drill/common/logical/data/Filter.java | 53 ++
.../drill/common/logical/data/Flatten.java | 67 ++
.../common/logical/data/GroupingAggregate.java | 101 +++
.../apache/drill/common/logical/data/Join.java | 140 ++++
.../common/logical/data/JoinCondition.java | 55 ++
.../apache/drill/common/logical/data/Limit.java | 79 +++
.../common/logical/data/LogicalOperator.java | 56 ++
.../logical/data/LogicalOperatorBase.java | 92 +++
.../common/logical/data/NamedExpression.java | 52 ++
.../apache/drill/common/logical/data/Order.java | 252 +++++++
.../drill/common/logical/data/Project.java | 92 +++
.../common/logical/data/RunningAggregate.java | 63 ++
.../apache/drill/common/logical/data/Scan.java | 78 +++
.../drill/common/logical/data/Sequence.java | 187 ++++++
.../logical/data/SingleInputOperator.java | 53 ++
.../drill/common/logical/data/SinkOperator.java | 30 +
.../common/logical/data/SourceOperator.java | 35 +
.../apache/drill/common/logical/data/Store.java | 85 +++
.../drill/common/logical/data/Transform.java | 54 ++
.../apache/drill/common/logical/data/Union.java | 94 +++
.../drill/common/logical/data/Values.java | 69 ++
.../drill/common/logical/data/Window.java | 120 ++++
.../drill/common/logical/data/Writer.java | 61 ++
.../drill/common/logical/data/package-info.java | 21 +
.../data/visitors/AbstractLogicalVisitor.java | 126 ++++
.../logical/data/visitors/LogicalVisitor.java | 66 ++
.../logical/data/visitors/package-info.java | 21 +
.../drill/common/logical/package-info.java | 25 +
.../drill/common/util/AbstractDynamicBean.java | 81 +++
.../common/expression/PathSegmentTests.java | 46 ++
.../common/expression/parser/TreeTest.java | 118 ++++
.../drill/common/logical/data/OrderTest.java | 93 +++
.../drill/storage/CheckStorageConfig.java | 51 ++
214 files changed, 10109 insertions(+), 10008 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
deleted file mode 100644
index a881b5c..0000000
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
+++ /dev/null
@@ -1,154 +0,0 @@
-lexer grammar ExprLexer;
-
-options {
- language=Java;
-}
-
-@header {
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.common.expression.parser;
-}
-
-If : 'if';
-Else : 'else';
-Return : 'return';
-Then : 'then';
-End : 'end';
-In : 'in';
-Case : 'case';
-When : 'when';
-
-Cast: 'cast';
-Convert : 'convert_' ('from' | 'to');
-Nullable: 'nullable';
-Repeat: 'repeat';
-As: 'as';
-
-BIT : 'bit' | 'BIT';
-INT : 'int' | 'INT';
-BIGINT : 'bigint' | 'BIGINT';
-FLOAT4 : 'float4' | 'FLOAT4';
-FLOAT8 : 'float8' | 'FLOAT8';
-VARCHAR : 'varchar' | 'VARCHAR';
-VARBINARY: 'varbinary' | 'VARBINARY';
-DATE : 'date' | 'DATE';
-TIMESTAMP: 'timestamp' | 'TIMESTAMP';
-TIME : 'time' | 'TIME';
-TIMESTAMPTZ: 'timestamptz' | 'TIMESTAMPTZ';
-INTERVAL : 'interval' | 'INTERVAL';
-INTERVALYEAR : 'intervalyear' | 'INTERVALYEAR';
-INTERVALDAY : 'intervalday' | 'INTERVALDAY';
-Period : '.';
-DECIMAL9 : 'decimal9' | 'DECIMAL9';
-DECIMAL18 : 'decimal18' | 'DECIMAL18';
-DECIMAL28DENSE : 'decimal28dense' | 'DECIMAL28DENSE';
-DECIMAL28SPARSE : 'decimal28sparse' | 'DECIMAL28SPARSE';
-DECIMAL38DENSE : 'decimal38dense' | 'DECIMAL38DENSE';
-DECIMAL38SPARSE : 'decimal38sparse' | 'DECIMAL38SPARSE';
-Or : 'or' | 'OR' | 'Or';
-And : 'and' | 'AND' ;
-Equals : '==' | '=';
-NEquals : '<>' | '!=';
-GTEquals : '>=';
-LTEquals : '<=';
-Caret : '^';
-Excl : '!';
-GT : '>';
-LT : '<';
-Plus : '+';
-Minus : '-';
-Asterisk : '*';
-ForwardSlash : '/';
-Percent : '%';
-OBrace : '{';
-CBrace : '}';
-OBracket : '[';
-CBracket : ']';
-OParen : '(';
-CParen : ')';
-SColon : ';';
-Comma : ',';
-QMark : '?';
-Colon : ':';
-SingleQuote: '\'';
-
-Bool
- : 'true'
- | 'false'
- ;
-
-Number
- : Int ('.' Digit*)? (('e' | 'E') ('+' | '-')? Digit*)?
- ;
-
-//Float
-// : Int ('.' Digit*)? ('e' ('+' | '-')? Digit*)?
-// ;
-//
-//Integer
-// : Digit Digit*
-// ;
-
-Identifier
- : ('a'..'z' | 'A'..'Z' | '_' | '$') ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)*
- ;
-
-QuotedIdentifier
-@after {
- setText(getText().substring(1, getText().length()-1).replaceAll("\\\\(.)", "$1"));
-}
- : '`' (~('`' | '\\') | '\\' ('\\' | '`'))* '`'
- ;
-
-String
-@after {
- setText(getText().substring(1, getText().length()-1).replaceAll("\\\\(.)", "$1"));
-}
- : '\'' (~('\'' | '\\') | '\\' ('\\' | '\''))* '\''
- ;
-
-Comment
- : '//' ~('\r' | '\n')* {skip();}
- | '/*' .* '*/' {skip();}
- ;
-
-Space
- : (' ' | '\t' | '\r' | '\n' | '\u000C') {skip();}
- ;
-
-fragment Int
- : '1'..'9' Digit*
- | '0'
- ;
-
-fragment Digit
- : '0'..'9'
- ;
-
-FallThrough
- @after{
- throw new RuntimeException(java.lang.String.format(
- "Encountered an illegal char on line \%d, column \%d: '\%s'",
- getLine(), getCharPositionInLine(), getText()
- )
- );
- }
- :
- ;
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
deleted file mode 100644
index 600b791..0000000
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
+++ /dev/null
@@ -1,323 +0,0 @@
-parser grammar ExprParser;
-
-options{
- output=AST;
- language=Java;
- tokenVocab=ExprLexer;
- backtrack=true;
- memoize=true;
-}
-
-
-
-@header {
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.common.expression.parser;
-
-//Explicit import...
-import org.antlr.runtime.BitSet;
-import java.util.*;
-import org.apache.drill.common.expression.*;
-import org.apache.drill.common.expression.PathSegment.NameSegment;
-import org.apache.drill.common.expression.PathSegment.ArraySegment;
-import org.apache.drill.common.types.*;
-import org.apache.drill.common.types.TypeProtos.*;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.exceptions.ExpressionParsingException;
-}
-
-@members{
- private String fullExpression;
- private int tokenPos;
-
- public static void p(String s){
- System.out.println(s);
- }
-
- public ExpressionPosition pos(Token token){
- return new ExpressionPosition(fullExpression, token.getTokenIndex());
- }
-
- @Override
- public void displayRecognitionError(String[] tokenNames, RecognitionException e) {
- String hdr = getErrorHeader(e);
- String msg = getErrorMessage(e, tokenNames);
- throw new ExpressionParsingException("Expression has syntax error! " + hdr + ":" + msg);
- }
-}
-
-parse returns [LogicalExpression e]
- : expression EOF {
- $e = $expression.e;
- if(fullExpression == null) fullExpression = $expression.text;
- tokenPos = $expression.start.getTokenIndex();
- }
- ;
-
-functionCall returns [LogicalExpression e]
- : Identifier OParen exprList? CParen {$e = FunctionCallFactory.createExpression($Identifier.text, pos($Identifier), $exprList.listE); }
- ;
-
-convertCall returns [LogicalExpression e]
- : Convert OParen expression Comma String CParen
- { $e = FunctionCallFactory.createConvert($Convert.text, $String.text, $expression.e, pos($Convert));}
- ;
-
-castCall returns [LogicalExpression e]
- @init{
- List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
- ExpressionPosition p = null;
- }
- : Cast OParen expression As dataType repeat? CParen
- { if ($repeat.isRep!=null && $repeat.isRep.compareTo(Boolean.TRUE)==0)
- $e = FunctionCallFactory.createCast(TypeProtos.MajorType.newBuilder().mergeFrom($dataType.type).setMode(DataMode.REPEATED).build(), pos($Cast), $expression.e);
- else
- $e = FunctionCallFactory.createCast($dataType.type, pos($Cast), $expression.e);}
- ;
-
-repeat returns [Boolean isRep]
- : Repeat { $isRep = Boolean.TRUE;}
- ;
-
-dataType returns [MajorType type]
- : numType {$type =$numType.type;}
- | charType {$type =$charType.type;}
- | dateType {$type =$dateType.type;}
- | booleanType {$type =$booleanType.type;}
- ;
-
-booleanType returns [MajorType type]
- : BIT { $type = Types.required(TypeProtos.MinorType.BIT); }
- ;
-
-numType returns [MajorType type]
- : INT { $type = Types.required(TypeProtos.MinorType.INT); }
- | BIGINT { $type = Types.required(TypeProtos.MinorType.BIGINT); }
- | FLOAT4 { $type = Types.required(TypeProtos.MinorType.FLOAT4); }
- | FLOAT8 { $type = Types.required(TypeProtos.MinorType.FLOAT8); }
- | DECIMAL9 OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL9).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
- | DECIMAL18 OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL18).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
- | DECIMAL28DENSE OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL28DENSE).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
- | DECIMAL28SPARSE OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL28SPARSE).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
- | DECIMAL38DENSE OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL38DENSE).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
- | DECIMAL38SPARSE OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL38SPARSE).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
- ;
-
-charType returns [MajorType type]
- : VARCHAR typeLen {$type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.VARCHAR).setMode(DataMode.REQUIRED).setWidth($typeLen.length.intValue()).build(); }
- | VARBINARY typeLen {$type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.VARBINARY).setMode(DataMode.REQUIRED).setWidth($typeLen.length.intValue()).build();}
- ;
-
-precision returns [Integer value]
- : Number {$value = Integer.parseInt($Number.text); }
- ;
-
-scale returns [Integer value]
- : Number {$value = Integer.parseInt($Number.text); }
- ;
-
-dateType returns [MajorType type]
- : DATE { $type = Types.required(TypeProtos.MinorType.DATE); }
- | TIMESTAMP { $type = Types.required(TypeProtos.MinorType.TIMESTAMP); }
- | TIME { $type = Types.required(TypeProtos.MinorType.TIME); }
- | TIMESTAMPTZ { $type = Types.required(TypeProtos.MinorType.TIMESTAMPTZ); }
- | INTERVAL { $type = Types.required(TypeProtos.MinorType.INTERVAL); }
- | INTERVALYEAR { $type = Types.required(TypeProtos.MinorType.INTERVALYEAR); }
- | INTERVALDAY { $type = Types.required(TypeProtos.MinorType.INTERVALDAY); }
- ;
-
-typeLen returns [Integer length]
- : OParen Number CParen {$length = Integer.parseInt($Number.text);}
- ;
-
-ifStatement returns [LogicalExpression e]
- @init {
- IfExpression.Builder s = IfExpression.newBuilder();
- }
- @after {
- $e = s.build();
- }
- : i1=ifStat {s.setIfCondition($i1.i); s.setPosition(pos($i1.start)); } (elseIfStat { s.setIfCondition($elseIfStat.i); } )* Else expression { s.setElse($expression.e); }End
- ;
-
-ifStat returns [IfExpression.IfCondition i]
- : If e1=expression Then e2=expression { $i = new IfExpression.IfCondition($e1.e, $e2.e); }
- ;
-elseIfStat returns [IfExpression.IfCondition i]
- : Else If e1=expression Then e2=expression { $i = new IfExpression.IfCondition($e1.e, $e2.e); }
- ;
-
-caseStatement returns [LogicalExpression e]
- @init {
- IfExpression.Builder s = IfExpression.newBuilder();
- }
- @after {
- $e = s.build();
- }
- : Case (caseWhenStat {s.setIfCondition($caseWhenStat.e); }) + caseElseStat { s.setElse($caseElseStat.e); } End
- ;
-
-caseWhenStat returns [IfExpression.IfCondition e]
- : When e1=expression Then e2=expression {$e = new IfExpression.IfCondition($e1.e, $e2.e); }
- ;
-
-caseElseStat returns [LogicalExpression e]
- : Else expression {$e = $expression.e; }
- ;
-
-exprList returns [List<LogicalExpression> listE]
- @init{
- $listE = new ArrayList<LogicalExpression>();
- }
- : e1=expression {$listE.add($e1.e); } (Comma e2=expression {$listE.add($e2.e); } )*
- ;
-
-expression returns [LogicalExpression e]
- : ifStatement {$e = $ifStatement.e; }
- | caseStatement {$e = $caseStatement.e; }
- | condExpr {$e = $condExpr.e; }
- ;
-
-condExpr returns [LogicalExpression e]
- : orExpr {$e = $orExpr.e; }
- ;
-
-orExpr returns [LogicalExpression e]
- @init{
- List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
- ExpressionPosition p = null;
- }
- @after{
- if(exprs.size() == 1){
- $e = exprs.get(0);
- }else{
- $e = FunctionCallFactory.createBooleanOperator("or", p, exprs);
- }
- }
- : a1=andExpr { exprs.add($a1.e); p = pos( $a1.start );} (Or a2=andExpr { exprs.add($a2.e); })*
- ;
-
-andExpr returns [LogicalExpression e]
- @init{
- List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
- ExpressionPosition p = null;
- }
- @after{
- if(exprs.size() == 1){
- $e = exprs.get(0);
- }else{
- $e = FunctionCallFactory.createBooleanOperator("and", p, exprs);
- }
- }
- : e1=equExpr { exprs.add($e1.e); p = pos( $e1.start ); } ( And e2=equExpr { exprs.add($e2.e); })*
- ;
-
-equExpr returns [LogicalExpression e]
- @init{
- List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
- List<String> cmps = new ArrayList();
- ExpressionPosition p = null;
- }
- @after{
- $e = FunctionCallFactory.createByOp(exprs, p, cmps);
- }
- : r1=relExpr { exprs.add($r1.e); p = pos( $r1.start );
- } ( cmpr= ( Equals | NEquals ) r2=relExpr {exprs.add($r2.e); cmps.add($cmpr.text); })*
- ;
-
-relExpr returns [LogicalExpression e]
- : left=addExpr {$e = $left.e; } (cmpr = (GTEquals | LTEquals | GT | LT) right=addExpr {$e = FunctionCallFactory.createExpression($cmpr.text, pos($left.start), $left.e, $right.e); } )?
- ;
-
-addExpr returns [LogicalExpression e]
- @init{
- List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
- List<String> ops = new ArrayList();
- ExpressionPosition p = null;
- }
- @after{
- $e = FunctionCallFactory.createByOp(exprs, p, ops);
- }
- : m1=mulExpr {exprs.add($m1.e); p = pos($m1.start); } ( op=(Plus|Minus) m2=mulExpr {exprs.add($m2.e); ops.add($op.text); })*
- ;
-
-mulExpr returns [LogicalExpression e]
- @init{
- List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
- List<String> ops = new ArrayList();
- ExpressionPosition p = null;
- }
- @after{
- $e = FunctionCallFactory.createByOp(exprs, p, ops);
- }
- : p1=xorExpr {exprs.add($p1.e); p = pos($p1.start);} (op=(Asterisk|ForwardSlash|Percent) p2=xorExpr {exprs.add($p2.e); ops.add($op.text); } )*
- ;
-
-xorExpr returns [LogicalExpression e]
- @init{
- List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
- List<String> ops = new ArrayList();
- ExpressionPosition p = null;
- }
- @after{
- $e = FunctionCallFactory.createByOp(exprs, p, ops);
- }
- : u1=unaryExpr {exprs.add($u1.e); p = pos($u1.start);} (Caret u2=unaryExpr {exprs.add($u2.e); ops.add($Caret.text);} )*
- ;
-
-unaryExpr returns [LogicalExpression e]
- : sign=(Plus|Minus)? Number {$e = ValueExpressions.getNumericExpression($sign.text, $Number.text, pos(($sign != null) ? $sign : $Number)); }
- | Minus atom {$e = FunctionCallFactory.createExpression("u-", pos($Minus), $atom.e); }
- | Excl atom {$e= FunctionCallFactory.createExpression("!", pos($Excl), $atom.e); }
- | atom {$e = $atom.e; }
- ;
-
-atom returns [LogicalExpression e]
- : Bool {$e = new ValueExpressions.BooleanExpression($Bool.text, pos($Bool)); }
- | lookup {$e = $lookup.e; }
- ;
-
-pathSegment returns [NameSegment seg]
- : s1=nameSegment {$seg = $s1.seg;}
- ;
-
-nameSegment returns [NameSegment seg]
- : QuotedIdentifier ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new NameSegment($QuotedIdentifier.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
- | Identifier ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new NameSegment($Identifier.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
- ;
-
-arraySegment returns [PathSegment seg]
- : OBracket Number CBracket ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new ArraySegment($Number.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
- ;
-
-
-lookup returns [LogicalExpression e]
- : functionCall {$e = $functionCall.e ;}
- | convertCall {$e = $convertCall.e; }
- | castCall {$e = $castCall.e; }
- | pathSegment {$e = new SchemaPath($pathSegment.seg, pos($pathSegment.start) ); }
- | String {$e = new ValueExpressions.QuotedString($String.text, pos($String) ); }
- | OParen expression CParen {$e = $expression.e; }
- | SingleQuote Identifier SingleQuote {$e = new SchemaPath($Identifier.text, pos($Identifier) ); }
- ;
-
-
-
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/JSONOptions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/JSONOptions.java b/common/src/main/java/org/apache/drill/common/JSONOptions.java
deleted file mode 100644
index e432135..0000000
--- a/common/src/main/java/org/apache/drill/common/JSONOptions.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common;
-
-import java.io.IOException;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
-
-import org.apache.drill.common.JSONOptions.De;
-import org.apache.drill.common.JSONOptions.Se;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.config.LogicalPlanPersistence;
-import org.apache.drill.common.exceptions.LogicalPlanParsingException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonLocation;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.TreeNode;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.node.TreeTraversingParser;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-import com.google.common.base.Preconditions;
-
-@JsonSerialize(using = Se.class)
-@JsonDeserialize(using = De.class)
-public class JSONOptions {
-
- private final static Logger logger = LoggerFactory.getLogger(JSONOptions.class);
-
- private JsonNode root;
- private JsonLocation location;
- private Object opaque;
-
- public JSONOptions(Object opaque) {
- this.opaque = opaque;
- }
-
- public JSONOptions(JsonNode n, JsonLocation location) {
- this.root = n;
- this.location = location;
- }
-
- @SuppressWarnings("unchecked")
- public <T> T getWith(LogicalPlanPersistence lpPersistance, Class<T> c) {
- try {
- if (opaque != null) {
- final Class<?> opaqueClass = opaque.getClass();
- if (opaqueClass.equals(c)) {
- return (T) opaque;
- } else {
- // Enum values that override methods are given $1, $2 ... extensions. Ignore the extension.
- // e.g. SystemTable$1 for SystemTable.OPTION
- if (c.isEnum()) {
- final String opaqueName = opaqueClass.getName().replaceAll("\\$\\d+$", "");
- final String cName = c.getName();
- if(opaqueName.equals(cName)) {
- return (T) opaque;
- }
- }
- throw new IllegalArgumentException(String.format("Attempted to retrieve a option with type of %s. " +
- "However, the JSON options carried an opaque value of type %s.", c.getName(), opaqueClass.getName()));
- }
- }
-
- //logger.debug("Read tree {}", root);
- return lpPersistance.getMapper().treeToValue(root, c);
- } catch (JsonProcessingException e) {
- throw new LogicalPlanParsingException(String.format("Failure while trying to convert late bound " +
- "json options to type of %s. Reference was originally located at line %d, column %d.",
- c.getCanonicalName(), location.getLineNr(), location.getColumnNr()), e);
- }
- }
-
- public <T> T getListWith(LogicalPlanPersistence config, TypeReference<T> t) throws IOException {
- return getListWith(config.getMapper(), t);
- }
-
- public JsonNode asNode(){
- Preconditions.checkArgument(this.root != null, "Attempted to grab JSONOptions as JsonNode when no root node was stored. You can only convert non-opaque JSONOptions values to JsonNodes.");
- return root;
- }
-
- public JsonParser asParser(){
- Preconditions.checkArgument(this.root != null, "Attempted to grab JSONOptions as Parser when no root node was stored. You can only convert non-opaque JSONOptions values to parsers.");
- return new TreeTraversingParser(root);
- }
-
- @SuppressWarnings("unchecked")
- public <T> T getListWith(ObjectMapper mapper, TypeReference<T> t) throws IOException {
- if (opaque != null) {
- Type c = t.getType();
- if (c instanceof ParameterizedType) {
- c = ((ParameterizedType)c).getRawType();
- }
- if ( c.equals(opaque.getClass())) {
- return (T) opaque;
- } else {
- throw new IOException(String.format("Attempted to retrieve a list with type of %s. However, the JSON " +
- "options carried an opaque value of type %s.", t.getType(), opaque.getClass().getName()));
- }
- }
- if (root == null) {
- return null;
- }
- return mapper.treeAsTokens(root).readValueAs(t);
- }
-
- public JsonNode path(String name) {
- return root.path(name);
- }
-
- public JsonNode getRoot() {
- return root;
- }
-
- public static class De extends StdDeserializer<JSONOptions> {
-
- public De() {
- super(JSONOptions.class);
- logger.debug("Creating Deserializer.");
- }
-
- @Override
- public JSONOptions deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
- JsonProcessingException {
- JsonLocation l = jp.getTokenLocation();
-// logger.debug("Reading tree.");
- TreeNode n = jp.readValueAsTree();
-// logger.debug("Tree {}", n);
- if (n instanceof JsonNode) {
- return new JSONOptions( (JsonNode) n, l);
- } else {
- throw new IllegalArgumentException(String.format("Received something other than a JsonNode %s", n));
- }
- }
-
- }
-
- public static class Se extends StdSerializer<JSONOptions> {
-
- public Se() {
- super(JSONOptions.class);
- }
-
- @Override
- public void serialize(JSONOptions value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
- JsonGenerationException {
- if (value.opaque != null) {
- jgen.writeObject(value.opaque);
- } else {
- jgen.writeTree(value.root);
- }
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/SerializedExecutor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/SerializedExecutor.java b/common/src/main/java/org/apache/drill/common/SerializedExecutor.java
deleted file mode 100644
index 6a3a823..0000000
--- a/common/src/main/java/org/apache/drill/common/SerializedExecutor.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common;
-
-import io.netty.util.Recycler;
-import io.netty.util.Recycler.Handle;
-
-import java.util.LinkedList;
-import java.util.concurrent.Executor;
-
-/**
- * Serializes execution of multiple submissions to a single target, while still
- * using a thread pool to execute those submissions. Provides an implicit
- * queueing capability for a single target that requires any commands that
- * execute against it to be serialized.
- */
-public abstract class SerializedExecutor implements Executor {
-
- private final Recycler<RunnableProcessor> processors = new Recycler<RunnableProcessor>() {
- @Override
- protected RunnableProcessor newObject(Handle handle) {
- return new RunnableProcessor(handle);
- }
- };
-
- private boolean isProcessing = false;
- private final LinkedList<Runnable> queuedRunnables = new LinkedList<>();
- private final Executor underlyingExecutor;
- private final String name;
-
- /**
- * Constructor.
- *
- * @param underlyingExecutor
- * underlying executor to use to execute commands submitted to this
- * SerializedExecutor
- */
- public SerializedExecutor(String name, Executor underlyingExecutor) {
- this.underlyingExecutor = underlyingExecutor;
- this.name = name;
- }
-
- /**
- * An exception occurred in the last command executed; this reports that to
- * the subclass of SerializedExecutor.
- *
- * <p>
- * The default implementation of this method throws an exception, which is
- * considered an error (see below). Implementors have two alternatives:
- * Arrange not to throw from your commands' run(), or if they do, provide an
- * override of this method that handles any exception that is thrown.
- * </p>
- *
- * <p>
- * It is an error for this to throw an exception, and doing so will terminate
- * the thread with an IllegalStateException. Derivers must handle any reported
- * exceptions in other ways.
- * </p>
- *
- * @param command
- * the command that caused the exception
- * @param t
- * the exception
- */
- protected abstract void runException(Runnable command, Throwable t);
-
- private class RunnableProcessor implements Runnable {
- private final Handle handle;
-
- private Runnable command;
-
- public RunnableProcessor(Handle handle) {
- this.handle = handle;
- }
-
- public Runnable set(Runnable command) {
- this.command = command;
- return this;
- }
-
- @Override
- public void run() {
- final Thread currentThread = Thread.currentThread();
- final String originalThreadName = currentThread.getName();
- currentThread.setName(name);
-
- try {
- while (true) {
- try {
- command.run();
- } catch (Exception | AssertionError e) {
- try {
- runException(command, e);
- } catch (Exception | AssertionError ee) {
- throw new IllegalStateException("Exception handler threw an exception", ee);
- }
- }
-
- synchronized (queuedRunnables) {
- if (queuedRunnables.isEmpty()) {
- isProcessing = false;
- break;
- }
-
- command = queuedRunnables.removeFirst();
- }
- }
- } finally {
- currentThread.setName(originalThreadName);
- command = null;
- processors.recycle(this, handle);
- }
- }
- }
-
- @Override
- public void execute(Runnable command) {
- synchronized (queuedRunnables) {
- if (isProcessing) {
- queuedRunnables.addLast(command);
- return;
- }
-
- isProcessing = true;
- }
-
- underlyingExecutor.execute(processors.get().set(command));
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java b/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
deleted file mode 100644
index cd7a8d0..0000000
--- a/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.config;
-
-import java.util.Set;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.FormatPluginConfigBase;
-import org.apache.drill.common.logical.StoragePluginConfigBase;
-import org.apache.drill.common.logical.data.LogicalOperatorBase;
-import org.apache.drill.common.scanner.persistence.ScanResult;
-
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser.Feature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-
-
-public class LogicalPlanPersistence {
- private ObjectMapper mapper;
-
- public ObjectMapper getMapper() {
- return mapper;
- }
-
- public LogicalPlanPersistence(DrillConfig conf, ScanResult scanResult) {
- mapper = new ObjectMapper();
-
- SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
- .addDeserializer(LogicalExpression.class, new LogicalExpression.De(conf))
- .addDeserializer(SchemaPath.class, new SchemaPath.De());
-
- mapper.registerModule(deserModule);
- mapper.enable(SerializationFeature.INDENT_OUTPUT);
- mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
- mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
- mapper.configure(Feature.ALLOW_COMMENTS, true);
- registerSubtypes(LogicalOperatorBase.getSubTypes(scanResult));
- registerSubtypes(StoragePluginConfigBase.getSubTypes(scanResult));
- registerSubtypes(FormatPluginConfigBase.getSubTypes(scanResult));
- }
-
- private <T> void registerSubtypes(Set<Class<? extends T>> types) {
- for (Class<? extends T> type : types) {
- mapper.registerSubtypes(type);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/exceptions/ExecutionSetupException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/ExecutionSetupException.java b/common/src/main/java/org/apache/drill/common/exceptions/ExecutionSetupException.java
deleted file mode 100644
index ae70ffa..0000000
--- a/common/src/main/java/org/apache/drill/common/exceptions/ExecutionSetupException.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.exceptions;
-
-import java.lang.reflect.InvocationTargetException;
-
-public class ExecutionSetupException extends DrillException {
-
- public static ExecutionSetupException fromThrowable(String message, Throwable cause) {
- Throwable t = cause instanceof InvocationTargetException
- ? ((InvocationTargetException)cause).getTargetException() : cause;
- if (t instanceof ExecutionSetupException) {
- return ((ExecutionSetupException) t);
- }
- return new ExecutionSetupException(message, t);
- }
- public ExecutionSetupException() {
- super();
-
- }
-
- public ExecutionSetupException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
-
- }
-
- public ExecutionSetupException(String message, Throwable cause) {
- super(message, cause);
-
- }
-
- public ExecutionSetupException(String message) {
- super(message);
-
- }
-
- public ExecutionSetupException(Throwable cause) {
- super(cause);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/exceptions/ExpressionParsingException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/ExpressionParsingException.java b/common/src/main/java/org/apache/drill/common/exceptions/ExpressionParsingException.java
deleted file mode 100644
index 835182d..0000000
--- a/common/src/main/java/org/apache/drill/common/exceptions/ExpressionParsingException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.exceptions;
-
-public class ExpressionParsingException extends LogicalPlanParsingException {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionParsingException.class);
-
- public ExpressionParsingException() {
- super();
- }
-
- public ExpressionParsingException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public ExpressionParsingException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ExpressionParsingException(String message) {
- super(message);
- }
-
- public ExpressionParsingException(Throwable cause) {
- super(cause);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/exceptions/LogicalOperatorValidationException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/LogicalOperatorValidationException.java b/common/src/main/java/org/apache/drill/common/exceptions/LogicalOperatorValidationException.java
deleted file mode 100644
index 3debf28..0000000
--- a/common/src/main/java/org/apache/drill/common/exceptions/LogicalOperatorValidationException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.exceptions;
-
-public class LogicalOperatorValidationException extends LogicalPlanParsingException {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalOperatorValidationException.class);
-
- public LogicalOperatorValidationException() {
- super();
- }
-
- public LogicalOperatorValidationException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public LogicalOperatorValidationException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public LogicalOperatorValidationException(String message) {
- super(message);
- }
-
- public LogicalOperatorValidationException(Throwable cause) {
- super(cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java b/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
deleted file mode 100644
index c8c0afa..0000000
--- a/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.exceptions;
-import org.apache.drill.common.logical.data.LogicalOperator;
-
-public class LogicalPlanParsingException extends DrillRuntimeException{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalPlanParsingException.class);
-
- private LogicalOperator operator;
-
- public LogicalPlanParsingException() {
- super();
- }
-
- public LogicalPlanParsingException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public LogicalPlanParsingException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public LogicalPlanParsingException(String message) {
- super(message);
- }
-
- public LogicalPlanParsingException(Throwable cause) {
- super(cause);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/BooleanOperator.java b/common/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
deleted file mode 100644
index 809d3e2..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.Types;
-
-public class BooleanOperator extends FunctionCall{
-
- public BooleanOperator(String name, List<LogicalExpression> args, ExpressionPosition pos) {
- super(name, args, pos);
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E{
- return visitor.visitBooleanOperator(this, value);
- }
-
- @Override
- public MajorType getMajorType() {
- // If any of argumgnet of a boolean "and"/"or" is nullable, the result is nullable bit.
- // Otherwise, it's non-nullable bit.
- for (LogicalExpression e : args) {
- if (e.getMajorType().getMode() == DataMode.OPTIONAL) {
- return Types.OPTIONAL_BIT;
- }
- }
- return Types.REQUIRED_BIT;
-
- }
-
- @Override
- public int getCumulativeCost() {
- // return the average cost of operands for a boolean "and" | "or"
- int cost = this.getSelfCost();
-
- int i = 0;
- for (LogicalExpression e : this) {
- cost += e.getCumulativeCost();
- i++;
- }
-
- return (int) (cost / i) ;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/CastExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/CastExpression.java b/common/src/main/java/org/apache/drill/common/expression/CastExpression.java
deleted file mode 100644
index 5d09a24..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/CastExpression.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.Collections;
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-public class CastExpression extends LogicalExpressionBase implements Iterable<LogicalExpression>{
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CastExpression.class);
-
- private final LogicalExpression input;
- private final MajorType type;
-
- public CastExpression(LogicalExpression input, MajorType type, ExpressionPosition pos) {
- super(pos);
- this.input = input;
- this.type = checkNotNull(type, "Major type cannot be null");
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitCastExpression(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Collections.singleton(input).iterator();
- }
-
- public LogicalExpression getInput() {
- return input;
- }
-
- @Override
- public MajorType getMajorType() {
- return type;
- }
-
- @Override
- public String toString() {
- return "CastExpression [input=" + input + ", type=" + type + "]";
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/ConvertExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ConvertExpression.java b/common/src/main/java/org/apache/drill/common/expression/ConvertExpression.java
deleted file mode 100644
index dda97e6..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ConvertExpression.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.Collections;
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.Types;
-
-public class ConvertExpression extends LogicalExpressionBase implements Iterable<LogicalExpression>{
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ConvertExpression.class);
-
- public static final String CONVERT_FROM = "convert_from";
- public static final String CONVERT_TO = "convert_to";
-
- private final LogicalExpression input;
- private final MajorType type;
- private final String convertFunction;
- private final String encodingType;
-
- /**
- * @param encodingType
- * @param convertFunction
- * @param input
- * @param pos
- */
- public ConvertExpression(String convertFunction, String encodingType, LogicalExpression input, ExpressionPosition pos) {
- super(pos);
- this.input = input;
- this.convertFunction = CONVERT_FROM.equals(convertFunction.toLowerCase()) ? CONVERT_FROM : CONVERT_TO;
- this.encodingType = encodingType.toUpperCase();
- this.type = Types.getMajorTypeFromName(encodingType.split("_", 2)[0].toLowerCase());
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitConvertExpression(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Collections.singleton(input).iterator();
- }
-
- public String getConvertFunction() {
- return convertFunction;
- }
-
- public LogicalExpression getInput() {
- return input;
- }
-
- @Override
- public MajorType getMajorType() {
- return type;
- }
-
- public String getEncodingType() {
- return encodingType;
- }
-
- @Override
- public String toString() {
- return "ConvertExpression [input=" + input + ", type=" + Types.toString(type) + ", convertFunction="
- + convertFunction + ", conversionType=" + encodingType + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/ErrorCollector.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ErrorCollector.java b/common/src/main/java/org/apache/drill/common/expression/ErrorCollector.java
deleted file mode 100644
index f229e53..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ErrorCollector.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import com.google.common.collect.Range;
-
-public interface ErrorCollector {
-
- public void addGeneralError(ExpressionPosition expr, String s);
-
- public void addUnexpectedArgumentType(ExpressionPosition expr, String name, MajorType actual, MajorType[] expected, int argumentIndex);
-
- public void addUnexpectedArgumentCount(ExpressionPosition expr, int actual, Range<Integer> expected);
-
- public void addUnexpectedArgumentCount(ExpressionPosition expr, int actual, int expected);
-
- public void addNonNumericType(ExpressionPosition expr, MajorType actual);
-
- public void addUnexpectedType(ExpressionPosition expr, int index, MajorType actual);
-
- public void addExpectedConstantValue(ExpressionPosition expr, int actual, String s);
-
- boolean hasErrors();
-
- public int getErrorCount();
-
- String toErrorString();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java b/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
deleted file mode 100644
index 4f958ec..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.drill.common.expression.visitors.ExpressionValidationError;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-
-public class ErrorCollectorImpl implements ErrorCollector {
- List<ExpressionValidationError> errors;
-
- public ErrorCollectorImpl() {
- errors = Lists.newArrayList();
- }
-
- private String addExpr(ExpressionPosition expr, String message) {
- return String.format("Error in expression at index %d. Error: %s. Full expression: %s.", expr.getCharIndex(), message, expr.getExpression());
- }
-
- @Override
- public void addGeneralError(ExpressionPosition expr, String s) {
- errors.add(new ExpressionValidationError(addExpr(expr, s)));
- }
-
- @Override
- public void addUnexpectedArgumentType(ExpressionPosition expr, String name, MajorType actual, MajorType[] expected, int argumentIndex) {
- errors.add(
- new ExpressionValidationError(
- addExpr(expr, String.format(
- "Unexpected argument type. Index :%d Name: %s, Type: %s, Expected type(s): %s",
- argumentIndex, name, actual, Arrays.toString(expected)
- ))
- )
- );
- }
-
- @Override
- public void addUnexpectedArgumentCount(ExpressionPosition expr, int actual, Range<Integer> expected) {
- errors.add(new ExpressionValidationError(
- addExpr(expr, String.format("Unexpected argument count. Actual argument count: %d, Expected range: %s", actual, expected))
- ));
- }
-
- @Override
- public void addUnexpectedArgumentCount(ExpressionPosition expr, int actual, int expected) {
- errors.add(new ExpressionValidationError(
- addExpr(expr, String.format("Unexpected argument count. Actual argument count: %d, Expected count: %d", actual, expected))
- ));
- }
-
- @Override
- public void addNonNumericType(ExpressionPosition expr, MajorType actual) {
- errors.add(new ExpressionValidationError(
- addExpr(expr, String.format("Unexpected numeric type. Actual type: %s", actual))
- ));
- }
-
- @Override
- public void addUnexpectedType(ExpressionPosition expr, int index, MajorType actual) {
- errors.add(new ExpressionValidationError(
- addExpr(expr, String.format("Unexpected argument type. Actual type: %s, Index: %d", actual, index))
- ));
- }
-
- @Override
- public void addExpectedConstantValue(ExpressionPosition expr, int actual, String s) {
- errors.add(new ExpressionValidationError(
- addExpr(expr, String.format("Unexpected constant value. Name: %s, Actual: %s", s, actual))
- ));
- }
-
- @Override
- public boolean hasErrors() {
- return !errors.isEmpty();
- }
-
-
- @Override
- public int getErrorCount() {
- return errors.size();
- }
-
- @Override
- public String toErrorString() {
- return "\n" + Joiner.on("\n").join(errors);
- }
-
- @Override
- public String toString() {
- return toErrorString();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
deleted file mode 100644
index bc478dd..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Type;
-import java.util.List;
-
-import org.apache.drill.common.expression.visitors.ExpressionValidationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.reflect.TypeToken;
-
-@SuppressWarnings("unchecked")
-public class ExpressionFunction {
- final static Logger logger = LoggerFactory.getLogger(ExpressionFunction.class);
-
- @SuppressWarnings("serial")
- final static Type EXPR_LIST = (new TypeToken<List<LogicalExpression>>() {
- }).getType();
-
- private static final Class<?>[] FUNCTIONS = {};
-
- private static final ImmutableMap<String, Constructor<LogicalExpression>> FUNCTION_MAP;
-
- static {
- ImmutableMap.Builder<String, Constructor<LogicalExpression>> builder = ImmutableMap.builder();
- for (Class<?> c : FUNCTIONS) {
-
- // logger.debug("Adding {} and function", c);
- if (!LogicalExpression.class.isAssignableFrom(c)) {
- logger.error(
- "The provided Class [{}] does not derive from LogicalExpression. Skipping inclusion in registry.", c);
- continue;
- }
- FunctionName fn = c.getAnnotation(FunctionName.class);
- if (fn == null) {
- logger.error(
- "The provided Class [{}] did not have a FunctionName annotation. Skipping inclusion in registry.", c);
- continue;
- }
- String name = fn.value();
- try {
- Constructor<LogicalExpression> m = (Constructor<LogicalExpression>) c.getConstructor(List.class);
- if (!EXPR_LIST.equals(m.getGenericParameterTypes()[0])) {
- logger
- .error(
- "The constructor for each function must have a argument list that only contains a List<LogicalExpression>. The class[{}] has an inccorect List<{}> argument.",
- c, m.getGenericParameterTypes()[0]);
- continue;
- }
-
- builder.put(name, m);
- } catch (Exception e) {
- logger
- .error(
- "Failure while attempting to retrieve Logical Expression list constructor on class [{}]. Functions must have one of these.",
- c, e);
- }
- }
-
- FUNCTION_MAP = builder.build();
- }
-
- public static LogicalExpression create(String functionName, List<LogicalExpression> expressions)
- throws ExpressionValidationException {
- // logger.debug("Requesting generation of new function with name {}.",
- // functionName);
- if (!FUNCTION_MAP.containsKey(functionName)) {
- throw new ExpressionValidationException(String.format("Unknown function with name '%s'", functionName));
- }
- try {
- return FUNCTION_MAP.get(functionName).newInstance(expressions);
- } catch (Exception e) {
- throw new ExpressionValidationException("Failure while attempting to build type of " + functionName, e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/ExpressionPosition.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionPosition.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionPosition.java
deleted file mode 100644
index e3f34f4..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionPosition.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-public class ExpressionPosition {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionPosition.class);
-
- public static final ExpressionPosition UNKNOWN = new ExpressionPosition("--UNKNOWN EXPRESSION--", -1);
-
- private final String expression;
- private final int charIndex;
-
- public ExpressionPosition(String expression, int charIndex) {
- super();
- this.expression = expression;
- this.charIndex = charIndex;
- }
-
- @Override
- public String toString() {
- return super.toString()
- + "[charIndex = " + charIndex + ", expression = " + expression + "]";
- }
-
- public String getExpression() {
- return expression;
- }
-
- public int getCharIndex() {
- return charIndex;
- }
-
-
-
-}
[26/45] drill git commit: DRILL-3987: (MOVE) Extract RPC,
memory-base and memory-impl as separate modules.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
deleted file mode 100644
index 61922a1..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
+++ /dev/null
@@ -1,478 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.codec.MessageToMessageDecoder;
-import io.netty.util.Recycler;
-import io.netty.util.Recycler.Handle;
-import io.netty.util.concurrent.GenericFutureListener;
-
-import java.io.Closeable;
-import java.net.SocketAddress;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.drill.common.SerializedExecutor;
-import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
-import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
-import com.google.protobuf.Internal.EnumLite;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.MessageLite;
-import com.google.protobuf.Parser;
-
-/**
- * The Rpc Bus deals with incoming and outgoing communication and is used on both the server and the client side of a
- * system.
- *
- * @param <T>
- */
-public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> implements Closeable {
- final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(this.getClass());
-
- private static final OutboundRpcMessage PONG = new OutboundRpcMessage(RpcMode.PONG, 0, 0, Acks.OK);
-
- protected final CoordinationQueue queue = new CoordinationQueue(16, 16);
-
- protected abstract MessageLite getResponseDefaultInstance(int rpcType) throws RpcException;
-
- protected void handle(C connection, int rpcType, ByteBuf pBody, ByteBuf dBody, ResponseSender sender) throws RpcException{
- sender.send(handle(connection, rpcType, pBody, dBody));
- }
-
- protected abstract Response handle(C connection, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException;
-
- public abstract boolean isClient();
-
- protected final RpcConfig rpcConfig;
-
- protected volatile SocketAddress local;
- protected volatile SocketAddress remote;
-
-
- public RpcBus(RpcConfig rpcConfig) {
- this.rpcConfig = rpcConfig;
- }
-
- protected void setAddresses(SocketAddress remote, SocketAddress local){
- this.remote = remote;
- this.local = local;
- }
-
- <SEND extends MessageLite, RECEIVE extends MessageLite> DrillRpcFuture<RECEIVE> send(C connection, T rpcType,
- SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
- DrillRpcFutureImpl<RECEIVE> rpcFuture = new DrillRpcFutureImpl<RECEIVE>();
- this.send(rpcFuture, connection, rpcType, protobufBody, clazz, dataBodies);
- return rpcFuture;
- }
-
- public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener, C connection, T rpcType,
- SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
- send(listener, connection, rpcType, protobufBody, clazz, false, dataBodies);
- }
-
- public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener, C connection, T rpcType,
- SEND protobufBody, Class<RECEIVE> clazz, boolean allowInEventLoop, ByteBuf... dataBodies) {
-
- Preconditions
- .checkArgument(
- allowInEventLoop || !connection.inEventLoop(),
- "You attempted to send while inside the rpc event thread. This isn't allowed because sending will block if the channel is backed up.");
-
- ByteBuf pBuffer = null;
- boolean completed = false;
-
- try {
-
- if (!allowInEventLoop && !connection.blockOnNotWritable(listener)) {
- // if we're in not in the event loop and we're interrupted while blocking, skip sending this message.
- return;
- }
-
- assert !Arrays.asList(dataBodies).contains(null);
- assert rpcConfig.checkSend(rpcType, protobufBody.getClass(), clazz);
-
- Preconditions.checkNotNull(protobufBody);
- ChannelListenerWithCoordinationId futureListener = queue.get(listener, clazz, connection);
- OutboundRpcMessage m = new OutboundRpcMessage(RpcMode.REQUEST, rpcType, futureListener.getCoordinationId(), protobufBody, dataBodies);
- ChannelFuture channelFuture = connection.getChannel().writeAndFlush(m);
- channelFuture.addListener(futureListener);
- channelFuture.addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
- completed = true;
- } catch (Exception | AssertionError e) {
- listener.failed(new RpcException("Failure sending message.", e));
- } finally {
- if (!completed) {
- if (pBuffer != null) {
- pBuffer.release();
- }
- if (dataBodies != null) {
- for (ByteBuf b : dataBodies) {
- b.release();
- }
-
- }
- }
- ;
- }
- }
-
- public abstract C initRemoteConnection(SocketChannel channel);
-
- public class ChannelClosedHandler implements GenericFutureListener<ChannelFuture> {
-
- final C clientConnection;
- private final Channel channel;
-
- public ChannelClosedHandler(C clientConnection, Channel channel) {
- this.channel = channel;
- this.clientConnection = clientConnection;
- }
-
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
- String msg;
- if(local!=null) {
- msg = String.format("Channel closed %s <--> %s.", local, remote);
- }else{
- msg = String.format("Channel closed %s <--> %s.", future.channel().localAddress(), future.channel().remoteAddress());
- }
-
- if (RpcBus.this.isClient()) {
- if(local != null) {
- logger.info(String.format(msg));
- }
- } else {
- queue.channelClosed(new ChannelClosedException(msg));
- }
-
- clientConnection.close();
- }
-
- }
-
- protected GenericFutureListener<ChannelFuture> getCloseHandler(SocketChannel channel, C clientConnection) {
- return new ChannelClosedHandler(clientConnection, channel);
- }
-
- private interface Recyclable {
- public void recycle();
- }
-
- private class ResponseSenderImpl implements ResponseSender {
-
- private RemoteConnection connection;
- private int coordinationId;
- private final AtomicBoolean sent = new AtomicBoolean(false);
- private final Recyclable recyclable;
-
- public ResponseSenderImpl(Recyclable recyclable) {
- this.recyclable = recyclable;
- }
-
- void set(RemoteConnection connection, int coordinationId){
- this.connection = connection;
- this.coordinationId = coordinationId;
- sent.set(false);
- }
-
- public void send(Response r) {
- try {
- assert rpcConfig.checkResponseSend(r.rpcType, r.pBody.getClass());
- sendOnce();
- OutboundRpcMessage outMessage = new OutboundRpcMessage(RpcMode.RESPONSE, r.rpcType, coordinationId,
- r.pBody, r.dBodies);
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Adding message to outbound buffer. {}", outMessage);
- }
- logger.debug("Sending response with Sender {}", System.identityHashCode(this));
- connection.getChannel().writeAndFlush(outMessage);
- } finally {
- recyclable.recycle();
- }
- }
-
- /**
- * Ensures that each sender is only used once.
- */
- private void sendOnce() {
- if (!sent.compareAndSet(false, true)) {
- throw new IllegalStateException("Attempted to utilize a sender multiple times.");
- }
- }
-
- void sendFailure(UserRpcException e){
- try {
- sendOnce();
- UserException uex = UserException.systemError(e)
- .addIdentity(e.getEndpoint())
- .build(logger);
-
- logger.error("Unexpected Error while handling request message", e);
-
- OutboundRpcMessage outMessage = new OutboundRpcMessage(
- RpcMode.RESPONSE_FAILURE,
- 0,
- coordinationId,
- uex.getOrCreatePBError(false)
- );
-
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Adding message to outbound buffer. {}", outMessage);
- }
- connection.getChannel().writeAndFlush(outMessage);
- } finally {
- recyclable.recycle();
- }
- }
-
- }
-
-
- protected class InboundHandler extends MessageToMessageDecoder<InboundRpcMessage> {
-
- private final RpcEventHandler exec;
- private final C connection;
-
- public InboundHandler(C connection) {
- super();
- this.connection = connection;
- this.exec = new RpcEventHandler(rpcConfig.getExecutor());
- }
-
- @Override
- protected void decode(final ChannelHandlerContext ctx, final InboundRpcMessage msg, final List<Object> output) throws Exception {
- if (!ctx.channel().isOpen()) {
- return;
- }
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Received message {}", msg);
- }
- final Channel channel = connection.getChannel();
- final Stopwatch watch = new Stopwatch().start();
-
- try{
-
- switch (msg.mode) {
- case REQUEST:
- RequestEvent reqEvent = requestRecycler.get();
- reqEvent.set(msg.coordinationId, connection, msg.rpcType, msg.pBody, msg.dBody);
- exec.execute(reqEvent);
- break;
-
- case RESPONSE:
- ResponseEvent respEvent = responseRecycler.get();
- respEvent.set(msg.rpcType, msg.coordinationId, msg.pBody, msg.dBody);
- exec.execute(respEvent);
- break;
-
- case RESPONSE_FAILURE:
- DrillPBError failure = DrillPBError.parseFrom(new ByteBufInputStream(msg.pBody, msg.pBody.readableBytes()));
- queue.updateFailedFuture(msg.coordinationId, failure);
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Updated rpc future with coordinationId {} with failure ", msg.coordinationId, failure);
- }
- break;
-
- case PING:
- channel.writeAndFlush(PONG);
- break;
-
- case PONG:
- // noop.
- break;
-
- default:
- throw new UnsupportedOperationException();
- }
- } finally {
- long time = watch.elapsed(TimeUnit.MILLISECONDS);
- long delayThreshold = Integer.parseInt(System.getProperty("drill.exec.rpcDelayWarning", "500"));
- if (time > delayThreshold) {
- logger.warn(String.format(
- "Message of mode %s of rpc type %d took longer than %dms. Actual duration was %dms.",
- msg.mode, msg.rpcType, delayThreshold, time));
- }
- msg.release();
- }
- }
- }
-
- public static <T> T get(ByteBuf pBody, Parser<T> parser) throws RpcException {
- try {
- ByteBufInputStream is = new ByteBufInputStream(pBody);
- return parser.parseFrom(is);
- } catch (InvalidProtocolBufferException e) {
- throw new RpcException(String.format("Failure while decoding message with parser of type. %s", parser.getClass().getCanonicalName()), e);
- }
- }
-
- class RpcEventHandler extends SerializedExecutor {
-
- public RpcEventHandler(Executor underlyingExecutor) {
- super(rpcConfig.getName() + "-rpc-event-queue", underlyingExecutor);
- }
-
- @Override
- protected void runException(Runnable command, Throwable t) {
- logger.error("Failure while running rpc command.", t);
- }
-
- }
-
- private final Recycler<RequestEvent> requestRecycler = new Recycler<RequestEvent>() {
- @Override
- protected RequestEvent newObject(Handle handle) {
- return new RequestEvent(handle);
- }
- };
-
- private class RequestEvent implements Runnable, Recyclable {
- private final ResponseSenderImpl sender;
- private final Handle handle;
- private C connection;
- private int rpcType;
- private ByteBuf pBody;
- private ByteBuf dBody;
-
- RequestEvent(Handle handle){
- this.handle = handle;
- sender = new ResponseSenderImpl(this);
- }
-
- public void set(int coordinationId, C connection, int rpcType, ByteBuf pBody, ByteBuf dBody) {
- this.connection = connection;
- this.rpcType = rpcType;
- this.pBody = pBody;
- this.dBody = dBody;
- sender.set(connection, coordinationId);
-
- if(pBody != null){
- pBody.retain();
- }
-
- if(dBody != null){
- dBody.retain();
- }
- }
-
- @Override
- public void run() {
- try {
- handle(connection, rpcType, pBody, dBody, sender);
- } catch (UserRpcException e) {
- sender.sendFailure(e);
- } catch (Exception e) {
- logger.error("Failure while handling message.", e);
- }finally{
- if(pBody != null){
- pBody.release();
- }
-
- if(dBody != null){
- dBody.release();
- }
- }
-
- }
-
- @Override
- public void recycle() {
- // We must defer recycling until the sender has been used.
- requestRecycler.recycle(this, handle);
- }
-
-
- }
-
- private final Recycler<ResponseEvent> responseRecycler = new Recycler<ResponseEvent>() {
- @Override
- protected ResponseEvent newObject(Handle handle) {
- return new ResponseEvent(handle);
- }
- };
-
- private class ResponseEvent implements Runnable {
- private final Handle handle;
-
- private int rpcType;
- private int coordinationId;
- private ByteBuf pBody;
- private ByteBuf dBody;
-
- public ResponseEvent(Handle handle){
- this.handle = handle;
- }
-
- public void set(int rpcType, int coordinationId, ByteBuf pBody, ByteBuf dBody) {
- this.rpcType = rpcType;
- this.coordinationId = coordinationId;
- this.pBody = pBody;
- this.dBody = dBody;
-
- if(pBody != null){
- pBody.retain();
- }
-
- if(dBody != null){
- dBody.retain();
- }
- }
-
- public void run(){
- try {
- MessageLite m = getResponseDefaultInstance(rpcType);
- assert rpcConfig.checkReceive(rpcType, m.getClass());
- RpcOutcome<?> rpcFuture = queue.getFuture(rpcType, coordinationId, m.getClass());
- Parser<?> parser = m.getParserForType();
- Object value = parser.parseFrom(new ByteBufInputStream(pBody, pBody.readableBytes()));
- rpcFuture.set(value, dBody);
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Updated rpc future {} with value {}", rpcFuture, value);
- }
- } catch (Exception ex) {
- logger.error("Failure while handling response.", ex);
- }finally{
- if(pBody != null){
- pBody.release();
- }
-
- if(dBody != null){
- dBody.release();
- }
-
- responseRecycler.recycle(this, handle);
- }
-
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcCheckedFuture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcCheckedFuture.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcCheckedFuture.java
deleted file mode 100644
index 11b07ad..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcCheckedFuture.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-import com.google.common.util.concurrent.AbstractCheckedFuture;
-import com.google.common.util.concurrent.ListenableFuture;
-
-public class RpcCheckedFuture<T> extends AbstractCheckedFuture<T, RpcException> implements DrillRpcFuture<T>{
-
- volatile ByteBuf buffer;
-
- public RpcCheckedFuture(ListenableFuture<T> delegate) {
- super(delegate);
- }
-
- public void set(T obj, ByteBuf buffer){
- this.buffer = buffer;
- }
-
- @Override
- protected RpcException mapException(Exception e) {
- return RpcException.mapException(e);
- }
-
- @Override
- public ByteBuf getBuffer() {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcCommand.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcCommand.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcCommand.java
deleted file mode 100644
index 93b901c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcCommand.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import com.google.protobuf.MessageLite;
-
-public interface RpcCommand<T extends MessageLite, C extends RemoteConnection> extends RpcConnectionHandler<C>{
-
- public abstract void connectionAvailable(C connection);
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConfig.java
deleted file mode 100644
index 22b253a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConfig.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import java.util.Map;
-import java.util.concurrent.Executor;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-import com.google.protobuf.Internal.EnumLite;
-import com.google.protobuf.MessageLite;
-
-public class RpcConfig {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcConfig.class);
-
- private final String name;
- private final int timeout;
- private final Map<EnumLite, RpcMessageType<?, ?, ?>> sendMap;
- private final Map<Integer, RpcMessageType<?, ?, ?>> receiveMap;
- private final Executor executor;
-
- private RpcConfig(String name, Map<EnumLite, RpcMessageType<?, ?, ?>> sendMap,
- Map<Integer, RpcMessageType<?, ?, ?>> receiveMap, int timeout, Executor executor) {
- Preconditions.checkNotNull(executor, "Executor must be defined.");
- this.name = name;
- this.timeout = timeout;
- this.sendMap = ImmutableMap.copyOf(sendMap);
- this.receiveMap = ImmutableMap.copyOf(receiveMap);
- this.executor = executor;
- }
-
- public String getName() {
- return name;
- }
-
- public int getTimeout() {
- return timeout;
- }
-
- public boolean hasTimeout() {
- return timeout > 0;
- }
-
- public Executor getExecutor() {
- return executor;
- }
-
- public boolean checkReceive(int rpcType, Class<?> receiveClass) {
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug(String.format("Checking reception for rpcType %d and receive class %s.", rpcType, receiveClass));
- }
- RpcMessageType<?,?,?> type = receiveMap.get(rpcType);
- if (type == null) {
- throw new IllegalStateException(String.format("%s: There is no defined RpcMessage type for a Rpc receive type number of %s.", name, rpcType));
- }
-
- if (receiveClass != type.getRet()) {
- throw new IllegalStateException(String.format("%s: The definition for receive doesn't match implementation code. The definition is %s however the current receive for this type was of type %s.", name, type, receiveClass.getCanonicalName()));
- }
- return true;
- }
-
- public boolean checkSend(EnumLite send, Class<?> sendClass, Class<?> receiveClass) {
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug(String.format("Checking send classes for send RpcType %s. Send Class is %s and Receive class is %s.", send, sendClass, receiveClass));
- }
- RpcMessageType<?,?,?> type = sendMap.get(send);
- if (type == null) {
- throw new IllegalStateException(String.format("%s: There is no defined RpcMessage type for a Rpc send type of %s.", name, send));
- }
-
- if (type.getSend() != sendClass) {
- throw new IllegalStateException(String.format("%s: The definition for send doesn't match implementation code. The definition is %s however the current send is trying to send an object of type %s.", name, type, sendClass.getCanonicalName()));
- }
- if (type.getRet() != receiveClass) {
- throw new IllegalStateException(String.format("%s: The definition for send doesn't match implementation code. The definition is %s however the current send is trying to setup an expected reception of an object of type %s.", name, type, receiveClass.getCanonicalName()));
- }
-
- return true;
- }
-
- public boolean checkResponseSend(EnumLite responseType, Class<?> responseClass) {
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug(String.format("Checking responce send of type %s with response class of %s.", responseType, responseClass));
- }
- RpcMessageType<?,?,?> type = receiveMap.get(responseType.getNumber());
- if (type == null) {
- throw new IllegalStateException(String.format("%s: There is no defined RpcMessage type for a Rpc response of type %s.", name, responseType));
- }
- if (type.getRet() != responseClass) {
- throw new IllegalStateException(String.format("%s: The definition for the response doesn't match implementation code. The definition is %s however the current response is trying to response with an object of type %s.", name, type, responseClass.getCanonicalName()));
- }
-
- return true;
- }
-
- public static class RpcMessageType<SEND extends MessageLite, RECEIVE extends MessageLite, T extends EnumLite>{
- private T sendEnum;
- private Class<SEND> send;
- private T receiveEnum;
- private Class<RECEIVE> ret;
- public RpcMessageType(T sendEnum, Class<SEND> send, T receiveEnum, Class<RECEIVE> ret) {
- super();
- this.sendEnum = sendEnum;
- this.send = send;
- this.receiveEnum = receiveEnum;
- this.ret = ret;
- }
- public Class<SEND> getSend() {
- return send;
- }
- public void setSend(Class<SEND> send) {
- this.send = send;
- }
- public T getSendEnum() {
- return sendEnum;
- }
- public void setSendEnum(T sendEnum) {
- this.sendEnum = sendEnum;
- }
- public Class<RECEIVE> getRet() {
- return ret;
- }
- public void setRet(Class<RECEIVE> ret) {
- this.ret = ret;
- }
- public T getReceiveEnum() {
- return receiveEnum;
- }
- public void setReceiveEnum(T receiveEnum) {
- this.receiveEnum = receiveEnum;
- }
- @Override
- public String toString() {
- return "RpcMessageType [sendEnum=" + sendEnum + ", send=" + send + ", receiveEnum=" + receiveEnum + ", ret="
- + ret + "]";
- }
-
- }
-
- public static RpcConfigBuilder newBuilder() {
- return new RpcConfigBuilder();
- }
-
- public static class RpcConfigBuilder {
- private String name;
- private int timeout = -1;
- private Executor executor;
- private Map<EnumLite, RpcMessageType<?, ?, ?>> sendMap = Maps.newHashMap();
- private Map<Integer, RpcMessageType<?, ?, ?>> receiveMap = Maps.newHashMap();
-
- private RpcConfigBuilder() {
- }
-
- public RpcConfigBuilder name(String name) {
- this.name = name;
- return this;
- }
-
- public RpcConfigBuilder timeout(int timeoutInSeconds) {
- this.timeout = timeoutInSeconds;
- return this;
- }
-
- public <SEND extends MessageLite, RECEIVE extends MessageLite, T extends EnumLite> RpcConfigBuilder add(T sendEnum, Class<SEND> send, T receiveEnum, Class<RECEIVE> rec) {
- RpcMessageType<SEND, RECEIVE, T> type = new RpcMessageType<SEND, RECEIVE, T>(sendEnum, send, receiveEnum, rec);
- this.sendMap.put(sendEnum, type);
- this.receiveMap.put(receiveEnum.getNumber(), type);
- return this;
- }
-
- public RpcConfigBuilder executor(Executor executor) {
- this.executor = executor;
- return this;
- }
-
- public RpcConfig build() {
- Preconditions.checkArgument(timeout > -1, "Timeout must be a positive number or zero for disabled.");
- Preconditions.checkArgument(name != null, "RpcConfig name must be set.");
- return new RpcConfig(name, sendMap, receiveMap, timeout, executor);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConnectionHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConnectionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConnectionHandler.java
deleted file mode 100644
index 7618231..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConnectionHandler.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-public interface RpcConnectionHandler<T extends RemoteConnection> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcConnectionHandler.class);
-
- public static enum FailureType{CONNECTION, HANDSHAKE_COMMUNICATION, HANDSHAKE_VALIDATION}
-
- public void connectionSucceeded(T connection);
- public void connectionFailed(FailureType type, Throwable t);
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConstants.java
deleted file mode 100644
index 4be365c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcConstants.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-public class RpcConstants {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcConstants.class);
-
- private RpcConstants(){}
-
- public static final boolean SOME_DEBUGGING = false;
- public static final boolean EXTRA_DEBUGGING = false;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcDecoder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcDecoder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcDecoder.java
deleted file mode 100644
index ac48187..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcDecoder.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.CorruptedFrameException;
-import io.netty.handler.codec.MessageToMessageDecoder;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcHeader;
-
-/**
- * Converts a previously length adjusted buffer into an RpcMessage.
- */
-class RpcDecoder extends MessageToMessageDecoder<ByteBuf> {
- final org.slf4j.Logger logger;
-
- private final AtomicLong messageCounter = new AtomicLong();
-
- public RpcDecoder(String name) {
- this.logger = org.slf4j.LoggerFactory.getLogger(RpcDecoder.class.getCanonicalName() + "-" + name);
- }
-
-
- @Override
- protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> out) throws Exception {
- if (!ctx.channel().isOpen()) {
- return;
- }
-
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Inbound rpc message received.");
- }
-
- // now, we know the entire message is in the buffer and the buffer is constrained to this message. Additionally,
- // this process should avoid reading beyond the end of this buffer so we inform the ByteBufInputStream to throw an
- // exception if be go beyond readable bytes (as opposed to blocking).
- final ByteBufInputStream is = new ByteBufInputStream(buffer, buffer.readableBytes());
-
- // read the rpc header, saved in delimited format.
- checkTag(is, RpcEncoder.HEADER_TAG);
- final RpcHeader header = RpcHeader.parseDelimitedFrom(is);
-
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug(" post header read index {}", buffer.readerIndex());
- }
-
- // read the protobuf body into a buffer.
- checkTag(is, RpcEncoder.PROTOBUF_BODY_TAG);
- final int pBodyLength = readRawVarint32(is);
- final ByteBuf pBody = buffer.slice(buffer.readerIndex(), pBodyLength);
- buffer.skipBytes(pBodyLength);
- pBody.retain(1);
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Read protobuf body of length {} into buffer {}.", pBodyLength, pBody);
- }
-
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("post protobufbody read index {}", buffer.readerIndex());
- }
-
- ByteBuf dBody = null;
- int dBodyLength = 0;
-
- // read the data body.
- if (buffer.readableBytes() > 0) {
-
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Reading raw body, buffer has {} bytes available, is available {}.", buffer.readableBytes(), is.available());
- }
- checkTag(is, RpcEncoder.RAW_BODY_TAG);
- dBodyLength = readRawVarint32(is);
- if (buffer.readableBytes() != dBodyLength) {
- throw new CorruptedFrameException(String.format("Expected to receive a raw body of %d bytes but received a buffer with %d bytes.", dBodyLength, buffer.readableBytes()));
- }
- dBody = buffer.slice();
- dBody.retain(1);
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Read raw body of {}", dBody);
- }
-
- }else{
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("No need to read raw body, no readable bytes left.");
- }
- }
-
-
- // return the rpc message.
- InboundRpcMessage m = new InboundRpcMessage(header.getMode(), header.getRpcType(), header.getCoordinationId(),
- pBody, dBody);
-
- // move the reader index forward so the next rpc call won't try to work with it.
- buffer.skipBytes(dBodyLength);
- messageCounter.incrementAndGet();
- if (RpcConstants.SOME_DEBUGGING) {
- logger.debug("Inbound Rpc Message Decoded {}.", m);
- }
- out.add(m);
-
- }
-
- private void checkTag(ByteBufInputStream is, int expectedTag) throws IOException {
- int actualTag = readRawVarint32(is);
- if (actualTag != expectedTag) {
- throw new CorruptedFrameException(String.format("Expected to read a tag of %d but actually received a value of %d. Happened after reading %d message.", expectedTag, actualTag, messageCounter.get()));
- }
- }
-
- // Taken from CodedInputStream and modified to enable ByteBufInterface.
- public static int readRawVarint32(ByteBufInputStream is) throws IOException {
- byte tmp = is.readByte();
- if (tmp >= 0) {
- return tmp;
- }
- int result = tmp & 0x7f;
- if ((tmp = is.readByte()) >= 0) {
- result |= tmp << 7;
- } else {
- result |= (tmp & 0x7f) << 7;
- if ((tmp = is.readByte()) >= 0) {
- result |= tmp << 14;
- } else {
- result |= (tmp & 0x7f) << 14;
- if ((tmp = is.readByte()) >= 0) {
- result |= tmp << 21;
- } else {
- result |= (tmp & 0x7f) << 21;
- result |= (tmp = is.readByte()) << 28;
- if (tmp < 0) {
- // Discard upper 32 bits.
- for (int i = 0; i < 5; i++) {
- if (is.readByte() >= 0) {
- return result;
- }
- }
- throw new CorruptedFrameException("Encountered a malformed varint.");
- }
- }
- }
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcEncoder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcEncoder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcEncoder.java
deleted file mode 100644
index f9da6f1..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcEncoder.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufOutputStream;
-import io.netty.buffer.CompositeByteBuf;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.MessageToMessageEncoder;
-
-import java.io.OutputStream;
-import java.util.List;
-
-import org.apache.drill.exec.proto.GeneralRPCProtos.CompleteRpcMessage;
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcHeader;
-
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.WireFormat;
-
-/**
- * Converts an RPCMessage into wire format.
- */
-class RpcEncoder extends MessageToMessageEncoder<OutboundRpcMessage>{
- final org.slf4j.Logger logger;
-
- static final int HEADER_TAG = makeTag(CompleteRpcMessage.HEADER_FIELD_NUMBER, WireFormat.WIRETYPE_LENGTH_DELIMITED);
- static final int PROTOBUF_BODY_TAG = makeTag(CompleteRpcMessage.PROTOBUF_BODY_FIELD_NUMBER, WireFormat.WIRETYPE_LENGTH_DELIMITED);
- static final int RAW_BODY_TAG = makeTag(CompleteRpcMessage.RAW_BODY_FIELD_NUMBER, WireFormat.WIRETYPE_LENGTH_DELIMITED);
- static final int HEADER_TAG_LENGTH = getRawVarintSize(HEADER_TAG);
- static final int PROTOBUF_BODY_TAG_LENGTH = getRawVarintSize(PROTOBUF_BODY_TAG);
- static final int RAW_BODY_TAG_LENGTH = getRawVarintSize(RAW_BODY_TAG);
-
- public RpcEncoder(String name) {
- this.logger = org.slf4j.LoggerFactory.getLogger(RpcEncoder.class.getCanonicalName() + "-" + name);
- }
-
- @Override
- protected void encode(ChannelHandlerContext ctx, OutboundRpcMessage msg, List<Object> out) throws Exception {
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Rpc Encoder called with msg {}", msg);
- }
-
- if (!ctx.channel().isOpen()) {
- //output.add(ctx.alloc().buffer(0));
- logger.debug("Channel closed, skipping encode.");
- msg.release();
- return;
- }
-
- try{
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Encoding outbound message {}", msg);
- }
- // first we build the RpcHeader
- RpcHeader header = RpcHeader.newBuilder() //
- .setMode(msg.mode) //
- .setCoordinationId(msg.coordinationId) //
- .setRpcType(msg.rpcType).build();
-
- // figure out the full length
- int headerLength = header.getSerializedSize();
- int protoBodyLength = msg.pBody.getSerializedSize();
- int rawBodyLength = msg.getRawBodySize();
- int fullLength = //
- HEADER_TAG_LENGTH + getRawVarintSize(headerLength) + headerLength + //
- PROTOBUF_BODY_TAG_LENGTH + getRawVarintSize(protoBodyLength) + protoBodyLength; //
-
- if (rawBodyLength > 0) {
- fullLength += (RAW_BODY_TAG_LENGTH + getRawVarintSize(rawBodyLength) + rawBodyLength);
- }
-
- ByteBuf buf = ctx.alloc().buffer();
- OutputStream os = new ByteBufOutputStream(buf);
- CodedOutputStream cos = CodedOutputStream.newInstance(os);
-
- // write full length first (this is length delimited stream).
- cos.writeRawVarint32(fullLength);
-
- // write header
- cos.writeRawVarint32(HEADER_TAG);
- cos.writeRawVarint32(headerLength);
- header.writeTo(cos);
-
- // write protobuf body length and body
- cos.writeRawVarint32(PROTOBUF_BODY_TAG);
- cos.writeRawVarint32(protoBodyLength);
- msg.pBody.writeTo(cos);
-
- // if exists, write data body and tag.
- if (msg.getRawBodySize() > 0) {
- if(RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Writing raw body of size {}", msg.getRawBodySize());
- }
-
- cos.writeRawVarint32(RAW_BODY_TAG);
- cos.writeRawVarint32(rawBodyLength);
- cos.flush(); // need to flush so that dbody goes after if cos is caching.
-
- CompositeByteBuf cbb = new CompositeByteBuf(buf.alloc(), true, msg.dBodies.length + 1);
- cbb.addComponent(buf);
- int bufLength = buf.readableBytes();
- for (ByteBuf b : msg.dBodies) {
- cbb.addComponent(b);
- bufLength += b.readableBytes();
- }
- cbb.writerIndex(bufLength);
- out.add(cbb);
- } else {
- cos.flush();
- out.add(buf);
- }
-
- if (RpcConstants.SOME_DEBUGGING) {
- logger.debug("Wrote message length {}:{} bytes (head:body). Message: " + msg, getRawVarintSize(fullLength), fullLength);
- }
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Sent message. Ending writer index was {}.", buf.writerIndex());
- }
- } finally {
- // make sure to release Rpc Messages underlying byte buffers.
- //msg.release();
- }
- }
-
- /** Makes a tag value given a field number and wire type, copied from WireFormat since it isn't public. */
- static int makeTag(final int fieldNumber, final int wireType) {
- return (fieldNumber << 3) | wireType;
- }
-
- public static int getRawVarintSize(int value) {
- int count = 0;
- while (true) {
- if ((value & ~0x7F) == 0) {
- count++;
- return count;
- } else {
- count++;
- value >>>= 7;
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
deleted file mode 100644
index a6d0e8e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import java.util.concurrent.ExecutionException;
-
-import org.apache.drill.common.exceptions.DrillIOException;
-import org.apache.drill.common.util.DrillStringUtils;
-import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
-
-/**
- * Parent class for all rpc exceptions.
- */
-public class RpcException extends DrillIOException{
- private static final long serialVersionUID = -5964230316010502319L;
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcException.class);
-
- public RpcException() {
- super();
- }
-
- public RpcException(String message, Throwable cause) {
- super(format(message), cause);
- }
-
- private static String format(String message) {
- return DrillStringUtils.unescapeJava(message);
- }
-
- public RpcException(String message) {
- super(format(message));
- }
-
- public RpcException(Throwable cause) {
- super(cause);
- }
-
- public static RpcException mapException(Throwable t) {
- while (t instanceof ExecutionException) {
- t = ((ExecutionException)t).getCause();
- }
- if (t instanceof RpcException) {
- return ((RpcException) t);
- }
- return new RpcException(t);
- }
-
- public static RpcException mapException(String message, Throwable t) {
- while (t instanceof ExecutionException) {
- t = ((ExecutionException)t).getCause();
- }
- return new RpcException(message, t);
- }
-
- public boolean isRemote(){
- return false;
- }
-
- public DrillPBError getRemoteError(){
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
deleted file mode 100644
index 46b7702..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import org.eclipse.jetty.io.Connection;
-
-public class RpcExceptionHandler<C extends RemoteConnection> implements ChannelHandler{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcExceptionHandler.class);
-
- private final C connection;
-
- public RpcExceptionHandler(C connection){
- this.connection = connection;
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
- if(!ctx.channel().isOpen() || cause.getMessage().equals("Connection reset by peer")){
- logger.warn("Exception occurred with closed channel. Connection: {}", connection.getName(), cause);
- return;
- }else{
- logger.error("Exception in RPC communication. Connection: {}. Closing connection.", connection.getName(), cause);
- ctx.close();
- }
- }
-
- @Override
- public void handlerAdded(ChannelHandlerContext ctx) throws Exception {
- }
-
-
- @Override
- public void handlerRemoved(ChannelHandlerContext ctx) throws Exception {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcMessage.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcMessage.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcMessage.java
deleted file mode 100644
index 9712c9a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcMessage.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
-
-public abstract class RpcMessage {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcMessage.class);
-
- public RpcMode mode;
- public int rpcType;
- public int coordinationId;
-
- public RpcMessage(RpcMode mode, int rpcType, int coordinationId) {
- this.mode = mode;
- this.rpcType = rpcType;
- this.coordinationId = coordinationId;
- }
-
- public abstract int getBodySize();
- abstract void release();
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcOutcome.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcOutcome.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcOutcome.java
deleted file mode 100644
index af9aa01..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcOutcome.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-public interface RpcOutcome<T> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcOutcome.class);
-
- public void set(Object value, ByteBuf buffer);
- public void setException(Throwable t);
- public Class<T> getOutcomeType();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcOutcomeListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcOutcomeListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcOutcomeListener.java
deleted file mode 100644
index 4485cf9..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcOutcomeListener.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-public interface RpcOutcomeListener<V> {
-
- /**
- * Called when an error occurred while waiting for the RPC outcome.
- * @param ex
- */
- void failed(RpcException ex);
-
-
- void success(V value, ByteBuf buffer);
-
- /**
- * Called when the sending thread is interrupted. Possible when the fragment is cancelled due to query cancellations or
- * failures.
- */
- void interrupted(final InterruptedException e);
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
deleted file mode 100644
index 34da53c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/TransportCheck.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.epoll.EpollEventLoopGroup;
-import io.netty.channel.epoll.EpollServerSocketChannel;
-import io.netty.channel.epoll.EpollSocketChannel;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.ServerSocketChannel;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.util.internal.SystemPropertyUtil;
-
-import java.util.Locale;
-
-import org.apache.drill.exec.ExecConstants;
-
-/**
- * TransportCheck decides whether or not to use the native EPOLL mechanism for communication.
- */
-public class TransportCheck {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TransportCheck.class);
-
- public static final boolean SUPPORTS_EPOLL;
-
- static{
-
- String name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.US).trim();
-
- if (name.startsWith("linux") && SystemPropertyUtil.getBoolean(ExecConstants.USE_LINUX_EPOLL, false)) {
- SUPPORTS_EPOLL = true;
- } else {
- SUPPORTS_EPOLL = false;
- }
- }
-
- public static Class<? extends ServerSocketChannel> getServerSocketChannel(){
- if(SUPPORTS_EPOLL){
- return EpollServerSocketChannel.class;
- }else{
- return NioServerSocketChannel.class;
- }
- }
-
- public static Class<? extends SocketChannel> getClientSocketChannel(){
- if(SUPPORTS_EPOLL){
- return EpollSocketChannel.class;
- }else{
- return NioSocketChannel.class;
- }
- }
-
- public static EventLoopGroup createEventLoopGroup(int nThreads, String prefix) {
- if(SUPPORTS_EPOLL){
- return new EpollEventLoopGroup(nThreads, new NamedThreadFactory(prefix));
- }else{
- return new NioEventLoopGroup(nThreads, new NamedThreadFactory(prefix));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/UserRpcException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/UserRpcException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/UserRpcException.java
deleted file mode 100644
index 1d2cc1a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/UserRpcException.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-
-public class UserRpcException extends RpcException {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UserRpcException.class);
-
- private final String message;
- private final DrillbitEndpoint endpoint;
- private final Throwable t;
-
- public UserRpcException(DrillbitEndpoint endpoint, String message, Throwable t) {
- super(t);
- this.message = message;
- this.endpoint = endpoint;
- this.t = t;
- }
-
- public String getUserMessage() {
- return message;
- }
-
- public DrillbitEndpoint getEndpoint() {
- return endpoint;
- }
-
- public Throwable getUserException() {
- return t;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/AwaitableUserResultsListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/AwaitableUserResultsListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/AwaitableUserResultsListener.java
index 74b48b6..347d076 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/AwaitableUserResultsListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/AwaitableUserResultsListener.java
@@ -24,6 +24,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
/**
* General mechanism for waiting on the query to be executed
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/ConnectionThrottle.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/ConnectionThrottle.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/ConnectionThrottle.java
deleted file mode 100644
index e2ffcc0..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/ConnectionThrottle.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc.user;
-
-public interface ConnectionThrottle {
- public void setAutoRead(boolean enableAutoRead);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
index 14c7154..16d957b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
@@ -35,6 +35,7 @@ import org.apache.drill.exec.proto.UserBitShared.QueryResult;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.rpc.BaseRpcOutcomeListener;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.RemoteConnection;
import org.apache.drill.exec.rpc.RpcBus;
import org.apache.drill.exec.rpc.RpcException;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
index 84c98b4..824e6eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
@@ -38,6 +38,7 @@ import org.apache.drill.exec.proto.UserProtos.UserProperties;
import org.apache.drill.exec.proto.UserProtos.UserToBitHandshake;
import org.apache.drill.exec.rpc.Acks;
import org.apache.drill.exec.rpc.BasicClientWithConnection;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.OutOfMemoryHandler;
import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
import org.apache.drill.exec.rpc.Response;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java
index 01a44b8..1a55716 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.rpc.user;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
public interface UserResultsListener {
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
index ddbbe7d..0ca8e74 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
@@ -29,6 +29,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.client.DrillClient;
@@ -39,7 +40,7 @@ import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.vector.ValueVector;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 6adbfc0..d609915 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -44,6 +44,7 @@ import org.apache.drill.exec.coord.DistributedSemaphore;
import org.apache.drill.exec.coord.DistributedSemaphore.DistributedLease;
import org.apache.drill.exec.exception.OptimizerException;
import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.opt.BasicOptimizer;
@@ -434,7 +435,7 @@ public class Foreman implements Runnable {
final OptionManager optionManager = queryContext.getOptions();
final long maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(),
- queryContext.getConfig().getLong(ExecConstants.TOP_LEVEL_MAX_ALLOC));
+ queryContext.getConfig().getLong(TopLevelAllocator.TOP_LEVEL_MAX_ALLOC));
maxAllocPerNode = Math.min(maxAllocPerNode,
optionManager.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val);
final long maxSortAlloc = maxAllocPerNode / (sortList.size() * maxWidthPerNode);
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index 1a95e77..8737c29 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -45,8 +45,8 @@ import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.AwaitableUserResultsListener;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.rpc.user.UserSession;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/test/java/org/apache/drill/SingleRowListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/SingleRowListener.java b/exec/java-exec/src/test/java/org/apache/drill/SingleRowListener.java
index 2cd5c95..4b3c008 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/SingleRowListener.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/SingleRowListener.java
@@ -28,7 +28,7 @@ import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryData;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
index 76bc3bf..26b2a4f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
@@ -28,7 +28,6 @@ import java.util.Properties;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -45,18 +44,19 @@ import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.test.DrillTest;
import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-public class TestAllocators {
+public class TestAllocators extends DrillTest {
private static final Properties TEST_CONFIGURATIONS = new Properties() {
{
- put(ExecConstants.TOP_LEVEL_MAX_ALLOC, "14000000");
- put(ExecConstants.ENABLE_FRAGMENT_MEMORY_LIMIT, "true");
- put(ExecConstants.FRAGMENT_MEM_OVERCOMMIT_FACTOR, "1.1");
+ put(TopLevelAllocator.TOP_LEVEL_MAX_ALLOC, "14000000");
+ put(AccountorImpl.ENABLE_FRAGMENT_MEMORY_LIMIT, "true");
+ put(AccountorImpl.FRAGMENT_MEM_OVERCOMMIT_FACTOR, "1.1");
}
};
@@ -66,8 +66,8 @@ public class TestAllocators {
public void testTransfer() throws Exception {
final Properties props = new Properties() {
{
- put(ExecConstants.TOP_LEVEL_MAX_ALLOC, "1000000");
- put(ExecConstants.ERROR_ON_MEMORY_LEAK, "true");
+ put(TopLevelAllocator.TOP_LEVEL_MAX_ALLOC, "1000000");
+ put(TopLevelAllocator.ERROR_ON_MEMORY_LEAK, "true");
}
};
final DrillConfig config = DrillConfig.create(props);
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
deleted file mode 100644
index bb1e718..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import static org.junit.Assert.assertEquals;
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.common.DrillAutoCloseables;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecTest;
-import org.junit.Test;
-
-
-public class TestEndianess extends ExecTest {
- @Test
- public void testLittleEndian() {
- final DrillConfig drillConfig = DrillConfig.create();
- final BufferAllocator a = RootAllocatorFactory.newRoot(drillConfig);
- final ByteBuf b = a.buffer(4);
- b.setInt(0, 35);
- assertEquals(b.getByte(0), 35);
- assertEquals(b.getByte(1), 0);
- assertEquals(b.getByte(2), 0);
- assertEquals(b.getByte(3), 0);
- b.release();
- DrillAutoCloseables.closeNoChecked(a);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index 986bfcd..ae3c97f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -68,9 +68,9 @@ import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.DrillRpcFuture;
import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.store.pojo.PojoRecordReader;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
index 9f622f5..4976bca 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
@@ -32,8 +32,8 @@ import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.vector.ValueVector;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
index e1b03d5..7afc05d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
@@ -30,7 +30,7 @@ import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.server.Drillbit;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/pom.xml
----------------------------------------------------------------------
diff --git a/exec/memory/base/pom.xml b/exec/memory/base/pom.xml
new file mode 100644
index 0000000..6bfd29b
--- /dev/null
+++ b/exec/memory/base/pom.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>memory-parent</artifactId>
+ <groupId>org.apache.drill.memory</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>drill-memory-base</artifactId>
+ <name>exec/memory/base</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-protocol</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
+
+ </dependencies>
+
+
+ <build>
+ </build>
+
+
+
+</project>
[34/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
new file mode 100644
index 0000000..2fd8e67
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -0,0 +1,664 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.math.BigDecimal;
+import java.util.GregorianCalendar;
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.util.CoreDecimalUtility;
+
+import com.google.common.collect.Iterators;
+
+public class ValueExpressions {
+
+ public static LogicalExpression getBigInt(long l){
+ return new LongExpression(l);
+ }
+
+ public static LogicalExpression getInt(int i){
+ return new IntExpression(i, ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getFloat8(double d){
+ return new DoubleExpression(d, ExpressionPosition.UNKNOWN);
+ }
+ public static LogicalExpression getFloat4(float f){
+ return new FloatExpression(f, ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getBit(boolean b){
+ return new BooleanExpression(Boolean.toString(b), ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getChar(String s){
+ return new QuotedString(s, ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getDate(GregorianCalendar date) {
+ return new org.apache.drill.common.expression.ValueExpressions.DateExpression(date.getTimeInMillis());
+ }
+
+ public static LogicalExpression getTime(GregorianCalendar time) {
+ int millis = time.get(GregorianCalendar.HOUR_OF_DAY) * 60 * 60 * 1000 +
+ time.get(GregorianCalendar.MINUTE) * 60 * 1000 +
+ time.get(GregorianCalendar.SECOND) * 1000 +
+ time.get(GregorianCalendar.MILLISECOND);
+
+ return new TimeExpression(millis);
+ }
+
+ public static LogicalExpression getTimeStamp(GregorianCalendar date) {
+ return new org.apache.drill.common.expression.ValueExpressions.TimeStampExpression(date.getTimeInMillis());
+ }
+ public static LogicalExpression getIntervalYear(int months) {
+ return new IntervalYearExpression(months);
+ }
+
+ public static LogicalExpression getIntervalDay(long intervalInMillis) {
+ return new IntervalDayExpression(intervalInMillis);
+ }
+
+ public static LogicalExpression getDecimal9(BigDecimal i) {
+ return new Decimal9Expression(i, ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getDecimal18(BigDecimal i) {
+ return new Decimal18Expression(i, ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getDecimal28(BigDecimal i) {
+ return new Decimal28Expression(i, ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getDecimal38(BigDecimal i) {
+ return new Decimal38Expression(i, ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getNumericExpression(String sign, String s, ExpressionPosition ep) {
+ String numStr = (sign == null) ? s : sign+s;
+ try {
+ int a = Integer.parseInt(numStr);
+ return new IntExpression(a, ep);
+ } catch (Exception e) {
+
+ }
+ try {
+ long l = Long.parseLong(numStr);
+ return new LongExpression(l, ep);
+ } catch (Exception e) {
+
+ }
+
+ try {
+ double d = Double.parseDouble(numStr);
+ return new DoubleExpression(d, ep);
+ } catch (Exception e) {
+
+ }
+
+ throw new IllegalArgumentException(String.format("Unable to parse string %s as integer or floating point number.",
+ numStr));
+
+ }
+
+ protected static abstract class ValueExpression<V> extends LogicalExpressionBase {
+ public final V value;
+
+ protected ValueExpression(String value, ExpressionPosition pos) {
+ super(pos);
+ this.value = parseValue(value);
+ }
+
+ protected abstract V parseValue(String s);
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+ }
+
+ public static class BooleanExpression extends ValueExpression<Boolean> {
+
+
+ public BooleanExpression(String value, ExpressionPosition pos) {
+ super(value, pos);
+ }
+
+
+ @Override
+ protected Boolean parseValue(String s) {
+ return Boolean.parseBoolean(s);
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return Types.REQUIRED_BIT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitBooleanConstant(this, value);
+ }
+
+ public boolean getBoolean() {
+ return value;
+ }
+
+ }
+
+ public static class FloatExpression extends LogicalExpressionBase {
+ private float f;
+
+ private static final MajorType FLOAT_CONSTANT = Types.required(MinorType.FLOAT4);
+
+ public FloatExpression(float f, ExpressionPosition pos) {
+ super(pos);
+ this.f = f;
+ }
+
+ public float getFloat() {
+ return f;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return FLOAT_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitFloatConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class IntExpression extends LogicalExpressionBase {
+
+ private static final MajorType INT_CONSTANT = Types.required(MinorType.INT);
+
+ private int i;
+
+ public IntExpression(int i, ExpressionPosition pos) {
+ super(pos);
+ this.i = i;
+ }
+
+ public int getInt() {
+ return i;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return INT_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitIntConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class Decimal9Expression extends LogicalExpressionBase {
+
+ private int decimal;
+ private int scale;
+ private int precision;
+
+ public Decimal9Expression(BigDecimal input, ExpressionPosition pos) {
+ super(pos);
+ this.scale = input.scale();
+ this.precision = input.precision();
+ this.decimal = CoreDecimalUtility.getDecimal9FromBigDecimal(input, scale, precision);
+ }
+
+
+ public int getIntFromDecimal() {
+ return decimal;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return MajorType.newBuilder().setMinorType(MinorType.DECIMAL9).setScale(scale).setPrecision(precision).setMode(DataMode.REQUIRED).build();
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitDecimal9Constant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+ }
+
+ public static class Decimal18Expression extends LogicalExpressionBase {
+
+ private long decimal;
+ private int scale;
+ private int precision;
+
+ public Decimal18Expression(BigDecimal input, ExpressionPosition pos) {
+ super(pos);
+ this.scale = input.scale();
+ this.precision = input.precision();
+ this.decimal = CoreDecimalUtility.getDecimal18FromBigDecimal(input, scale, precision);
+ }
+
+
+ public long getLongFromDecimal() {
+ return decimal;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return MajorType.newBuilder().setMinorType(MinorType.DECIMAL18).setScale(scale).setPrecision(precision).setMode(DataMode.REQUIRED).build();
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitDecimal18Constant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class Decimal28Expression extends LogicalExpressionBase {
+
+ private BigDecimal bigDecimal;
+
+ public Decimal28Expression(BigDecimal input, ExpressionPosition pos) {
+ super(pos);
+ this.bigDecimal = input;
+ }
+
+
+ public BigDecimal getBigDecimal() {
+ return bigDecimal;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return MajorType.newBuilder().setMinorType(MinorType.DECIMAL28SPARSE).setScale(bigDecimal.scale()).setPrecision(bigDecimal.precision()).setMode(DataMode.REQUIRED).build();
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitDecimal28Constant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class Decimal38Expression extends LogicalExpressionBase {
+
+ private BigDecimal bigDecimal;
+
+ public Decimal38Expression(BigDecimal input, ExpressionPosition pos) {
+ super(pos);
+ this.bigDecimal = input;
+ }
+
+ public BigDecimal getBigDecimal() {
+ return bigDecimal;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return MajorType.newBuilder().setMinorType(MinorType.DECIMAL38SPARSE).setScale(bigDecimal.scale()).setPrecision(bigDecimal.precision()).setMode(DataMode.REQUIRED).build();
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitDecimal38Constant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+
+ public static class DoubleExpression extends LogicalExpressionBase {
+ private double d;
+
+ private static final MajorType DOUBLE_CONSTANT = Types.required(MinorType.FLOAT8);
+
+ public DoubleExpression(double d, ExpressionPosition pos) {
+ super(pos);
+ this.d = d;
+ }
+
+ public double getDouble() {
+ return d;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return DOUBLE_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitDoubleConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class LongExpression extends LogicalExpressionBase {
+
+ private static final MajorType LONG_CONSTANT = Types.required(MinorType.BIGINT);
+
+ private long l;
+
+ public LongExpression(long l) {
+ this(l, ExpressionPosition.UNKNOWN);
+ }
+
+ public LongExpression(long l, ExpressionPosition pos) {
+ super(pos);
+ this.l = l;
+ }
+
+ public long getLong() {
+ return l;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return LONG_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitLongConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+
+ public static class DateExpression extends LogicalExpressionBase {
+
+ private static final MajorType DATE_CONSTANT = Types.required(MinorType.DATE);
+
+ private long dateInMillis;
+
+ public DateExpression(long l) {
+ this(l, ExpressionPosition.UNKNOWN);
+ }
+
+ public DateExpression(long dateInMillis, ExpressionPosition pos) {
+ super(pos);
+ this.dateInMillis = dateInMillis;
+ }
+
+ public long getDate() {
+ return dateInMillis;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return DATE_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitDateConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+
+ public static class TimeExpression extends LogicalExpressionBase {
+
+ private static final MajorType TIME_CONSTANT = Types.required(MinorType.TIME);
+
+ private int timeInMillis;
+
+ public TimeExpression(int timeInMillis) {
+ this(timeInMillis, ExpressionPosition.UNKNOWN);
+ }
+
+ public TimeExpression(int timeInMillis, ExpressionPosition pos) {
+ super(pos);
+ this.timeInMillis = timeInMillis;
+ }
+
+ public int getTime() {
+ return timeInMillis;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return TIME_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitTimeConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class TimeStampExpression extends LogicalExpressionBase {
+
+ private static final MajorType TIMESTAMP_CONSTANT = Types.required(MinorType.TIMESTAMP);
+
+ private long timeInMillis;
+
+ public TimeStampExpression(long timeInMillis) {
+ this(timeInMillis, ExpressionPosition.UNKNOWN);
+ }
+
+ public TimeStampExpression(long timeInMillis, ExpressionPosition pos) {
+ super(pos);
+ this.timeInMillis = timeInMillis;
+ }
+
+ public long getTimeStamp() {
+ return timeInMillis;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return TIMESTAMP_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitTimeStampConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class IntervalYearExpression extends LogicalExpressionBase {
+
+ private static final MajorType INTERVALYEAR_CONSTANT = Types.required(MinorType.INTERVALYEAR);
+
+ private int months;
+
+ public IntervalYearExpression(int months) {
+ this(months, ExpressionPosition.UNKNOWN);
+ }
+
+ public IntervalYearExpression(int months, ExpressionPosition pos) {
+ super(pos);
+ this.months = months;
+ }
+
+ public int getIntervalYear() {
+ return months;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return INTERVALYEAR_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitIntervalYearConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class IntervalDayExpression extends LogicalExpressionBase {
+
+ private static final MajorType INTERVALDAY_CONSTANT = Types.required(MinorType.INTERVALDAY);
+ private static final long MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
+
+ private int days;
+ private int millis;
+
+ public IntervalDayExpression(long intervalInMillis) {
+ this((int) (intervalInMillis / MILLIS_IN_DAY), (int) (intervalInMillis % MILLIS_IN_DAY), ExpressionPosition.UNKNOWN);
+ }
+
+ public IntervalDayExpression(int days, int millis, ExpressionPosition pos) {
+ super(pos);
+ this.days = days;
+ this.millis = millis;
+ }
+
+ public int getIntervalDay() {
+ return days;
+ }
+
+ public int getIntervalMillis() {
+ return millis;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return INTERVALDAY_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitIntervalDayConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class QuotedString extends ValueExpression<String> {
+
+ private static final MajorType QUOTED_STRING_CONSTANT = Types.required(MinorType.VARCHAR);
+
+ public QuotedString(String value, ExpressionPosition pos) {
+ super(value, pos);
+ }
+
+ public String getString() {
+ return value;
+ }
+
+ @Override
+ protected String parseValue(String s) {
+ return s;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return QUOTED_STRING_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitQuotedStringConstant(this, value);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java b/logical/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
new file mode 100644
index 0000000..34997ab
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.fn;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public class CastFunctions {
+
+ private static Map<MinorType, String> TYPE2FUNC = new HashMap<>();
+ /** The cast functions that need to be replaced (if
+ * "drill.exec.functions.cast_empty_string_to_null" is set to true). */
+ private static Set<String> CAST_FUNC_REPLACEMENT_NEEDED = new HashSet<>();
+ /** Map from the replaced functions to the new ones (for non-nullable VARCHAR). */
+ private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE = new HashMap<>();
+ /** Map from the replaced functions to the new ones (for nullable VARCHAR). */
+ private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NULLABLE = new HashMap<>();
+
+ static {
+ TYPE2FUNC.put(MinorType.UNION, "castUNION");
+ TYPE2FUNC.put(MinorType.BIGINT, "castBIGINT");
+ TYPE2FUNC.put(MinorType.INT, "castINT");
+ TYPE2FUNC.put(MinorType.BIT, "castBIT");
+ TYPE2FUNC.put(MinorType.TINYINT, "castTINYINT");
+ TYPE2FUNC.put(MinorType.FLOAT4, "castFLOAT4");
+ TYPE2FUNC.put(MinorType.FLOAT8, "castFLOAT8");
+ TYPE2FUNC.put(MinorType.VARCHAR, "castVARCHAR");
+ TYPE2FUNC.put(MinorType.VAR16CHAR, "castVAR16CHAR");
+ TYPE2FUNC.put(MinorType.VARBINARY, "castVARBINARY");
+ TYPE2FUNC.put(MinorType.DATE, "castDATE");
+ TYPE2FUNC.put(MinorType.TIME, "castTIME");
+ TYPE2FUNC.put(MinorType.TIMESTAMP, "castTIMESTAMP");
+ TYPE2FUNC.put(MinorType.TIMESTAMPTZ, "castTIMESTAMPTZ");
+ TYPE2FUNC.put(MinorType.INTERVALDAY, "castINTERVALDAY");
+ TYPE2FUNC.put(MinorType.INTERVALYEAR, "castINTERVALYEAR");
+ TYPE2FUNC.put(MinorType.INTERVAL, "castINTERVAL");
+ TYPE2FUNC.put(MinorType.DECIMAL9, "castDECIMAL9");
+ TYPE2FUNC.put(MinorType.DECIMAL18, "castDECIMAL18");
+ TYPE2FUNC.put(MinorType.DECIMAL28SPARSE, "castDECIMAL28SPARSE");
+ TYPE2FUNC.put(MinorType.DECIMAL28DENSE, "castDECIMAL28DENSE");
+ TYPE2FUNC.put(MinorType.DECIMAL38SPARSE, "castDECIMAL38SPARSE");
+ TYPE2FUNC.put(MinorType.DECIMAL38DENSE, "castDECIMAL38DENSE");
+
+ CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.INT));
+ CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.BIGINT));
+ CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.FLOAT4));
+ CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.FLOAT8));
+ CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL9));
+ CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL18));
+ CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE));
+ CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE));
+
+ CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringVarCharToNullableINT");
+ CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringVarCharToNullableBIGINT");
+ CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringVarCharToNullableFLOAT4");
+ CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringVarCharToNullableFLOAT8");
+ CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringVarCharToNullableDECIMAL9");
+ CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringVarCharToNullableDECIMAL18");
+ CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringVarCharToNullableDECIMAL28SPARSE");
+ CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringVarCharToNullableDECIMAL38SPARSE");
+
+ CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringNullableVarCharToNullableINT");
+ CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringNullableVarCharToNullableBIGINT");
+ CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringNullableVarCharToNullableFLOAT4");
+ CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringNullableVarCharToNullableFLOAT8");
+ CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringNullableVarCharToNullableDECIMAL9");
+ CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringNullableVarCharToNullableDECIMAL18");
+ CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringNullableVarCharToNullableDECIMAL28SPARSE");
+ CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringNullableVarCharToNullableDECIMAL38SPARSE");
+ }
+
+ /**
+ * Given the target type, get the appropriate cast function
+ * @param targetMinorType the target data type
+ * @return
+ */
+ public static String getCastFunc(MinorType targetMinorType) {
+ String func = TYPE2FUNC.get(targetMinorType);
+ if (func != null) {
+ return func;
+ }
+
+ throw new RuntimeException(
+ String.format("cast function for type %s is not defined", targetMinorType.name()));
+ }
+
+ /**
+ * Get a replacing cast function for the original function, based on the specified data mode
+ * @param originalCastFunction original cast function
+ * @param dataMode data mode of the input data
+ * @return
+ */
+ public static String getReplacingCastFunction(String originalCastFunction, org.apache.drill.common.types.TypeProtos.DataMode dataMode) {
+ if(dataMode == TypeProtos.DataMode.OPTIONAL) {
+ return getReplacingCastFunctionFromNullable(originalCastFunction);
+ }
+
+ if(dataMode == TypeProtos.DataMode.REQUIRED) {
+ return getReplacingCastFunctionFromNonNullable(originalCastFunction);
+ }
+
+ throw new RuntimeException(
+ String.format("replacing cast function for datatype %s is not defined", dataMode));
+ }
+
+ /**
+ * Check if a replacing cast function is available for the the original function
+ * @param originalfunction original cast function
+ * @return
+ */
+ public static boolean isReplacementNeeded(MinorType inputType, String originalfunction) {
+ return inputType == MinorType.VARCHAR && CAST_FUNC_REPLACEMENT_NEEDED.contains(originalfunction);
+ }
+
+ private static String getReplacingCastFunctionFromNonNullable(String originalCastFunction) {
+ if(CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.containsKey(originalCastFunction)) {
+ return CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.get(originalCastFunction);
+ }
+
+ throw new RuntimeException(
+ String.format("replacing cast function for %s is not defined", originalCastFunction));
+ }
+
+ private static String getReplacingCastFunctionFromNullable(String originalCastFunction) {
+ if(CAST_FUNC_REPLACEMENT_FROM_NULLABLE.containsKey(originalCastFunction)) {
+ return CAST_FUNC_REPLACEMENT_FROM_NULLABLE.get(originalCastFunction);
+ }
+
+ throw new RuntimeException(
+ String.format("replacing cast function for %s is not defined", originalCastFunction));
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/fn/FuncHolder.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/fn/FuncHolder.java b/logical/src/main/java/org/apache/drill/common/expression/fn/FuncHolder.java
new file mode 100644
index 0000000..a8824e0
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/fn/FuncHolder.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.fn;
+
+/** This should be removed once common and exec/java-exec modules are merged (DRILL-507). */
+public interface FuncHolder {
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/fn/package-info.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/fn/package-info.java b/logical/src/main/java/org/apache/drill/common/expression/fn/package-info.java
new file mode 100644
index 0000000..6af3f5f
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/fn/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Logical function definitions.
+ */
+package org.apache.drill.common.expression.fn;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/package-info.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/package-info.java b/logical/src/main/java/org/apache/drill/common/expression/package-info.java
new file mode 100644
index 0000000..0c32bf6
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Logical expression tree representation.
+ *
+ * Drill manages expressions provided in many different parts of SQL queries.
+ * This includes scalar expressions in select, filter and join conditions, as
+ * well as aggregate and window functions. These expressions are represented
+ * logically as ASTs during planning. The classes defined here provide the
+ * different nodes in the expression tree, as well as utilities for building
+ * and manipulating expressions during parsing and planning.
+ *
+ */
+package org.apache.drill.common.expression;
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/types/AtomType.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/types/AtomType.java b/logical/src/main/java/org/apache/drill/common/expression/types/AtomType.java
new file mode 100644
index 0000000..2918216
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/types/AtomType.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.types;
+
+
+public class AtomType extends DataType {
+ private String name;
+ private Comparability comparability;
+ private boolean isNumericType;
+
+ public AtomType(String name, Comparability comparability, boolean isNumericType) {
+ super();
+ this.name = name;
+ this.comparability = comparability;
+ this.isNumericType = isNumericType;
+ }
+
+
+ @Override
+ public boolean isNumericType() {
+ return isNumericType;
+ }
+
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public boolean isLateBind() {
+ return false;
+ }
+
+ @Override
+ public boolean hasChildType() {
+ return false;
+ }
+
+ @Override
+ public DataType getChildType() {
+ return null;
+ }
+
+ @Override
+ public Comparability getComparability() {
+ return comparability;
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/types/DataType.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/types/DataType.java b/logical/src/main/java/org/apache/drill/common/expression/types/DataType.java
new file mode 100644
index 0000000..56102ee
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/types/DataType.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.types;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+
+@JsonSerialize(using = DataType.Se.class)
+@JsonDeserialize(using = DataType.De.class)
+abstract class DataType {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataType.class);
+
+ public static enum Comparability{
+ UNKNOWN, NONE, EQUAL, ORDERED;
+ }
+
+ public abstract String getName();
+ public abstract boolean isLateBind();
+ public abstract boolean hasChildType();
+ public abstract DataType getChildType();
+ public abstract Comparability getComparability();
+ public abstract boolean isNumericType();
+
+
+
+ public static final DataType LATEBIND = new LateBindType();
+ public static final DataType BOOLEAN = new AtomType("BOOLEAN", Comparability.EQUAL, false);
+ public static final DataType BYTES = new AtomType("BYTES", Comparability.ORDERED, false);
+ public static final DataType SIGNED_BYTE = new AtomType("SIGNED_BYTE", Comparability.ORDERED, true);
+ public static final DataType SIGNED_INT16 = new AtomType("SIGNED_INT16", Comparability.ORDERED, true);
+ public static final DataType NVARCHAR = new AtomType("VARCHAR", Comparability.ORDERED, false);
+ public static final DataType FLOAT32 = new AtomType("FLOAT32", Comparability.ORDERED, true);
+ public static final DataType FLOAT64 = new AtomType("FLOAT64", Comparability.ORDERED, true);
+ public static final DataType INT64 = new AtomType("INT64", Comparability.ORDERED, true);
+ public static final DataType INT32 = new AtomType("INT32", Comparability.ORDERED, true);
+ public static final DataType INT16 = new AtomType("INT16", Comparability.ORDERED, true);
+ public static final DataType UINT16 = new AtomType("UINT16", Comparability.ORDERED, true);
+// public static final DataType INT16 = new AtomType("int16", Comparability.ORDERED, true);
+// public static final DataType BIG_INTEGER = new AtomType("bigint", Comparability.ORDERED, true);
+// public static final DataType BIG_DECIMAL = new AtomType("bigdecimal", Comparability.ORDERED, true);
+ public static final DataType DATE = new AtomType("DATE", Comparability.ORDERED, false);
+ public static final DataType DATETIME = new AtomType("DATETIME", Comparability.ORDERED, false);
+ public static final DataType MAP = new AtomType("MAP", Comparability.NONE, false);
+ public static final DataType ARRAY = new AtomType("ARRAY", Comparability.NONE, false);
+ public static final DataType NULL = new AtomType("NULL", Comparability.NONE, false);
+
+ //TODO: Hack to get some size data, needs to be fixed so that each type reveals it's size.
+ public int size() {
+ if(this == BOOLEAN) {
+ return 1;
+ }else if(this == INT32) {
+ return 4;
+ }else if(this == INT16) {
+ return 4;
+ }
+ return 2;
+ }
+
+ static final Map<String, DataType> TYPES;
+ static {
+ Field[] fields = DataType.class.getFields();
+ Map<String, DataType> types = new HashMap<String, DataType>();
+ for (Field f : fields) {
+ //logger.debug("Reviewing {}, Field: {}", f.getClass(), f);
+ if (Modifier.isStatic(f.getModifiers())) {
+ try {
+ Object o = f.get(null);
+ //logger.debug("Object {}", o);
+
+ if (o instanceof DataType) {
+ types.put(((DataType) o).getName(), (DataType) o);
+ }
+ } catch (IllegalArgumentException | IllegalAccessException e) {
+ logger.warn("Failure while reading DataType.", e);
+ }
+ }
+ }
+ TYPES = Collections.unmodifiableMap(types);
+ }
+
+ public static DataType getDataType(String name) {
+ if (TYPES.containsKey(name)) {
+ return TYPES.get(name);
+ } else {
+ throw new IllegalArgumentException(String.format("Unknown type requested of [%s].", name));
+ }
+ }
+
+ public static class De extends StdDeserializer<DataType> {
+
+ public De() {
+ super(DataType.class);
+ }
+
+ @Override
+ public DataType deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
+ JsonProcessingException {
+ return getDataType(this._parseString(jp, ctxt));
+ }
+
+ }
+
+ public static class Se extends StdSerializer<DataType> {
+
+ public Se() {
+ super(DataType.class);
+ }
+
+ @Override
+ public void serialize(DataType value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
+ JsonGenerationException {
+ jgen.writeString(value.getName());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/types/DataTypeFactory.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/types/DataTypeFactory.java b/logical/src/main/java/org/apache/drill/common/expression/types/DataTypeFactory.java
new file mode 100644
index 0000000..c95bf1f
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/types/DataTypeFactory.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.types;
+
+public abstract class DataTypeFactory {
+
+ public abstract DataType getArrayType(DataType containedType);
+ public abstract DataType getMapType(DataType keyType, DataType valueType);
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/types/LateBindType.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/types/LateBindType.java b/logical/src/main/java/org/apache/drill/common/expression/types/LateBindType.java
new file mode 100644
index 0000000..db0b09c
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/types/LateBindType.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.types;
+
+class LateBindType extends DataType {
+
+ @Override
+ public String getName() {
+ return "LATE";
+ }
+
+ @Override
+ public boolean isLateBind() {
+ return true;
+ }
+
+ @Override
+ public boolean hasChildType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataType getChildType() {
+ throw new UnsupportedOperationException();
+ }
+
+
+ @Override
+ public boolean isNumericType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Comparability getComparability() {
+ return Comparability.UNKNOWN;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/types/package-info.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/types/package-info.java b/logical/src/main/java/org/apache/drill/common/expression/types/package-info.java
new file mode 100644
index 0000000..744b174
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/types/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Logical type definitions.
+ */
+package org.apache.drill.common.expression.types;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
new file mode 100644
index 0000000..a31e6e8
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+import org.apache.drill.common.expression.BooleanOperator;
+import org.apache.drill.common.expression.CastExpression;
+import org.apache.drill.common.expression.ConvertExpression;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.NullExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedNullConstant;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
+
+public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> implements ExprVisitor<T, VAL, EXCEP> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractExprVisitor.class);
+
+ @Override
+ public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP {
+ return visitUnknown(call, value);
+ }
+
+ @Override
+ public T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP {
+ return visitUnknown(holder, value);
+ }
+
+ @Override
+ public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP {
+ return visitUnknown(ifExpr, value);
+ }
+
+ @Override
+ public T visitBooleanOperator(BooleanOperator op, VAL value) throws EXCEP {
+ return visitUnknown(op, value);
+ }
+
+ @Override
+ public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP {
+ return visitUnknown(path, value);
+ }
+
+ @Override
+ public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP {
+ return visitUnknown(fExpr, value);
+ }
+
+ @Override
+ public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP {
+ return visitUnknown(intExpr, value);
+ }
+
+ @Override
+ public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP {
+ return visitUnknown(intExpr, value);
+ }
+
+
+ @Override
+ public T visitDecimal9Constant(Decimal9Expression decExpr, VAL value) throws EXCEP {
+ return visitUnknown(decExpr, value);
+ }
+
+ @Override
+ public T visitDecimal18Constant(Decimal18Expression decExpr, VAL value) throws EXCEP {
+ return visitUnknown(decExpr, value);
+ }
+
+ @Override
+ public T visitDecimal28Constant(Decimal28Expression decExpr, VAL value) throws EXCEP {
+ return visitUnknown(decExpr, value);
+ }
+
+ @Override
+ public T visitDecimal38Constant(Decimal38Expression decExpr, VAL value) throws EXCEP {
+ return visitUnknown(decExpr, value);
+ }
+
+ @Override
+ public T visitDateConstant(DateExpression intExpr, VAL value) throws EXCEP {
+ return visitUnknown(intExpr, value);
+ }
+
+ @Override
+ public T visitTimeConstant(TimeExpression intExpr, VAL value) throws EXCEP {
+ return visitUnknown(intExpr, value);
+ }
+
+ @Override
+ public T visitTimeStampConstant(TimeStampExpression intExpr, VAL value) throws EXCEP {
+ return visitUnknown(intExpr, value);
+ }
+
+ @Override
+ public T visitIntervalYearConstant(IntervalYearExpression intExpr, VAL value) throws EXCEP {
+ return visitUnknown(intExpr, value);
+ }
+
+ @Override
+ public T visitIntervalDayConstant(IntervalDayExpression intExpr, VAL value) throws EXCEP {
+ return visitUnknown(intExpr, value);
+ }
+
+ @Override
+ public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP {
+ return visitUnknown(dExpr, value);
+ }
+
+ @Override
+ public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP {
+ return visitUnknown(e, value);
+ }
+
+ @Override
+ public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP {
+ return visitUnknown(e, value);
+ }
+
+ @Override
+ public T visitCastExpression(CastExpression e, VAL value) throws EXCEP {
+ return visitUnknown(e, value);
+ }
+
+ @Override
+ public T visitConvertExpression(ConvertExpression e, VAL value) throws EXCEP {
+ return visitUnknown(e, value);
+ }
+
+ @Override
+ public T visitNullConstant(TypedNullConstant e, VAL value) throws EXCEP {
+ return visitUnknown(e, value);
+ }
+
+ @Override
+ public T visitNullExpression(NullExpression e, VAL value) throws EXCEP {
+ return visitUnknown(e, value);
+ }
+
+ @Override
+ public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP {
+ throw new UnsupportedOperationException(String.format("Expression of type %s not handled by visitor type %s.", e.getClass().getCanonicalName(), this.getClass().getCanonicalName()));
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
new file mode 100644
index 0000000..46789e3
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+import org.apache.drill.common.expression.BooleanOperator;
+import org.apache.drill.common.expression.CastExpression;
+import org.apache.drill.common.expression.ConvertExpression;
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.IfExpression.IfCondition;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.NullExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedNullConstant;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
+
+public final class AggregateChecker implements ExprVisitor<Boolean, ErrorCollector, RuntimeException>{
+
+ public static final AggregateChecker INSTANCE = new AggregateChecker();
+
+ public static boolean isAggregating(LogicalExpression e, ErrorCollector errors) {
+ return e.accept(INSTANCE, errors);
+ }
+
+ @Override
+ public Boolean visitFunctionCall(FunctionCall call, ErrorCollector errors) {
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "+
+ "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors) {
+ if (holder.isAggregating()) {
+ for (int i = 0; i < holder.args.size(); i++) {
+ LogicalExpression e = holder.args.get(i);
+ if(e.accept(this, errors)) {
+ errors.addGeneralError(e.getPosition(),
+ String.format("Aggregating function call %s includes nested aggregations at arguments number %d. " +
+ "This isn't allowed.", holder.getName(), i));
+ }
+ }
+ return true;
+ } else {
+ for (LogicalExpression e : holder.args) {
+ if (e.accept(this, errors)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public Boolean visitBooleanOperator(BooleanOperator op, ErrorCollector errors) {
+ for (LogicalExpression arg : op.args) {
+ if (arg.accept(this, errors)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public Boolean visitIfExpression(IfExpression ifExpr, ErrorCollector errors) {
+ IfCondition c = ifExpr.ifCondition;
+ if (c.condition.accept(this, errors) || c.expression.accept(this, errors)) {
+ return true;
+ }
+ return ifExpr.elseExpression.accept(this, errors);
+ }
+
+ @Override
+ public Boolean visitSchemaPath(SchemaPath path, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitIntConstant(IntExpression intExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitFloatConstant(FloatExpression fExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitLongConstant(LongExpression intExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitBooleanConstant(BooleanExpression e, ErrorCollector errors) {
+ return false;
+ }
+ public Boolean visitDecimal9Constant(Decimal9Expression decExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDecimal18Constant(Decimal18Expression decExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDecimal28Constant(Decimal28Expression decExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDecimal38Constant(Decimal38Expression decExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitQuotedStringConstant(QuotedString e, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitUnknown(LogicalExpression e, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitCastExpression(CastExpression e, ErrorCollector errors) {
+ return e.getInput().accept(this, errors);
+ }
+
+ @Override
+ public Boolean visitConvertExpression(ConvertExpression e, ErrorCollector errors) throws RuntimeException {
+ return e.getInput().accept(this, errors);
+ }
+
+ @Override
+ public Boolean visitDateConstant(DateExpression intExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitTimeConstant(TimeExpression intExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitTimeStampConstant(TimeStampExpression intExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitIntervalYearConstant(IntervalYearExpression intExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitIntervalDayConstant(IntervalDayExpression intExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitNullConstant(TypedNullConstant e, ErrorCollector value) throws RuntimeException {
+ return false;
+ }
+
+ @Override
+ public Boolean visitNullExpression(NullExpression e, ErrorCollector value) throws RuntimeException {
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
new file mode 100644
index 0000000..254926a
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.common.expression.visitors;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.drill.common.expression.BooleanOperator;
+import org.apache.drill.common.expression.CastExpression;
+import org.apache.drill.common.expression.ConvertExpression;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.IfExpression.IfCondition;
+import org.apache.drill.common.expression.LogicalExpression;
+
+import com.google.common.collect.Lists;
+
+public class ConditionalExprOptimizer extends AbstractExprVisitor<LogicalExpression, Void, RuntimeException> {
+
+ public static ConditionalExprOptimizer INSTANCE = new ConditionalExprOptimizer();
+
+ @Override
+ public LogicalExpression visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException {
+
+ List<LogicalExpression> newArgs = Lists.newArrayList();
+
+ newArgs.addAll(op.args);
+
+ Collections.sort(newArgs, costComparator);
+
+ return new BooleanOperator(op.getName(), newArgs, op.getPosition());
+ }
+
+
+ @Override
+ public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holder, Void value) throws RuntimeException {
+ List<LogicalExpression> args = Lists.newArrayList();
+ for (int i = 0; i < holder.args.size(); ++i) {
+ LogicalExpression newExpr = holder.args.get(i).accept(this, value);
+ assert newExpr != null;
+ args.add(newExpr);
+ }
+
+ //replace with a new function call, since its argument could be changed.
+
+ return holder.copy(args);
+ }
+
+ @Override
+ public LogicalExpression visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
+ return e;
+ }
+
+
+ @Override
+ public LogicalExpression visitIfExpression(IfExpression ifExpr, Void value) throws RuntimeException{
+ LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, value);
+ IfCondition conditions = ifExpr.ifCondition;
+
+ LogicalExpression newCondition = conditions.condition.accept(this, value);
+ LogicalExpression newExpr = conditions.expression.accept(this, value);
+ conditions = new IfExpression.IfCondition(newCondition, newExpr);
+
+ return IfExpression.newBuilder().setElse(newElseExpr).setIfCondition(conditions).build();
+ }
+
+ @Override
+ public LogicalExpression visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "
+ + "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public LogicalExpression visitCastExpression(CastExpression cast, Void value) throws RuntimeException {
+ throw new UnsupportedOperationException("CastExpression is not expected here. "
+ + "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+
+ @Override
+ public LogicalExpression visitConvertExpression(ConvertExpression cast, Void value) throws RuntimeException {
+ throw new UnsupportedOperationException("ConvertExpression is not expected here. "
+ + "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ private static Comparator<LogicalExpression> costComparator = new Comparator<LogicalExpression> () {
+ public int compare(LogicalExpression e1, LogicalExpression e2) {
+ return e1.getCumulativeCost() <= e2.getCumulativeCost() ? -1 : 1;
+ }
+ };
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
new file mode 100644
index 0000000..ccb11a6
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+import org.apache.drill.common.expression.BooleanOperator;
+import org.apache.drill.common.expression.CastExpression;
+import org.apache.drill.common.expression.ConvertExpression;
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.IfExpression.IfCondition;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.NullExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedNullConstant;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
+
+final class ConstantChecker implements ExprVisitor<Boolean, ErrorCollector, RuntimeException> {
+
+
+ private final static ConstantChecker INSTANCE = new ConstantChecker();
+
+ private ConstantChecker() {}
+
+ public static void checkConstants(LogicalExpression e, ErrorCollector errors) {
+ e.accept(INSTANCE, errors);
+ }
+
+ @Override
+ public Boolean visitFunctionCall(FunctionCall call, ErrorCollector errors) {
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "
+ + "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors) {
+ boolean allArgsAreConstant = true;
+ for (int i = 0; i < holder.args.size(); i++) {
+ boolean thisArgIsConstant = holder.args.get(i).accept(this, errors);
+ if (!thisArgIsConstant) {
+ allArgsAreConstant = false;
+ if (holder.argConstantOnly(i)) {
+ errors.addGeneralError( //
+ holder.args.get(i).getPosition(), //
+ String.format("Function %s expects constant input for argument number %d", holder.getName(), i));
+ }
+ }
+ }
+ return allArgsAreConstant;
+ }
+
+ @Override
+ public Boolean visitBooleanOperator(BooleanOperator op, ErrorCollector errors) {
+ for (LogicalExpression e : op.args) {
+ if (!e.accept(this, errors)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public Boolean visitIfExpression(IfExpression ifExpr, ErrorCollector errors) {
+ IfCondition c = ifExpr.ifCondition;
+ if (!c.condition.accept(this, errors) || !c.expression.accept(this, errors)) {
+ return false;
+ }
+
+ if (!ifExpr.elseExpression.accept(this, errors)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public Boolean visitSchemaPath(SchemaPath path, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitIntConstant(IntExpression intExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitFloatConstant(FloatExpression fExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitLongConstant(LongExpression intExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitDateConstant(DateExpression intExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitTimeConstant(TimeExpression intExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitTimeStampConstant(TimeStampExpression intExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitIntervalYearConstant(IntervalYearExpression intExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitIntervalDayConstant(IntervalDayExpression intExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitDecimal9Constant(Decimal9Expression decExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDecimal18Constant(Decimal18Expression decExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDecimal28Constant(Decimal28Expression decExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDecimal38Constant(Decimal38Expression decExpr, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitBooleanConstant(BooleanExpression e, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitQuotedStringConstant(QuotedString e, ErrorCollector errors) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitUnknown(LogicalExpression e, ErrorCollector errors) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitCastExpression(CastExpression e, ErrorCollector value) throws RuntimeException {
+ return e.getInput().accept(this, value);
+ }
+
+ @Override
+ public Boolean visitConvertExpression(ConvertExpression e, ErrorCollector value) throws RuntimeException {
+ return e.getInput().accept(this, value);
+ }
+
+ @Override
+ public Boolean visitNullConstant(TypedNullConstant e, ErrorCollector value) throws RuntimeException {
+ return true;
+ }
+
+ @Override
+ public Boolean visitNullExpression(NullExpression e, ErrorCollector value) throws RuntimeException {
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
new file mode 100644
index 0000000..2d0cd8c
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+import org.apache.drill.common.expression.BooleanOperator;
+import org.apache.drill.common.expression.CastExpression;
+import org.apache.drill.common.expression.ConvertExpression;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.NullExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedNullConstant;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
+
+public interface ExprVisitor<T, VAL, EXCEP extends Exception> {
+ public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP;
+ public T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP;
+ public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP;
+ public T visitBooleanOperator(BooleanOperator call, VAL value) throws EXCEP;
+ public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
+ public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP;
+ public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP;
+ public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP;
+ public T visitDateConstant(DateExpression intExpr, VAL value) throws EXCEP;
+ public T visitTimeConstant(TimeExpression intExpr, VAL value) throws EXCEP;
+ public T visitTimeStampConstant(TimeStampExpression intExpr, VAL value) throws EXCEP;
+ public T visitIntervalYearConstant(IntervalYearExpression intExpr, VAL value) throws EXCEP;
+ public T visitIntervalDayConstant(IntervalDayExpression intExpr, VAL value) throws EXCEP;
+ public T visitDecimal9Constant(Decimal9Expression decExpr, VAL value) throws EXCEP;
+ public T visitDecimal18Constant(Decimal18Expression decExpr, VAL value) throws EXCEP;
+ public T visitDecimal28Constant(Decimal28Expression decExpr, VAL value) throws EXCEP;
+ public T visitDecimal38Constant(Decimal38Expression decExpr, VAL value) throws EXCEP;
+ public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP;
+ public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;
+ public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP;
+ public T visitNullConstant(TypedNullConstant e, VAL value) throws EXCEP;
+ public T visitNullExpression(NullExpression e, VAL value) throws EXCEP;
+ public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP;
+ public T visitCastExpression(CastExpression e, VAL value) throws EXCEP;
+ public T visitConvertExpression(ConvertExpression e, VAL value) throws EXCEP;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
new file mode 100644
index 0000000..e905638
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+public class ExpressionValidationError {
+ String message;
+
+ public ExpressionValidationError(String message) {
+ this.message = message;
+ }
+
+ @Override
+ public String toString() {
+ return message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
new file mode 100644
index 0000000..3d944da
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+public class ExpressionValidationException extends RuntimeException {
+
+ public ExpressionValidationException() {
+ super();
+ }
+
+ public ExpressionValidationException(String arg0, Throwable arg1) {
+ super(arg0, arg1);
+ }
+
+ public ExpressionValidationException(String arg0) {
+ super(arg0);
+ }
+
+ public ExpressionValidationException(Throwable arg0) {
+ super(arg0);
+ }
+
+}
[27/45] drill git commit: DRILL-3987: (MOVE) Extract RPC,
memory-base and memory-impl as separate modules.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
deleted file mode 100644
index cf09be3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.bootstrap.Bootstrap;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.timeout.IdleState;
-import io.netty.handler.timeout.IdleStateEvent;
-import io.netty.handler.timeout.IdleStateHandler;
-import io.netty.util.concurrent.Future;
-import io.netty.util.concurrent.GenericFutureListener;
-
-import java.net.SocketAddress;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
-import org.apache.drill.exec.rpc.RpcConnectionHandler.FailureType;
-
-import com.google.protobuf.Internal.EnumLite;
-import com.google.protobuf.MessageLite;
-import com.google.protobuf.Parser;
-
-public abstract class BasicClient<T extends EnumLite, R extends RemoteConnection, HANDSHAKE_SEND extends MessageLite, HANDSHAKE_RESPONSE extends MessageLite>
- extends RpcBus<T, R> {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicClient.class);
-
- // The percentage of time that should pass before sending a ping message to ensure server doesn't time us out. For
- // example, if timeout is set to 30 seconds and we set percentage to 0.5, then if no write has happened within 15
- // seconds, the idle state handler will send a ping message.
- private static final double PERCENT_TIMEOUT_BEFORE_SENDING_PING = 0.5;
-
- private final Bootstrap b;
- protected R connection;
- private final T handshakeType;
- private final Class<HANDSHAKE_RESPONSE> responseClass;
- private final Parser<HANDSHAKE_RESPONSE> handshakeParser;
-
- private final IdlePingHandler pingHandler;
-
- public BasicClient(RpcConfig rpcMapping, ByteBufAllocator alloc, EventLoopGroup eventLoopGroup, T handshakeType,
- Class<HANDSHAKE_RESPONSE> responseClass, Parser<HANDSHAKE_RESPONSE> handshakeParser) {
- super(rpcMapping);
- this.responseClass = responseClass;
- this.handshakeType = handshakeType;
- this.handshakeParser = handshakeParser;
- final long timeoutInMillis = rpcMapping.hasTimeout() ? (long) (rpcMapping.getTimeout() * 1000.0 * PERCENT_TIMEOUT_BEFORE_SENDING_PING)
- : -1;
- this.pingHandler = rpcMapping.hasTimeout() ? new IdlePingHandler(timeoutInMillis) : null;
-
- b = new Bootstrap() //
- .group(eventLoopGroup) //
- .channel(TransportCheck.getClientSocketChannel()) //
- .option(ChannelOption.ALLOCATOR, alloc) //
- .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30 * 1000)
- .option(ChannelOption.SO_REUSEADDR, true)
- .option(ChannelOption.SO_RCVBUF, 1 << 17) //
- .option(ChannelOption.SO_SNDBUF, 1 << 17) //
- .option(ChannelOption.TCP_NODELAY, true)
- .handler(new ChannelInitializer<SocketChannel>() {
-
- @Override
- protected void initChannel(SocketChannel ch) throws Exception {
- // logger.debug("initializing client connection.");
- connection = initRemoteConnection(ch);
-
- ch.closeFuture().addListener(getCloseHandler(ch, connection));
-
- final ChannelPipeline pipe = ch.pipeline();
-
- pipe.addLast("protocol-decoder", getDecoder(connection.getAllocator()));
- pipe.addLast("message-decoder", new RpcDecoder("c-" + rpcConfig.getName()));
- pipe.addLast("protocol-encoder", new RpcEncoder("c-" + rpcConfig.getName()));
- pipe.addLast("handshake-handler", new ClientHandshakeHandler());
-
- if(pingHandler != null){
- pipe.addLast("idle-state-handler", pingHandler);
- }
-
- pipe.addLast("message-handler", new InboundHandler(connection));
- pipe.addLast("exception-handler", new RpcExceptionHandler(connection));
- }
- }); //
-
- // if(TransportCheck.SUPPORTS_EPOLL){
- // b.option(EpollChannelOption.SO_REUSEPORT, true); //
- // }
- }
-
- public R initRemoteConnection(SocketChannel channel){
- local=channel.localAddress();
- remote=channel.remoteAddress();
- return null;
- };
-
- private static final OutboundRpcMessage PING_MESSAGE = new OutboundRpcMessage(RpcMode.PING, 0, 0, Acks.OK);
-
- /**
- * Handler that watches for situations where we haven't read from the socket in a certain timeout. If we exceed this
- * timeout, we send a PING message to the server to state that we are still alive.
- */
- private class IdlePingHandler extends IdleStateHandler {
-
- private GenericFutureListener<Future<? super Void>> pingFailedHandler = new GenericFutureListener<Future<? super Void>>() {
- public void operationComplete(Future<? super Void> future) throws Exception {
- if (!future.isSuccess()) {
- logger.error("Unable to maintain connection {}. Closing connection.", connection.getName());
- connection.close();
- }
- }
- };
-
- public IdlePingHandler(long idleWaitInMillis) {
- super(0, idleWaitInMillis, 0, TimeUnit.MILLISECONDS);
- }
-
- @Override
- protected void channelIdle(ChannelHandlerContext ctx, IdleStateEvent evt) throws Exception {
- if (evt.state() == IdleState.WRITER_IDLE) {
- ctx.writeAndFlush(PING_MESSAGE).addListener(pingFailedHandler);
- }
- }
- }
-
- public abstract ProtobufLengthDecoder getDecoder(BufferAllocator allocator);
-
- public boolean isActive() {
- return connection != null
- && connection.getChannel() != null
- && connection.getChannel().isActive();
- }
-
- protected abstract void validateHandshake(HANDSHAKE_RESPONSE validateHandshake) throws RpcException;
-
- protected abstract void finalizeConnection(HANDSHAKE_RESPONSE handshake, R connection);
-
- public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener,
- T rpcType, SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
- super.send(listener, connection, rpcType, protobufBody, clazz, dataBodies);
- }
-
- public <SEND extends MessageLite, RECEIVE extends MessageLite> DrillRpcFuture<RECEIVE> send(T rpcType,
- SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
- return super.send(connection, rpcType, protobufBody, clazz, dataBodies);
- }
-
- @Override
- public boolean isClient() {
- return true;
- }
-
- protected void connectAsClient(RpcConnectionHandler<R> connectionListener, HANDSHAKE_SEND handshakeValue,
- String host, int port) {
- ConnectionMultiListener cml = new ConnectionMultiListener(connectionListener, handshakeValue);
- b.connect(host, port).addListener(cml.connectionHandler);
- }
-
- private class ConnectionMultiListener {
- private final RpcConnectionHandler<R> l;
- private final HANDSHAKE_SEND handshakeValue;
-
- public ConnectionMultiListener(RpcConnectionHandler<R> l, HANDSHAKE_SEND handshakeValue) {
- assert l != null;
- assert handshakeValue != null;
-
- this.l = l;
- this.handshakeValue = handshakeValue;
- }
-
- public final ConnectionHandler connectionHandler = new ConnectionHandler();
- public final HandshakeSendHandler handshakeSendHandler = new HandshakeSendHandler();
-
- /**
- * Manages connection establishment outcomes.
- */
- private class ConnectionHandler implements GenericFutureListener<ChannelFuture> {
-
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
- boolean isInterrupted = false;
-
- // We want to wait for at least 120 secs when interrupts occur. Establishing a connection fails/succeeds quickly,
- // So there is no point propagating the interruption as failure immediately.
- long remainingWaitTimeMills = 120000;
- long startTime = System.currentTimeMillis();
- // logger.debug("Connection operation finished. Success: {}", future.isSuccess());
- while(true) {
- try {
- future.get(remainingWaitTimeMills, TimeUnit.MILLISECONDS);
- if (future.isSuccess()) {
- SocketAddress remote = future.channel().remoteAddress();
- SocketAddress local = future.channel().localAddress();
- setAddresses(remote, local);
- // send a handshake on the current thread. This is the only time we will send from within the event thread.
- // We can do this because the connection will not be backed up.
- send(handshakeSendHandler, connection, handshakeType, handshakeValue, responseClass, true);
- } else {
- l.connectionFailed(FailureType.CONNECTION, new RpcException("General connection failure."));
- }
- // logger.debug("Handshake queued for send.");
- break;
- } catch (final InterruptedException interruptEx) {
- remainingWaitTimeMills -= (System.currentTimeMillis() - startTime);
- startTime = System.currentTimeMillis();
- isInterrupted = true;
- if (remainingWaitTimeMills < 1) {
- l.connectionFailed(FailureType.CONNECTION, interruptEx);
- break;
- }
- // Ignore the interrupt and continue to wait until we elapse remainingWaitTimeMills.
- } catch (final Exception ex) {
- logger.error("Failed to establish connection", ex);
- l.connectionFailed(FailureType.CONNECTION, ex);
- break;
- }
- }
-
- if (isInterrupted) {
- // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
- // interruption and respond to it if it wants to.
- Thread.currentThread().interrupt();
- }
- }
- }
-
- /**
- * manages handshake outcomes.
- */
- private class HandshakeSendHandler implements RpcOutcomeListener<HANDSHAKE_RESPONSE> {
-
- @Override
- public void failed(RpcException ex) {
- logger.debug("Failure while initiating handshake", ex);
- l.connectionFailed(FailureType.HANDSHAKE_COMMUNICATION, ex);
- }
-
- @Override
- public void success(HANDSHAKE_RESPONSE value, ByteBuf buffer) {
- // logger.debug("Handshake received. {}", value);
- try {
- BasicClient.this.validateHandshake(value);
- BasicClient.this.finalizeConnection(value, connection);
- l.connectionSucceeded(connection);
- // logger.debug("Handshake completed succesfully.");
- } catch (RpcException ex) {
- l.connectionFailed(FailureType.HANDSHAKE_VALIDATION, ex);
- }
- }
-
- @Override
- public void interrupted(final InterruptedException ex) {
- logger.warn("Interrupted while waiting for handshake response", ex);
- l.connectionFailed(FailureType.HANDSHAKE_COMMUNICATION, ex);
- }
- }
-
- }
-
- private class ClientHandshakeHandler extends AbstractHandshakeHandler<HANDSHAKE_RESPONSE> {
-
- public ClientHandshakeHandler() {
- super(BasicClient.this.handshakeType, BasicClient.this.handshakeParser);
- }
-
- @Override
- protected final void consumeHandshake(ChannelHandlerContext ctx, HANDSHAKE_RESPONSE msg) throws Exception {
- // remove the handshake information from the queue so it doesn't sit there forever.
- RpcOutcome<HANDSHAKE_RESPONSE> response = queue.getFuture(handshakeType.getNumber(), coordinationId,
- responseClass);
- response.set(msg, null);
- }
-
- }
-
- public void setAutoRead(boolean enableAutoRead) {
- connection.setAutoRead(enableAutoRead);
- }
-
- public void close() {
- logger.debug("Closing client");
- try {
- connection.getChannel().close().get();
- } catch (final InterruptedException | ExecutionException e) {
- logger.warn("Failure while shutting {}", this.getClass().getName(), e);
-
- // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
- // interruption and respond to it if it wants to.
- Thread.currentThread().interrupt();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
deleted file mode 100644
index c194b5e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.rpc.BasicClientWithConnection.ServerConnection;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-
-import com.google.protobuf.Internal.EnumLite;
-import com.google.protobuf.MessageLite;
-import com.google.protobuf.Parser;
-
-public abstract class BasicClientWithConnection<T extends EnumLite, HANDSHAKE_SEND extends MessageLite, HANDSHAKE_RESPONSE extends MessageLite> extends BasicClient<T, ServerConnection, HANDSHAKE_SEND, HANDSHAKE_RESPONSE>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicClientWithConnection.class);
-
- private BufferAllocator alloc;
- private final String connectionName;
-
- public BasicClientWithConnection(RpcConfig rpcMapping, BufferAllocator alloc, EventLoopGroup eventLoopGroup, T handshakeType,
- Class<HANDSHAKE_RESPONSE> responseClass, Parser<HANDSHAKE_RESPONSE> handshakeParser, String connectionName) {
- super(rpcMapping, alloc.getUnderlyingAllocator(), eventLoopGroup, handshakeType, responseClass, handshakeParser);
- this.alloc = alloc;
- this.connectionName = connectionName;
- }
-
- @Override
- protected Response handle(ServerConnection connection, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException {
- return handleReponse( (ConnectionThrottle) connection, rpcType, pBody, dBody);
- }
-
- protected abstract Response handleReponse(ConnectionThrottle throttle, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException ;
-
- @Override
- public ServerConnection initRemoteConnection(SocketChannel channel) {
- super.initRemoteConnection(channel);
- return new ServerConnection(connectionName, channel, alloc);
- }
-
- public static class ServerConnection extends RemoteConnection{
-
- private final BufferAllocator alloc;
-
- public ServerConnection(String name, SocketChannel channel, BufferAllocator alloc) {
- super(channel, name);
- this.alloc = alloc;
- }
-
- @Override
- public BufferAllocator getAllocator() {
- return alloc;
- }
-
-
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
deleted file mode 100644
index df12fe7..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.timeout.ReadTimeoutHandler;
-import io.netty.util.concurrent.EventExecutor;
-import io.netty.util.concurrent.Future;
-
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
-import java.io.IOException;
-import java.net.BindException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.exec.exception.DrillbitStartupException;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
-
-import com.google.protobuf.Internal.EnumLite;
-import com.google.common.base.Stopwatch;
-import com.google.common.io.Closeables;
-import com.google.protobuf.MessageLite;
-import com.google.protobuf.Parser;
-
-/**
- * A server is bound to a port and is responsible for responding to various type of requests. In some cases, the inbound
- * requests will generate more than one outbound request.
- */
-public abstract class BasicServer<T extends EnumLite, C extends RemoteConnection> extends RpcBus<T, C> {
- final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(this.getClass());
-
- protected static final String TIMEOUT_HANDLER = "timeout-handler";
-
- private ServerBootstrap b;
- private volatile boolean connect = false;
- private final EventLoopGroup eventLoopGroup;
-
- public BasicServer(final RpcConfig rpcMapping, ByteBufAllocator alloc, EventLoopGroup eventLoopGroup) {
- super(rpcMapping);
- this.eventLoopGroup = eventLoopGroup;
-
- b = new ServerBootstrap()
- .channel(TransportCheck.getServerSocketChannel())
- .option(ChannelOption.SO_BACKLOG, 1000)
- .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30*1000)
- .option(ChannelOption.TCP_NODELAY, true)
- .option(ChannelOption.SO_REUSEADDR, true)
- .option(ChannelOption.SO_RCVBUF, 1 << 17)
- .option(ChannelOption.SO_SNDBUF, 1 << 17)
- .group(eventLoopGroup) //
- .childOption(ChannelOption.ALLOCATOR, alloc)
-
- // .handler(new LoggingHandler(LogLevel.INFO))
-
- .childHandler(new ChannelInitializer<SocketChannel>() {
- @Override
- protected void initChannel(SocketChannel ch) throws Exception {
-// logger.debug("Starting initialization of server connection.");
- C connection = initRemoteConnection(ch);
- ch.closeFuture().addListener(getCloseHandler(ch, connection));
-
- final ChannelPipeline pipe = ch.pipeline();
- pipe.addLast("protocol-decoder", getDecoder(connection.getAllocator(), getOutOfMemoryHandler()));
- pipe.addLast("message-decoder", new RpcDecoder("s-" + rpcConfig.getName()));
- pipe.addLast("protocol-encoder", new RpcEncoder("s-" + rpcConfig.getName()));
- pipe.addLast("handshake-handler", getHandshakeHandler(connection));
-
- if (rpcMapping.hasTimeout()) {
- pipe.addLast(TIMEOUT_HANDLER,
- new LogggingReadTimeoutHandler(connection, rpcMapping.getTimeout()));
- }
-
- pipe.addLast("message-handler", new InboundHandler(connection));
- pipe.addLast("exception-handler", new RpcExceptionHandler(connection));
-
- connect = true;
-// logger.debug("Server connection initialization completed.");
- }
- });
-
-// if(TransportCheck.SUPPORTS_EPOLL){
-// b.option(EpollChannelOption.SO_REUSEPORT, true); //
-// }
- }
-
- private class LogggingReadTimeoutHandler<C extends RemoteConnection> extends ReadTimeoutHandler {
-
- private final C connection;
- private final int timeoutSeconds;
- public LogggingReadTimeoutHandler(C connection, int timeoutSeconds) {
- super(timeoutSeconds);
- this.connection = connection;
- this.timeoutSeconds = timeoutSeconds;
- }
-
- @Override
- protected void readTimedOut(ChannelHandlerContext ctx) throws Exception {
- logger.info("RPC connection {} timed out. Timeout was set to {} seconds. Closing connection.", connection.getName(),
- timeoutSeconds);
- super.readTimedOut(ctx);
- }
-
- }
-
- public OutOfMemoryHandler getOutOfMemoryHandler() {
- return OutOfMemoryHandler.DEFAULT_INSTANCE;
- }
-
- protected void removeTimeoutHandler() {
-
- }
-
- public abstract ProtobufLengthDecoder getDecoder(BufferAllocator allocator, OutOfMemoryHandler outOfMemoryHandler);
-
- @Override
- public boolean isClient() {
- return false;
- }
-
- protected abstract ServerHandshakeHandler<?> getHandshakeHandler(C connection);
-
- protected static abstract class ServerHandshakeHandler<T extends MessageLite> extends AbstractHandshakeHandler<T> {
-
- public ServerHandshakeHandler(EnumLite handshakeType, Parser<T> parser) {
- super(handshakeType, parser);
- }
-
- @Override
- protected void consumeHandshake(ChannelHandlerContext ctx, T inbound) throws Exception {
- OutboundRpcMessage msg = new OutboundRpcMessage(RpcMode.RESPONSE, this.handshakeType, coordinationId,
- getHandshakeResponse(inbound));
- ctx.writeAndFlush(msg);
- }
-
- public abstract MessageLite getHandshakeResponse(T inbound) throws Exception;
-
- }
-
- @Override
- protected MessageLite getResponseDefaultInstance(int rpcType) throws RpcException {
- return null;
- }
-
- @Override
- protected Response handle(C connection, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException {
- return null;
- }
-
- @Override
- public <SEND extends MessageLite, RECEIVE extends MessageLite> DrillRpcFuture<RECEIVE> send(C connection, T rpcType,
- SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
- return super.send(connection, rpcType, protobufBody, clazz, dataBodies);
- }
-
- @Override
- public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener,
- C connection, T rpcType, SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
- super.send(listener, connection, rpcType, protobufBody, clazz, dataBodies);
- }
-
- @Override
- public C initRemoteConnection(SocketChannel channel) {
- local = channel.localAddress();
- remote = channel.remoteAddress();
- return null;
- }
-
- public int bind(final int initialPort, boolean allowPortHunting) throws DrillbitStartupException {
- int port = initialPort - 1;
- while (true) {
- try {
- b.bind(++port).sync();
- break;
- } catch (Exception e) {
- // TODO(DRILL-3026): Revisit: Exception is not (always) BindException.
- // One case is "java.io.IOException: bind() failed: Address already in
- // use".
- if (e instanceof BindException && allowPortHunting) {
- continue;
- }
- final UserException bindException =
- UserException
- .resourceError( e )
- .addContext( "Server type", getClass().getSimpleName() )
- .message( "Drillbit could not bind to port %s.", port )
- .build(logger);
- throw bindException;
- }
- }
-
- connect = !connect;
- logger.debug("Server of type {} started on port {}.", getClass().getSimpleName(), port);
- return port;
- }
-
- @Override
- public void close() throws IOException {
- try {
- Stopwatch watch = new Stopwatch().start();
- // this takes 1s to complete
- // known issue: https://github.com/netty/netty/issues/2545
- eventLoopGroup.shutdownGracefully(0, 0, TimeUnit.SECONDS).get();
- long elapsed = watch.elapsed(MILLISECONDS);
- if (elapsed > 500) {
- logger.info("closed eventLoopGroup " + eventLoopGroup + " in " + elapsed + " ms");
- }
- } catch (final InterruptedException | ExecutionException e) {
- logger.warn("Failure while shutting down {}. ", this.getClass().getName(), e);
-
- // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
- // interruption and respond to it if it wants to.
- Thread.currentThread().interrupt();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ChannelClosedException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ChannelClosedException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ChannelClosedException.java
deleted file mode 100644
index b68efae..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ChannelClosedException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-public class ChannelClosedException extends RpcException{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ChannelClosedException.class);
-
- public ChannelClosedException() {
- super();
- }
-
- public ChannelClosedException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ChannelClosedException(String message) {
- super(message);
- }
-
- public ChannelClosedException(Throwable cause) {
- super(cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ChannelListenerWithCoordinationId.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ChannelListenerWithCoordinationId.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ChannelListenerWithCoordinationId.java
deleted file mode 100644
index 3b95dd8..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ChannelListenerWithCoordinationId.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.channel.ChannelFuture;
-import io.netty.util.concurrent.GenericFutureListener;
-
-public interface ChannelListenerWithCoordinationId extends GenericFutureListener<ChannelFuture>{
- public int getCoordinationId();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java
deleted file mode 100644
index 5a5bbab..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelFuture;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.drill.common.exceptions.UserRemoteException;
-import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
-
-/**
- * Manages the creation of rpc futures for a particular socket.
- */
-public class CoordinationQueue {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoordinationQueue.class);
-
- private final PositiveAtomicInteger circularInt = new PositiveAtomicInteger();
- private final Map<Integer, RpcOutcome<?>> map;
-
- public CoordinationQueue(int segmentSize, int segmentCount) {
- map = new ConcurrentHashMap<Integer, RpcOutcome<?>>(segmentSize, 0.75f, segmentCount);
- }
-
- void channelClosed(Throwable ex) {
- if (ex != null) {
- RpcException e;
- if (ex instanceof RpcException) {
- e = (RpcException) ex;
- } else {
- e = new RpcException(ex);
- }
- for (RpcOutcome<?> f : map.values()) {
- f.setException(e);
- }
- }
- }
-
- public <V> ChannelListenerWithCoordinationId get(RpcOutcomeListener<V> handler, Class<V> clazz, RemoteConnection connection) {
- int i = circularInt.getNext();
- RpcListener<V> future = new RpcListener<V>(handler, clazz, i, connection);
- Object old = map.put(i, future);
- if (old != null) {
- throw new IllegalStateException(
- "You attempted to reuse a coordination id when the previous coordination id has not been removed. This is likely rpc future callback memory leak.");
- }
- return future;
- }
-
- private class RpcListener<T> implements ChannelListenerWithCoordinationId, RpcOutcome<T>{
- final RpcOutcomeListener<T> handler;
- final Class<T> clazz;
- final int coordinationId;
- final RemoteConnection connection;
-
- public RpcListener(RpcOutcomeListener<T> handler, Class<T> clazz, int coordinationId, RemoteConnection connection) {
- super();
- this.handler = handler;
- this.clazz = clazz;
- this.coordinationId = coordinationId;
- this.connection = connection;
- }
-
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
-
- if (!future.isSuccess()) {
- removeFromMap(coordinationId);
- if (future.channel().isActive()) {
- throw new RpcException("Future failed") ;
- } else {
- setException(new ChannelClosedException());
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void set(Object value, ByteBuf buffer) {
- assert clazz.isAssignableFrom(value.getClass());
- handler.success( (T) value, buffer);
- }
-
- @Override
- public void setException(Throwable t) {
- handler.failed(RpcException.mapException(t));
- }
-
- @Override
- public Class<T> getOutcomeType() {
- return clazz;
- }
-
- @Override
- public int getCoordinationId() {
- return coordinationId;
- }
-
- }
-
- private RpcOutcome<?> removeFromMap(int coordinationId) {
- RpcOutcome<?> rpc = map.remove(coordinationId);
- if (rpc == null) {
- throw new IllegalStateException(
- "Attempting to retrieve an rpc that wasn't first stored in the rpc coordination queue. This would most likely happen if you're opposite endpoint sent multiple messages on the same coordination id.");
- }
- return rpc;
- }
-
- public <V> RpcOutcome<V> getFuture(int rpcType, int coordinationId, Class<V> clazz) {
-
- RpcOutcome<?> rpc = removeFromMap(coordinationId);
- // logger.debug("Got rpc from map {}", rpc);
- Class<?> outcomeClass = rpc.getOutcomeType();
-
- if (outcomeClass != clazz) {
- throw new IllegalStateException(
- String
- .format(
- "RPC Engine had a submission and response configuration mismatch. The RPC request that you submitted was defined with an expected response type of %s. However, "
- + "when the response returned, a call to getResponseDefaultInstance() with Rpc number %d provided an expected class of %s. This means either your submission uses the wrong type definition"
- + "or your getResponseDefaultInstance() method responds the wrong instance type ",
- clazz.getCanonicalName(), rpcType, outcomeClass.getCanonicalName()));
- }
-
- @SuppressWarnings("unchecked")
- RpcOutcome<V> crpc = (RpcOutcome<V>) rpc;
-
- // logger.debug("Returning casted future");
- return crpc;
- }
-
- public void updateFailedFuture(int coordinationId, DrillPBError failure) {
- // logger.debug("Updating failed future.");
- try {
- RpcOutcome<?> rpc = removeFromMap(coordinationId);
- rpc.setException(new UserRemoteException(failure));
- } catch(Exception ex) {
- logger.warn("Failed to remove from map. Not a problem since we were updating on failed future.", ex);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/DrillRpcFuture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/DrillRpcFuture.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/DrillRpcFuture.java
deleted file mode 100644
index d044432..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/DrillRpcFuture.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-import com.google.common.util.concurrent.CheckedFuture;
-
-public interface DrillRpcFuture<T> extends CheckedFuture<T,RpcException> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRpcFuture.class);
-
- public ByteBuf getBuffer();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/DrillRpcFutureImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/DrillRpcFutureImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/DrillRpcFutureImpl.java
deleted file mode 100644
index cbe63c6..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/DrillRpcFutureImpl.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-import com.google.common.util.concurrent.AbstractCheckedFuture;
-import com.google.common.util.concurrent.AbstractFuture;
-
-class DrillRpcFutureImpl<V> extends AbstractCheckedFuture<V, RpcException> implements DrillRpcFuture<V>, RpcOutcomeListener<V>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRpcFutureImpl.class);
-
- private volatile ByteBuf buffer;
-
- public DrillRpcFutureImpl() {
- super(new InnerFuture<V>());
- }
-
- @Override
- protected RpcException mapException(Exception ex) {
- return RpcException.mapException(ex);
- }
-
- private static class InnerFuture<T> extends AbstractFuture<T> {
- // we rewrite these so that the parent can see them
-
- void setValue(T value) {
- super.set(value);
- }
-
- @Override
- protected boolean setException(Throwable t) {
- return super.setException(t);
- }
- }
-
- @Override
- public void failed(RpcException ex) {
- ( (InnerFuture<V>)delegate()).setException(ex);
- }
-
- @Override
- public void success(V value, ByteBuf buffer) {
- this.buffer = buffer;
- ( (InnerFuture<V>)delegate()).setValue(value);
- }
-
- @Override
- public void interrupted(final InterruptedException ex) {
- // Propagate the interrupt to inner future
- ( (InnerFuture<V>)delegate()).cancel(true);
- }
-
- @Override
- public ByteBuf getBuffer() {
- return buffer;
- }
-
- public void release() {
- if (buffer != null) {
- buffer.release();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/FutureBitCommand.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/FutureBitCommand.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/FutureBitCommand.java
deleted file mode 100644
index 53d0772..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/FutureBitCommand.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.protobuf.MessageLite;
-
-public abstract class FutureBitCommand<T extends MessageLite, C extends RemoteConnection> implements RpcCommand<T,C> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FutureBitCommand.class);
-
- protected final SettableFuture<T> settableFuture;
- private final RpcCheckedFuture<T> parentFuture;
-
- public FutureBitCommand() {
- this.settableFuture = SettableFuture.create();
- this.parentFuture = new RpcCheckedFuture<T>(settableFuture);
- }
-
- public abstract void doRpcCall(RpcOutcomeListener<T> outcomeListener, C connection);
-
- @Override
- public void connectionAvailable(C connection) {
-
- doRpcCall(new DeferredRpcOutcome(), connection);
- }
-
- @Override
- public void connectionSucceeded(C connection) {
- connectionAvailable(connection);
- }
-
- private class DeferredRpcOutcome implements RpcOutcomeListener<T> {
-
- @Override
- public void failed(RpcException ex) {
- settableFuture.setException(ex);
- }
-
- @Override
- public void success(T value, ByteBuf buf) {
- settableFuture.set(value);
- }
-
- @Override
- public void interrupted(final InterruptedException e) {
- // If we are interrupted while performing the command, consider as failure.
- logger.warn("Interrupted while running the command", e);
- failed(new RpcException(e));
- }
- }
-
- public DrillRpcFuture<T> getFuture() {
- return parentFuture;
- }
-
- @Override
- public void connectionFailed(FailureType type, Throwable t) {
- settableFuture.setException(RpcException.mapException(
- String.format("Command failed while establishing connection. Failure type %s.", type), t));
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/InboundRpcMessage.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/InboundRpcMessage.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/InboundRpcMessage.java
deleted file mode 100644
index d739034..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/InboundRpcMessage.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
-
-import java.io.InputStream;
-
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
-
-public class InboundRpcMessage extends RpcMessage{
- public ByteBuf pBody;
- public ByteBuf dBody;
-
- public InboundRpcMessage(RpcMode mode, int rpcType, int coordinationId, ByteBuf pBody, ByteBuf dBody) {
- super(mode, rpcType, coordinationId);
- this.pBody = pBody;
- this.dBody = dBody;
- }
-
- @Override
- public int getBodySize() {
- int len = pBody.capacity();
- if (dBody != null) {
- len += dBody.capacity();
- }
- return len;
- }
-
- @Override
- void release() {
- if (pBody != null) {
- pBody.release();
- }
- if (dBody != null) {
- dBody.release();
- }
- }
-
- @Override
- public String toString() {
- return "InboundRpcMessage [pBody=" + pBody + ", mode=" + mode + ", rpcType=" + rpcType + ", coordinationId="
- + coordinationId + ", dBody=" + dBody + "]";
- }
-
- public InputStream getProtobufBodyAsIS() {
- return new ByteBufInputStream(pBody);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ListeningCommand.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ListeningCommand.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ListeningCommand.java
deleted file mode 100644
index 92022b0..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ListeningCommand.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-import com.google.protobuf.MessageLite;
-
-public abstract class ListeningCommand<T extends MessageLite, C extends RemoteConnection> implements RpcCommand<T, C> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ListeningCommand.class);
-
- private final RpcOutcomeListener<T> listener;
-
- public ListeningCommand(RpcOutcomeListener<T> listener) {
- this.listener = listener;
- }
-
- public abstract void doRpcCall(RpcOutcomeListener<T> outcomeListener, C connection);
-
- @Override
- public void connectionAvailable(C connection) {
-
- doRpcCall(new DeferredRpcOutcome(), connection);
- }
-
- @Override
- public void connectionSucceeded(C connection) {
- connectionAvailable(connection);
- }
-
- private class DeferredRpcOutcome implements RpcOutcomeListener<T> {
-
- @Override
- public void failed(RpcException ex) {
- listener.failed(ex);
- }
-
- @Override
- public void success(T value, ByteBuf buf) {
- listener.success(value, buf);
- }
-
- @Override
- public void interrupted(final InterruptedException e) {
- listener.interrupted(e);
- }
- }
-
- @Override
- public void connectionFailed(FailureType type, Throwable t) {
- listener.failed(RpcException.mapException(
- String.format("Command failed while establishing connection. Failure type %s.", type), t));
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/NamedThreadFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/NamedThreadFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/NamedThreadFactory.java
deleted file mode 100644
index e7580b2..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/NamedThreadFactory.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * {@link ThreadFactory} for {@link ExecutorServices} that names threads sequentially.
- * Creates Threads named with the prefix specified at construction time. Created threads
- * have the daemon bit set and priority Thread.MAX_PRIORITY.
- *
- * <p>An instance creates names with an instance-specific prefix suffixed with sequential
- * integers.</p>
- *
- * <p>Concurrency: See {@link newThread}.</p>
- */
-public class NamedThreadFactory implements ThreadFactory {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NamedThreadFactory.class);
- private final AtomicInteger nextId = new AtomicInteger(); // used to generate unique ids
- private final String prefix;
-
- /**
- * Constructor.
- *
- * @param prefix the string prefix that will be used to name threads created by this factory
- */
- public NamedThreadFactory(final String prefix) {
- this.prefix = prefix;
- }
-
- /**
- * Creates a sequentially named thread running a given Runnable.
- * <p>
- * The thread's name will be this instance's prefix concatenated with
- * this instance's next<sup><a href="#fn-1">*</a></sup> sequential integer.
- * </p>
- * <p>
- * Concurrency: Thread-safe.
- * </p>
- * <p>
- * (Concurrent calls get different numbers.
- * Calls started after other calls complete get later/higher numbers than
- * those other calls.
- * </p>
- * <p>
- * <a name="fn-1" />*However, for concurrent calls, the order of numbers
- * is not defined.)
- */
- @Override
- public Thread newThread(final Runnable runnable) {
- final Thread thread = new Thread(runnable, prefix + nextId.incrementAndGet());
- thread.setDaemon(true);
-
- try {
- if (thread.getPriority() != Thread.MAX_PRIORITY) {
- thread.setPriority(Thread.MAX_PRIORITY);
- }
- } catch (Exception ignored) {
- // Doesn't matter even if failed to set.
- logger.info("ignored exception " + ignored);
- }
- return thread;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/OutOfMemoryHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/OutOfMemoryHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/OutOfMemoryHandler.java
deleted file mode 100644
index 5d7db47..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/OutOfMemoryHandler.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-public interface OutOfMemoryHandler {
-
- public static OutOfMemoryHandler DEFAULT_INSTANCE = new OutOfMemoryHandler() {
- @Override
- public void handle() {
- throw new UnsupportedOperationException();
- }
- };
-
- public void handle();
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/OutboundRpcMessage.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/OutboundRpcMessage.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/OutboundRpcMessage.java
deleted file mode 100644
index 5eda350..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/OutboundRpcMessage.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.Internal.EnumLite;
-import com.google.protobuf.MessageLite;
-
-public class OutboundRpcMessage extends RpcMessage {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutboundRpcMessage.class);
-
- final MessageLite pBody;
- public ByteBuf[] dBodies;
-
-
-
- public OutboundRpcMessage(RpcMode mode, EnumLite rpcType, int coordinationId, MessageLite pBody, ByteBuf... dBodies) {
- this(mode, rpcType.getNumber(), coordinationId, pBody, dBodies);
- }
-
-
- OutboundRpcMessage(RpcMode mode, int rpcTypeNumber, int coordinationId, MessageLite pBody, ByteBuf... dBodies) {
- super(mode, rpcTypeNumber, coordinationId);
- this.pBody = pBody;
-
- // Netty doesn't traditionally release the reference on an unreadable buffer. However, we need to so that if we send a empty or unwritable buffer, we still release. otherwise we get weird memory leaks when sending empty vectors.
- List<ByteBuf> bufs = Lists.newArrayList();
- for (ByteBuf d : dBodies) {
- if (d.readableBytes() == 0) {
- d.release();
- } else {
- bufs.add(d);
- }
- }
- this.dBodies = bufs.toArray(new ByteBuf[bufs.size()]);
- }
-
- @Override
- public int getBodySize() {
- int len = pBody.getSerializedSize();
- len += RpcEncoder.getRawVarintSize(len);
- len += getRawBodySize();
- return len;
- }
-
- public int getRawBodySize() {
- if (dBodies == null) {
- return 0;
- }
- int len = 0;
-
- for (int i = 0; i < dBodies.length; i++) {
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Reader Index {}, Writer Index {}", dBodies[i].readerIndex(), dBodies[i].writerIndex());
- }
- len += dBodies[i].readableBytes();
- }
- return len;
- }
-
- @Override
- public String toString() {
- return "OutboundRpcMessage [pBody=" + pBody + ", mode=" + mode + ", rpcType=" + rpcType + ", coordinationId="
- + coordinationId + ", dBodies=" + Arrays.toString(dBodies) + "]";
- }
-
- @Override
- void release() {
- if (dBodies != null) {
- for (ByteBuf b : dBodies) {
- b.release();
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/PositiveAtomicInteger.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/PositiveAtomicInteger.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/PositiveAtomicInteger.java
deleted file mode 100644
index 401663d..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/PositiveAtomicInteger.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-/*
- * An atomic integer that only ever returns 0 > MAX_INT and then starts over. Should never has a negative overflow.
- */
-public class PositiveAtomicInteger {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PositiveAtomicInteger.class);
-
- private final AtomicInteger internal = new AtomicInteger(0);
-
- public int getNext(){
- int i = internal.addAndGet(1);
- if(i < 0){
- return i + (-Integer.MIN_VALUE);
- }else{
- return i;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
deleted file mode 100644
index 3dfe03f..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.ByteToMessageDecoder;
-import io.netty.handler.codec.CorruptedFrameException;
-
-import java.util.List;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-
-import com.google.protobuf.CodedInputStream;
-import org.apache.drill.exec.exception.OutOfMemoryException;
-
-/**
- * Modified version of {@link io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder} that avoids bytebuf copy.
- * See the documentation there.
- */
-public class ProtobufLengthDecoder extends ByteToMessageDecoder {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProtobufLengthDecoder.class);
-
- private BufferAllocator allocator;
- private OutOfMemoryHandler outOfMemoryHandler;
-
- public ProtobufLengthDecoder(BufferAllocator allocator, OutOfMemoryHandler outOfMemoryHandler) {
- super();
- this.allocator = allocator;
- this.outOfMemoryHandler = outOfMemoryHandler;
- }
-
-
- @Override
- protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
- if (!ctx.channel().isOpen()) {
- if (in.readableBytes() > 0) {
- logger.info("Channel is closed, discarding remaining {} byte(s) in buffer.", in.readableBytes());
- }
- in.skipBytes(in.readableBytes());
- return;
- }
-
- in.markReaderIndex();
- final byte[] buf = new byte[5];
- for (int i = 0; i < buf.length; i++) {
- if (!in.isReadable()) {
- in.resetReaderIndex();
- return;
- }
-
- buf[i] = in.readByte();
- if (buf[i] >= 0) {
-
- int length = CodedInputStream.newInstance(buf, 0, i + 1).readRawVarint32();
-
- if (length < 0) {
- throw new CorruptedFrameException("negative length: " + length);
- }
- if (length == 0) {
- throw new CorruptedFrameException("Received a message of length 0.");
- }
-
- if (in.readableBytes() < length) {
- in.resetReaderIndex();
- return;
- } else {
- // need to make buffer copy, otherwise netty will try to refill this buffer if we move the readerIndex forward...
- // TODO: Can we avoid this copy?
- ByteBuf outBuf;
- try {
- outBuf = allocator.buffer(length);
- } catch (OutOfMemoryException e) {
- logger.warn("Failure allocating buffer on incoming stream due to memory limits. Current Allocation: {}.", allocator.getAllocatedMemory());
- in.resetReaderIndex();
- outOfMemoryHandler.handle();
- return;
- }
- outBuf.writeBytes(in, in.readerIndex(), length);
-
- in.skipBytes(length);
-
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug(String.format(
- "ReaderIndex is %d after length header of %d bytes and frame body of length %d bytes.",
- in.readerIndex(), i + 1, length));
- }
-
- out.add(outBuf);
- return;
- }
- }
- }
-
- // Couldn't find the byte whose MSB is off.
- throw new CorruptedFrameException("length wider than 32-bit");
-
- }
-
- @Override
- public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
- ctx.fireChannelReadComplete();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java
deleted file mode 100644
index d62b6f2..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.channel.ChannelFuture;
-import io.netty.util.concurrent.GenericFutureListener;
-
-import java.io.Closeable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicReference;
-
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.AbstractFuture;
-import com.google.protobuf.MessageLite;
-
-/**
- * Manager all connections between two particular bits.
- */
-public abstract class ReconnectingConnection<CONNECTION_TYPE extends RemoteConnection, OUTBOUND_HANDSHAKE extends MessageLite>
- implements Closeable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReconnectingConnection.class);
-
- private final AtomicReference<CONNECTION_TYPE> connectionHolder = new AtomicReference<CONNECTION_TYPE>();
- private final String host;
- private final int port;
- private final OUTBOUND_HANDSHAKE handshake;
-
- public ReconnectingConnection(OUTBOUND_HANDSHAKE handshake, String host, int port) {
- Preconditions.checkNotNull(host);
- Preconditions.checkArgument(port > 0);
- this.host = host;
- this.port = port;
- this.handshake = handshake;
- }
-
- protected abstract BasicClient<?, CONNECTION_TYPE, OUTBOUND_HANDSHAKE, ?> getNewClient();
-
- public <R extends MessageLite, C extends RpcCommand<R, CONNECTION_TYPE>> void runCommand(C cmd) {
-// if(logger.isDebugEnabled()) logger.debug(String.format("Running command %s sending to host %s:%d", cmd, host, port));
- CONNECTION_TYPE connection = connectionHolder.get();
- if (connection != null) {
- if (connection.isActive()) {
- cmd.connectionAvailable(connection);
-// logger.debug("Connection available and active, command run inline.");
- return;
- } else {
- // remove the old connection. (don't worry if we fail since someone else should have done it.
- connectionHolder.compareAndSet(connection, null);
- }
- }
-
- /**
- * We've arrived here without a connection, let's make sure only one of us makes a connection. (fyi, another
- * endpoint could create a reverse connection
- **/
- synchronized (this) {
- connection = connectionHolder.get();
- if (connection != null) {
- cmd.connectionAvailable(connection);
-
- } else {
-// logger.debug("No connection active, opening client connection.");
- BasicClient<?, CONNECTION_TYPE, OUTBOUND_HANDSHAKE, ?> client = getNewClient();
- ConnectionListeningFuture<R, C> future = new ConnectionListeningFuture<R, C>(cmd);
- client.connectAsClient(future, handshake, host, port);
- future.waitAndRun();
-// logger.debug("Connection available and active, command now being run inline.");
- }
- return;
-
- }
- }
-
- public class ConnectionListeningFuture<R extends MessageLite, C extends RpcCommand<R, CONNECTION_TYPE>> extends
- AbstractFuture<CONNECTION_TYPE> implements RpcConnectionHandler<CONNECTION_TYPE> {
-
- private C cmd;
-
- public ConnectionListeningFuture(C cmd) {
- super();
- this.cmd = cmd;
- }
-
- /**
- * Called by
- */
- public void waitAndRun() {
- boolean isInterrupted = false;
-
- // We want to wait for at least 120 secs when interrupts occur. Establishing a connection fails/succeeds quickly,
- // So there is no point propagating the interruption as failure immediately.
- long remainingWaitTimeMills = 120000;
- long startTime = System.currentTimeMillis();
-
- while(true) {
- try {
- // logger.debug("Waiting for connection.");
- CONNECTION_TYPE connection = this.get(remainingWaitTimeMills, TimeUnit.MILLISECONDS);
-
- if (connection == null) {
- // logger.debug("Connection failed.");
- } else {
- // logger.debug("Connection received. {}", connection);
- cmd.connectionSucceeded(connection);
- // logger.debug("Finished connection succeeded activity.");
- }
- break;
- } catch (final InterruptedException interruptEx) {
- remainingWaitTimeMills -= (System.currentTimeMillis() - startTime);
- startTime = System.currentTimeMillis();
- isInterrupted = true;
- if (remainingWaitTimeMills < 1) {
- cmd.connectionFailed(FailureType.CONNECTION, interruptEx);
- break;
- }
- // Ignore the interrupt and continue to wait until we elapse remainingWaitTimeMills.
- } catch (final ExecutionException | TimeoutException ex) {
- logger.error("Failed to establish connection", ex);
- cmd.connectionFailed(FailureType.CONNECTION, ex);
- break;
- }
- }
-
- if (isInterrupted) {
- // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
- // interruption and respond to it if it wants to.
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void connectionFailed(org.apache.drill.exec.rpc.RpcConnectionHandler.FailureType type, Throwable t) {
- set(null);
- cmd.connectionFailed(type, t);
- }
-
- @Override
- public void connectionSucceeded(CONNECTION_TYPE incoming) {
- CONNECTION_TYPE connection = connectionHolder.get();
- while (true) {
- boolean setted = connectionHolder.compareAndSet(null, incoming);
- if (setted) {
- connection = incoming;
- break;
- }
- connection = connectionHolder.get();
- if (connection != null) {
- break;
- }
- }
-
- if (connection != incoming) {
- // close the incoming because another channel was created in the mean time (unless this is a self connection).
- logger.debug("Closing incoming connection because a connection was already set.");
- incoming.getChannel().close();
- }
- set(connection);
-
- }
-
- }
-
- /** Factory for close handlers **/
- public class CloseHandlerCreator {
- public GenericFutureListener<ChannelFuture> getHandler(CONNECTION_TYPE connection,
- GenericFutureListener<ChannelFuture> parent) {
- return new CloseHandler(connection, parent);
- }
- }
-
- /**
- * Listens for connection closes and clears connection holder.
- */
- protected class CloseHandler implements GenericFutureListener<ChannelFuture> {
- private CONNECTION_TYPE connection;
- private GenericFutureListener<ChannelFuture> parent;
-
- public CloseHandler(CONNECTION_TYPE connection, GenericFutureListener<ChannelFuture> parent) {
- super();
- this.connection = connection;
- this.parent = parent;
- }
-
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
- connectionHolder.compareAndSet(connection, null);
- parent.operationComplete(future);
- }
-
- }
-
- public CloseHandlerCreator getCloseHandlerCreator() {
- return new CloseHandlerCreator();
- }
-
- public void addExternalConnection(CONNECTION_TYPE connection) {
- // if the connection holder is not set, set it to this incoming connection. We'll simply ignore if already set.
- this.connectionHolder.compareAndSet(null, connection);
- }
-
- @Override
- public void close() {
- CONNECTION_TYPE c = connectionHolder.getAndSet(null);
- if (c != null) {
- c.getChannel().close();
- }
- }
-
- /**
- * Decorate a connection creation so that we capture a success and keep it available for future requests. If we have
- * raced and another is already available... we return that one and close things down on this one.
- */
- private class ConnectionListeningDecorator implements RpcConnectionHandler<CONNECTION_TYPE> {
-
- private final RpcConnectionHandler<CONNECTION_TYPE> delegate;
-
- public ConnectionListeningDecorator(RpcConnectionHandler<CONNECTION_TYPE> delegate) {
- this.delegate = delegate;
- }
-
- @Override
- public void connectionSucceeded(CONNECTION_TYPE incoming) {
- CONNECTION_TYPE connection = connectionHolder.get();
- while (true) {
- boolean setted = connectionHolder.compareAndSet(null, incoming);
- if (setted) {
- connection = incoming;
- break;
- }
- connection = connectionHolder.get();
- if (connection != null) {
- break;
- }
- }
-
- if (connection == incoming) {
- delegate.connectionSucceeded(connection);
- } else {
- // close the incoming because another channel was created in the mean time (unless this is a self connection).
- logger.debug("Closing incoming connection because a connection was already set.");
- incoming.getChannel().close();
- delegate.connectionSucceeded(connection);
- }
- }
-
- @Override
- public void connectionFailed(org.apache.drill.exec.rpc.RpcConnectionHandler.FailureType type, Throwable t) {
- delegate.connectionFailed(type, t);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
deleted file mode 100644
index 30abcc4..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.util.concurrent.Future;
-import io.netty.util.concurrent.GenericFutureListener;
-
-import java.util.concurrent.ExecutionException;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-
-public abstract class RemoteConnection implements ConnectionThrottle, AutoCloseable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemoteConnection.class);
- private final Channel channel;
- private final WriteManager writeManager;
- private String name;
- private final String clientName;
-
- public boolean inEventLoop(){
- return channel.eventLoop().inEventLoop();
- }
-
- public RemoteConnection(SocketChannel channel, String name) {
- super();
- this.channel = channel;
- this.clientName = name;
- this.writeManager = new WriteManager();
- channel.pipeline().addLast(new BackPressureHandler());
- channel.closeFuture().addListener(new GenericFutureListener<Future<? super Void>>() {
- public void operationComplete(Future<? super Void> future) throws Exception {
- // this could possibly overrelease but it doesn't matter since we're only going to do this to ensure that we
- // fail out any pending messages
- writeManager.disable();
- writeManager.setWritable(true);
- }
- });
-
- }
-
- public String getName() {
- if(name == null){
- name = String.format("%s <--> %s (%s)", channel.localAddress(), channel.remoteAddress(), clientName);
- }
- return name;
- }
-
- public abstract BufferAllocator getAllocator();
-
- public final Channel getChannel() {
- return channel;
- }
-
- public boolean blockOnNotWritable(RpcOutcomeListener<?> listener){
- try{
- writeManager.waitForWritable();
- return true;
- }catch(final InterruptedException e){
- listener.interrupted(e);
-
- // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
- // interruption and respond to it if it wants to.
- Thread.currentThread().interrupt();
-
- return false;
- }
- }
-
- public void setAutoRead(boolean enableAutoRead){
- channel.config().setAutoRead(enableAutoRead);
- }
-
- public boolean isActive(){
- return channel.isActive();
- }
-
- /**
- * The write manager is responsible for controlling whether or not a write can be sent. It controls whether or not to block a sender if we have tcp backpressure on the receive side.
- */
- private static class WriteManager{
- private final ResettableBarrier barrier = new ResettableBarrier();
- private volatile boolean disabled = false;
-
- public WriteManager(){
- barrier.openBarrier();
- }
-
- public void waitForWritable() throws InterruptedException{
- barrier.await();
- }
-
- public void setWritable(boolean isWritable){
- if(isWritable){
- barrier.openBarrier();
- } else if (!disabled) {
- barrier.closeBarrier();
- }
-
- }
-
- public void disable() {
- disabled = true;
- }
- }
-
- private class BackPressureHandler extends ChannelInboundHandlerAdapter{
-
- @Override
- public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
-// logger.debug("Channel writability changed.", ctx.channel().isWritable());
- writeManager.setWritable(ctx.channel().isWritable());
- ctx.fireChannelWritabilityChanged();
- }
-
- }
-
- @Override
- public void close() {
- try {
- if (channel.isActive()) {
- channel.close().get();
- }
- } catch (final InterruptedException | ExecutionException e) {
- logger.warn("Caught exception while closing channel.", e);
-
- // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
- // interruption and respond to it if it wants to.
- Thread.currentThread().interrupt();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java
deleted file mode 100644
index a2a6d2a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.AbstractQueuedSynchronizer;
-
-/**
- * Modified implementation of countdown latch that allows a barrier to be unilaterally opened and closed. All others simply wait when it is closed. Is initialized in a closed state.
- */
-public class ResettableBarrier {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ResettableBarrier.class);
-
- private final InternalSynchronizer sync = new InternalSynchronizer();
-
- public ResettableBarrier() {
- }
-
- private static final class InternalSynchronizer extends AbstractQueuedSynchronizer {
-
- private InternalSynchronizer() {
- setState(1);
- }
-
- @Override
- protected int tryAcquireShared(int acquires) {
- assert acquires == 1;
- return (getState() == 0) ? 1 : -1;
- }
-
- @Override
- protected boolean tryReleaseShared(int releases) {
- assert releases == 1;
-
- while(true) {
- int c = getState();
- if (c == 0) {
- return false;
- }
- int nextc = c - 1;
- if (compareAndSetState(c, nextc)) {
- return nextc == 0;
- }
- }
- }
-
- protected void reset() {
- setState(1);
- }
-
- }
-
- public void await() throws InterruptedException {
-// logger.debug("awaiting barrier interruptibly.");
- sync.acquireSharedInterruptibly(1);
- }
-
- public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
-// logger.debug("awaiting barrier with timeout {}.", timeout);
- return sync.tryAcquireSharedNanos(1, unit.toNanos(timeout));
- }
-
- public void openBarrier() {
-// logger.debug("opening barrier.");
- sync.releaseShared(1);
- }
-
- public void closeBarrier() {
-// logger.debug("closing barrier.");
- sync.reset();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Response.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Response.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Response.java
deleted file mode 100644
index b48adec..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Response.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-import com.google.protobuf.Internal.EnumLite;
-import com.google.protobuf.MessageLite;
-
-public class Response {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Response.class);
-
- public EnumLite rpcType;
- public MessageLite pBody;
- public ByteBuf[] dBodies;
-
- public Response(EnumLite rpcType, MessageLite pBody, ByteBuf... dBodies) {
- super();
- this.rpcType = rpcType;
- this.pBody = pBody;
- this.dBodies = dBodies;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
deleted file mode 100644
index 6dc9ae1..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-
-public interface ResponseSender {
- public void send(Response r);
-
-}
[21/45] drill git commit: DRILL-3987: (REFACTOR) Extract
BoundsChecking check from AssertionUtil. Remove unused file.
Posted by ja...@apache.org.
DRILL-3987: (REFACTOR) Extract BoundsChecking check from AssertionUtil. Remove unused file.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/62353a6f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/62353a6f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/62353a6f
Branch: refs/heads/master
Commit: 62353a6fd301e4ff5ddce522b28b47a76c284af4
Parents: 1fde9bb
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Nov 7 19:44:58 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 17:00:47 2015 -0800
----------------------------------------------------------------------
.../com/google/protobuf/WireFormatProxy.java | 27 --------------------
.../apache/drill/exec/util/AssertionUtil.java | 6 ++---
.../expr/fn/impl/StringFunctionHelpers.java | 4 +--
.../apache/drill/exec/expr/fn/impl/XXHash.java | 6 ++---
.../store/easy/text/compliant/TextInput.java | 6 ++---
.../src/main/java/io/netty/buffer/DrillBuf.java | 4 +--
.../drill/exec/memory/BoundsChecking.java | 17 ++++++++++++
.../exec/expr/fn/impl/ByteFunctionHelpers.java | 4 +--
8 files changed, 30 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/62353a6f/common/src/main/java/com/google/protobuf/WireFormatProxy.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/google/protobuf/WireFormatProxy.java b/common/src/main/java/com/google/protobuf/WireFormatProxy.java
deleted file mode 100644
index 181b975..0000000
--- a/common/src/main/java/com/google/protobuf/WireFormatProxy.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.protobuf;
-
-public class WireFormatProxy {
-
- WireFormatProxy(){};
-
- public static int getFieldType(int tag){
- return WireFormat.getTagWireType(tag);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/62353a6f/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java b/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
index 2348e62..770ed07 100644
--- a/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
+++ b/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
@@ -20,20 +20,18 @@ package org.apache.drill.exec.util;
public class AssertionUtil {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
-
public static final boolean ASSERT_ENABLED;
- public static final boolean BOUNDS_CHECKING_ENABLED;
static{
boolean isAssertEnabled = false;
assert isAssertEnabled = true;
ASSERT_ENABLED = isAssertEnabled;
- BOUNDS_CHECKING_ENABLED = ASSERT_ENABLED || !"true".equals(System.getProperty("drill.enable_unsafe_memory_access"));
}
public static boolean isAssertionsEnabled(){
return ASSERT_ENABLED;
}
-
+ private AssertionUtil() {
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/62353a6f/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
index 2f5cf26..c919f28 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctionHelpers.java
@@ -22,7 +22,7 @@ import io.netty.buffer.DrillBuf;
import io.netty.util.internal.PlatformDependent;
import org.apache.drill.exec.expr.holders.VarCharHolder;
-import org.apache.drill.exec.util.AssertionUtil;
+import org.apache.drill.exec.memory.BoundsChecking;
import org.joda.time.chrono.ISOChronology;
import com.google.common.base.Charsets;
@@ -30,7 +30,7 @@ import com.google.common.base.Charsets;
public class StringFunctionHelpers {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StringFunctionHelpers.class);
- private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
+ private static final boolean BOUNDS_CHECKING_ENABLED = BoundsChecking.BOUNDS_CHECKING_ENABLED;
static final int RADIX = 10;
static final long MAX_LONG = -Long.MAX_VALUE / RADIX;
http://git-wip-us.apache.org/repos/asf/drill/blob/62353a6f/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java
index 81dbf5f..b18daed 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java
@@ -17,17 +17,17 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.exec.util.AssertionUtil;
-
import io.netty.buffer.DrillBuf;
import io.netty.util.internal.PlatformDependent;
+import org.apache.drill.exec.memory.BoundsChecking;
+
import com.google.common.primitives.UnsignedLongs;
public final class XXHash {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(XXHash.class);
- private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
+ private static final boolean BOUNDS_CHECKING_ENABLED = BoundsChecking.BOUNDS_CHECKING_ENABLED;
static final long PRIME64_1 = UnsignedLongs.decode("11400714785074694791");
static final long PRIME64_2 = UnsignedLongs.decode("14029467366897019727");
http://git-wip-us.apache.org/repos/asf/drill/blob/62353a6f/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/TextInput.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/TextInput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/TextInput.java
index c764f56..c99e867 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/TextInput.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/TextInput.java
@@ -41,9 +41,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
-import org.apache.drill.exec.ops.OperatorContext;
-import org.apache.drill.exec.ops.OperatorStats;
-import org.apache.drill.exec.util.AssertionUtil;
+import org.apache.drill.exec.memory.BoundsChecking;
import org.apache.hadoop.fs.ByteBufferReadable;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Seekable;
@@ -310,7 +308,7 @@ final class TextInput {
throw StreamFinishedPseudoException.INSTANCE;
}
- if(AssertionUtil.BOUNDS_CHECKING_ENABLED){
+ if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
buffer.checkBytes(bufferPtr - 1, bufferPtr);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/62353a6f/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
index 38eb622..b85502b 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
@@ -30,16 +30,16 @@ import java.nio.charset.Charset;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.exec.memory.Accountor;
+import org.apache.drill.exec.memory.BoundsChecking;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.BufferManager;
-import org.apache.drill.exec.util.AssertionUtil;
import com.google.common.base.Preconditions;
public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillBuf.class);
- private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
+ private static final boolean BOUNDS_CHECKING_ENABLED = BoundsChecking.BOUNDS_CHECKING_ENABLED;
private final ByteBuf b;
private final long addr;
http://git-wip-us.apache.org/repos/asf/drill/blob/62353a6f/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java
new file mode 100644
index 0000000..7588b2c
--- /dev/null
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java
@@ -0,0 +1,17 @@
+package org.apache.drill.exec.memory;
+
+public class BoundsChecking {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BoundsChecking.class);
+
+ public static final boolean BOUNDS_CHECKING_ENABLED;
+
+ static {
+ boolean isAssertEnabled = false;
+ assert isAssertEnabled = true;
+ BOUNDS_CHECKING_ENABLED = isAssertEnabled
+ || !"true".equals(System.getProperty("drill.enable_unsafe_memory_access"));
+ }
+
+ private BoundsChecking() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/62353a6f/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
index 006a084..8a330b8 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
@@ -21,7 +21,7 @@ package org.apache.drill.exec.expr.fn.impl;
import io.netty.buffer.DrillBuf;
import io.netty.util.internal.PlatformDependent;
-import org.apache.drill.exec.util.AssertionUtil;
+import org.apache.drill.exec.memory.BoundsChecking;
import org.apache.drill.exec.util.DecimalUtility;
import com.google.common.primitives.UnsignedLongs;
@@ -29,7 +29,7 @@ import com.google.common.primitives.UnsignedLongs;
public class ByteFunctionHelpers {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
- private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
+ private static final boolean BOUNDS_CHECKING_ENABLED = BoundsChecking.BOUNDS_CHECKING_ENABLED;
/**
* Helper function to check for equality of bytes in two DrillBuffers
[17/45] drill git commit: DRILL-3987: (REFACTOR) Common and Vector
modules building.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/TypeHelper.java b/exec/vector/src/main/codegen/templates/TypeHelper.java
deleted file mode 100644
index b8178cb..0000000
--- a/exec/vector/src/main/codegen/templates/TypeHelper.java
+++ /dev/null
@@ -1,600 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.drill.exec.vector.complex.UnionVector;
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/TypeHelper.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.expr;
-
-<#include "/@includes/vv_imports.ftl" />
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.vector.accessor.*;
-import org.apache.drill.exec.vector.complex.RepeatedMapVector;
-import org.apache.drill.exec.util.CallBack;
-
-public class TypeHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
-
- private static final int WIDTH_ESTIMATE = 50;
-
- // Default length when casting to varchar : 65536 = 2^16
- // This only defines an absolute maximum for values, setting
- // a high value like this will not inflate the size for small values
- public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
-
- private static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
- return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
- }
-
- private static String buildErrorMessage(final String operation, final MajorType type) {
- return buildErrorMessage(operation, type.getMinorType(), type.getMode());
- }
-
- public static int getSize(MajorType major) {
- switch (major.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
- minor.class?substring(0, 3) == "PRO" ||
- minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
- </#list>
-</#list>
- case FIXEDCHAR: return major.getWidth();
- case FIXED16CHAR: return major.getWidth();
- case FIXEDBINARY: return major.getWidth();
- }
- throw new UnsupportedOperationException(buildErrorMessage("get size", major));
- }
-
- public static SqlAccessor getSqlAccessor(ValueVector vector){
- final MajorType type = vector.getField().getType();
- switch(type.getMinorType()){
- case UNION:
- return new UnionSqlAccessor((UnionVector) vector);
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Accessor((${minor.class}Vector) vector);
- case OPTIONAL:
- return new Nullable${minor.class}Accessor((Nullable${minor.class}Vector) vector);
- case REPEATED:
- return new GenericAccessor(vector);
- }
- </#list>
- </#list>
- case MAP:
- case LIST:
- return new GenericAccessor(vector);
- }
- throw new UnsupportedOperationException(buildErrorMessage("find sql accessor", type));
- }
-
- public static ValueVector getNewVector(SchemaPath parentPath, String name, BufferAllocator allocator, MajorType type, CallBack callback){
- SchemaPath child = parentPath.getChild(name);
- MaterializedField field = MaterializedField.create(child, type);
- return getNewVector(field, allocator, callback);
- }
-
-
- public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return UnionVector.class;
- case MAP:
- switch (mode) {
- case OPTIONAL:
- case REQUIRED:
- return MapVector.class;
- case REPEATED:
- return RepeatedMapVector.class;
- }
-
- case LIST:
- switch (mode) {
- case REPEATED:
- return RepeatedListVector.class;
- case REQUIRED:
- case OPTIONAL:
- return ListVector.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}Vector.class;
- case OPTIONAL:
- return Nullable${minor.class}Vector.class;
- case REPEATED:
- return Repeated${minor.class}Vector.class;
- }
- </#list>
-</#list>
- case GENERIC_OBJECT :
- return ObjectVector.class ;
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
- }
- public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
- switch (type) {
- case MAP:
- switch (mode) {
- case REQUIRED:
- if (!isSingularRepeated)
- return SingleMapReaderImpl.class;
- else
- return SingleLikeRepeatedMapReaderImpl.class;
- case REPEATED:
- return RepeatedMapReaderImpl.class;
- }
- case LIST:
- switch (mode) {
- case REQUIRED:
- return SingleListReaderImpl.class;
- case REPEATED:
- return RepeatedListReaderImpl.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}ReaderImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}ReaderImpl.class;
- case REPEATED:
- return Repeated${minor.class}ReaderImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
- }
-
- public static Class<?> getWriterInterface( MinorType type, DataMode mode){
- switch (type) {
- case UNION: return UnionWriter.class;
- case MAP: return MapWriter.class;
- case LIST: return ListWriter.class;
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}: return ${minor.class}Writer.class;
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
- }
-
- public static Class<?> getWriterImpl( MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return UnionWriter.class;
- case MAP:
- switch (mode) {
- case REQUIRED:
- case OPTIONAL:
- return SingleMapWriter.class;
- case REPEATED:
- return RepeatedMapWriter.class;
- }
- case LIST:
- switch (mode) {
- case REQUIRED:
- case OPTIONAL:
- return UnionListWriter.class;
- case REPEATED:
- return RepeatedListWriter.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}WriterImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}WriterImpl.class;
- case REPEATED:
- return Repeated${minor.class}WriterImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
- }
-
- public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
- switch (type) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}HolderReaderImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}HolderReaderImpl.class;
- case REPEATED:
- return Repeated${minor.class}HolderReaderImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
- }
-
- public static JType getHolderType(JCodeModel model, MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return model._ref(UnionHolder.class);
- case MAP:
- case LIST:
- return model._ref(ComplexHolder.class);
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return model._ref(${minor.class}Holder.class);
- case OPTIONAL:
- return model._ref(Nullable${minor.class}Holder.class);
- case REPEATED:
- return model._ref(Repeated${minor.class}Holder.class);
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- return model._ref(ObjectHolder.class);
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get holder type", type, mode));
- }
-
- public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
- return getNewVector(field, allocator, null);
- }
- public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- MajorType type = field.getType();
-
- switch (type.getMinorType()) {
-
- case UNION:
- return new UnionVector(field, allocator, callBack);
-
- case MAP:
- switch (type.getMode()) {
- case REQUIRED:
- case OPTIONAL:
- return new MapVector(field, allocator, callBack);
- case REPEATED:
- return new RepeatedMapVector(field, allocator, callBack);
- }
- case LIST:
- switch (type.getMode()) {
- case REPEATED:
- return new RepeatedListVector(field, allocator, callBack);
- case OPTIONAL:
- case REQUIRED:
- return new ListVector(field, allocator, callBack);
- }
-<#list vv. types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Vector(field, allocator);
- case OPTIONAL:
- return new Nullable${minor.class}Vector(field, allocator);
- case REPEATED:
- return new Repeated${minor.class}Vector(field, allocator);
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- return new ObjectVector(field, allocator) ;
- default:
- break;
- }
- // All ValueVector types have been handled.
- throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
- }
-
- public static ValueHolder getValue(ValueVector vector, int index) {
- MajorType type = vector.getField().getType();
- ValueHolder holder;
- switch(type.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
- minor.class?starts_with("Decimal28") || minor.class?starts_with("Decimal38")>
- switch (type.getMode()) {
- case REQUIRED:
- holder = new ${minor.class}Holder();
- ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
- return holder;
- case OPTIONAL:
- holder = new Nullable${minor.class}Holder();
- ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
- if (((Nullable${minor.class}Holder)holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
- }
- return holder;
- }
- <#else>
- switch (type.getMode()) {
- case REQUIRED:
- holder = new ${minor.class}Holder();
- ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
- return holder;
- case OPTIONAL:
- holder = new Nullable${minor.class}Holder();
- ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
- if (((Nullable${minor.class}Holder)holder).isSet == 1) {
- ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
- }
- return holder;
- }
- </#if>
- </#list>
-</#list>
- case GENERIC_OBJECT:
- holder = new ObjectHolder();
- ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index) ;
- break;
- }
-
- throw new UnsupportedOperationException(buildErrorMessage("get value", type));
- }
-
- public static void setValue(ValueVector vector, int index, ValueHolder holder) {
- MajorType type = vector.getField().getType();
-
- switch(type.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
- return;
- case OPTIONAL:
- if (((Nullable${minor.class}Holder) holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
- }
- return;
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
- return;
- default:
- throw new UnsupportedOperationException(buildErrorMessage("set value", type));
- }
- }
-
- public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
- MajorType type = vector.getField().getType();
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
- return;
- case OPTIONAL:
- if (((Nullable${minor.class}Holder) holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
- } else {
- ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
- }
- return;
- }
- </#list>
- </#list>
- case GENERIC_OBJECT:
- ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
- default:
- throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
- }
- }
-
- public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
- MajorType type1 = v1.getField().getType();
- MajorType type2 = v2.getField().getType();
-
- if (type1.getMinorType() != type2.getMinorType()) {
- return false;
- }
-
- switch(type1.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) ==
- ((${minor.class}Vector) v2).getAccessor().get(v2index) )
- return true;
- break;
- </#list>
-</#list>
- default:
- break;
- }
- return false;
- }
-
- /**
- * Create a ValueHolder of MajorType.
- * @param type
- * @return
- */
- public static ValueHolder createValueHolder(MajorType type) {
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Holder();
- case OPTIONAL:
- return new Nullable${minor.class}Holder();
- case REPEATED:
- return new Repeated${minor.class}Holder();
- }
- </#list>
- </#list>
- case GENERIC_OBJECT:
- return new ObjectHolder();
- default:
- throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
- }
- }
-
- public static boolean isNull(ValueHolder holder) {
- MajorType type = TypeHelper.getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return true;
- case OPTIONAL:
- return ((Nullable${minor.class}Holder) holder).isSet == 0;
- case REPEATED:
- return true;
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
- }
- }
-
- public static ValueHolder deNullify(ValueHolder holder) {
- MajorType type = TypeHelper.getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return holder;
- case OPTIONAL:
- if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
- ${minor.class}Holder newHolder = new ${minor.class}Holder();
-
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
- </#list>
-
- return newHolder;
- } else {
- throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
- }
- case REPEATED:
- return holder;
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
- }
- }
-
- public static ValueHolder nullify(ValueHolder holder) {
- MajorType type = TypeHelper.getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
- newHolder.isSet = 1;
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
- </#list>
- return newHolder;
- case OPTIONAL:
- return holder;
- case REPEATED:
- throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
- }
- }
-
- public static MajorType getValueHolderType(ValueHolder holder) {
-
- if (0 == 1) {
- return null;
- }
- <#list vv.types as type>
- <#list type.minor as minor>
- else if (holder instanceof ${minor.class}Holder) {
- return ((${minor.class}Holder) holder).TYPE;
- } else if (holder instanceof Nullable${minor.class}Holder) {
- return ((Nullable${minor.class}Holder) holder).TYPE;
- }
- </#list>
- </#list>
-
- throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/UnionVector.java b/exec/vector/src/main/codegen/templates/UnionVector.java
index dfcccb2..2ac4e96 100644
--- a/exec/vector/src/main/codegen/templates/UnionVector.java
+++ b/exec/vector/src/main/codegen/templates/UnionVector.java
@@ -151,7 +151,7 @@ public class UnionVector implements ValueVector {
}
@Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
+ public void allocateNew() throws OutOfMemoryException {
internalMap.allocateNew();
if (typeVector != null) {
typeVector.zeroVector();
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
index bba39fe..f734734 100644
--- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -18,7 +18,7 @@
import java.lang.Override;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.vector.BaseDataValueVector;
import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.VariableWidthVector;
@@ -284,7 +284,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
@Override
public void allocateNew() {
if(!allocateNewSafe()){
- throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
+ throw new OutOfMemoryException("Failure while allocating buffer.");
}
}
@@ -314,7 +314,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
data = allocator.buffer(requestedSize);
allocationSizeInBytes = requestedSize;
offsetVector.allocateNew();
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
clear();
return false;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java b/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
index 5e437b2..38eb622 100644
--- a/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
+++ b/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
@@ -32,8 +32,6 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.exec.memory.Accountor;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.BufferManager;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.util.AssertionUtil;
import com.google.common.base.Preconditions;
@@ -58,8 +56,6 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
// to share code and to remove the hacky code here to use only
// one of these types at a time and use null checks to find out
// which.
- private OperatorContext context;
- private FragmentContext fContext;
private BufferManager bufManager;
public DrillBuf(BufferAllocator allocator, Accountor a, UnsafeDirectLittleEndian b) {
@@ -127,13 +123,6 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
this.allocator = allocator;
}
- public void setOperatorContext(OperatorContext c) {
- this.context = c;
- }
- public void setFragmentContext(FragmentContext c) {
- this.fContext = c;
- }
-
public void setBufferManager(BufferManager bufManager) {
this.bufManager = bufManager;
}
@@ -146,11 +135,8 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
if (this.capacity() >= size) {
return this;
}
- if (context != null) {
- return context.replace(this, size);
- } else if(fContext != null) {
- return fContext.replace(this, size);
- } else if (bufManager != null) {
+
+ if (bufManager != null) {
return bufManager.replace(this, size);
} else {
throw new UnsupportedOperationException("Realloc is only available in the context of an operator's UDFs");
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java b/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
index 721aff9..53ca91c 100644
--- a/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
+++ b/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
@@ -17,11 +17,10 @@
*/
package io.netty.buffer;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.Accountor;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.memory.LimitConsumer;
import org.apache.drill.exec.util.Pointer;
class FakeAllocator implements BufferAllocator {
@@ -47,7 +46,7 @@ class FakeAllocator implements BufferAllocator {
}
@Override
- public BufferAllocator getChildAllocator(FragmentContext context, long initialReservation, long maximumReservation,
+ public BufferAllocator getChildAllocator(LimitConsumer consumer, long initialReservation, long maximumReservation,
boolean applyFragmentLimit)
throws OutOfMemoryException {
throw new UnsupportedOperationException();
@@ -69,17 +68,17 @@ class FakeAllocator implements BufferAllocator {
}
@Override
- public void resetFragmentLimits() {
+ public void resetLimits() {
throw new UnsupportedOperationException();
}
@Override
- public void setFragmentLimit(long l) {
+ public void setLimit(long l) {
throw new UnsupportedOperationException();
}
@Override
- public long getFragmentLimit(){
+ public long getLimit(){
throw new UnsupportedOperationException();
}
@@ -97,10 +96,9 @@ class FakeAllocator implements BufferAllocator {
return 0;
}
- static class FakeAccountor extends Accountor {
+ static class FakeAccountor implements Accountor {
public FakeAccountor() {
- super(null, false, null, null, 0, 0, true);
}
@Override
@@ -140,17 +138,42 @@ class FakeAllocator implements BufferAllocator {
@Override
public void releasePartial(DrillBuf buf, long size) {
-
+ throw new UnsupportedOperationException();
}
@Override
public void release(DrillBuf buf, long size) {
-
+ throw new UnsupportedOperationException();
}
@Override
public void close() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean transferIn(DrillBuf buf, long size) {
+ return false;
+ }
+
+ @Override
+ public long getPeakMemoryAllocation() {
+ return 0;
+ }
+
+ @Override
+ public long resetFragmentLimits() {
+ return 0;
+ }
+ @Override
+ public void setFragmentLimit(long add) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getFragmentLimit() {
+ return 0;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index 2ca79f0..5809df5 100644
--- a/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -24,7 +24,6 @@ import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.exec.util.AssertionUtil;
import com.codahale.metrics.Gauge;
@@ -40,9 +39,7 @@ public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
private static final String METRIC_PREFIX = "drill.allocator.";
- public static final PooledByteBufAllocatorL DEFAULT = new PooledByteBufAllocatorL();
-
- private final MetricRegistry registry = DrillMetrics.getInstance();
+ private final MetricRegistry registry;
private final AtomicLong hugeBufferSize = new AtomicLong(0);
private final AtomicLong hugeBufferCount = new AtomicLong(0);
private final AtomicLong normalBufferSize = new AtomicLong(0);
@@ -53,8 +50,9 @@ public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
private final Histogram largeBuffersHist;
private final Histogram normalBuffersHist;
- private PooledByteBufAllocatorL() {
+ public PooledByteBufAllocatorL(MetricRegistry registry) {
super(true);
+ this.registry = registry;
try {
Field f = PooledByteBufAllocator.class.getDeclaredField("directArenas");
f.setAccessible(true);
@@ -110,6 +108,7 @@ public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
public boolean matches(String name, Metric metric) {
return name.startsWith("drill.allocator.");
}
+
});
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java b/exec/vector/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
new file mode 100644
index 0000000..17d099d
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.exception;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+public class OutOfMemoryException extends DrillRuntimeException {
+ private static final long serialVersionUID = -6858052345185793382L;
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutOfMemoryException.class);
+
+ public OutOfMemoryException() {
+ super();
+ }
+
+ public OutOfMemoryException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public OutOfMemoryException(String message, Throwable cause) {
+ super(message, cause);
+
+ }
+
+ public OutOfMemoryException(String message) {
+ super(message);
+
+ }
+
+ public OutOfMemoryException(Throwable cause) {
+ super(cause);
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java b/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
index f5ae70c..c192359 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.exception;
import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.record.RecordBatch;
/**
* An exception that is used to signal that allocation request in bytes is greater than the maximum allowed by
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
new file mode 100644
index 0000000..006a084
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
@@ -0,0 +1,236 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.expr.fn.impl;
+
+import io.netty.buffer.DrillBuf;
+import io.netty.util.internal.PlatformDependent;
+
+import org.apache.drill.exec.util.AssertionUtil;
+import org.apache.drill.exec.util.DecimalUtility;
+
+import com.google.common.primitives.UnsignedLongs;
+
+public class ByteFunctionHelpers {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
+
+ private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
+
+ /**
+ * Helper function to check for equality of bytes in two DrillBuffers
+ *
+ * @param left Left DrillBuf for comparison
+ * @param lStart start offset in the buffer
+ * @param lEnd end offset in the buffer
+ * @param right Right DrillBuf for comparison
+ * @param rStart start offset in the buffer
+ * @param rEnd end offset in the buffer
+ * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+ */
+ public static final int equal(final DrillBuf left, int lStart, int lEnd, final DrillBuf right, int rStart, int rEnd){
+ if(BOUNDS_CHECKING_ENABLED){
+ left.checkBytes(lStart, lEnd);
+ right.checkBytes(rStart, rEnd);
+ }
+ return memEqual(left.memoryAddress(), lStart, lEnd, right.memoryAddress(), rStart, rEnd);
+ }
+
+ private static final int memEqual(final long laddr, int lStart, int lEnd, final long raddr, int rStart,
+ final int rEnd) {
+
+ int n = lEnd - lStart;
+ if (n == rEnd - rStart) {
+ long lPos = laddr + lStart;
+ long rPos = raddr + rStart;
+
+ while (n > 7) {
+ long leftLong = PlatformDependent.getLong(lPos);
+ long rightLong = PlatformDependent.getLong(rPos);
+ if (leftLong != rightLong) {
+ return 0;
+ }
+ lPos += 8;
+ rPos += 8;
+ n -= 8;
+ }
+ while (n-- != 0) {
+ byte leftByte = PlatformDependent.getByte(lPos);
+ byte rightByte = PlatformDependent.getByte(rPos);
+ if (leftByte != rightByte) {
+ return 0;
+ }
+ lPos++;
+ rPos++;
+ }
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ /**
+ * Helper function to compare a set of bytes in two DrillBuffers.
+ *
+ * Function will check data before completing in the case that
+ *
+ * @param left Left DrillBuf to compare
+ * @param lStart start offset in the buffer
+ * @param lEnd end offset in the buffer
+ * @param right Right DrillBuf to compare
+ * @param rStart start offset in the buffer
+ * @param rEnd end offset in the buffer
+ * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+ */
+ public static final int compare(final DrillBuf left, int lStart, int lEnd, final DrillBuf right, int rStart, int rEnd){
+ if(BOUNDS_CHECKING_ENABLED){
+ left.checkBytes(lStart, lEnd);
+ right.checkBytes(rStart, rEnd);
+ }
+ return memcmp(left.memoryAddress(), lStart, lEnd, right.memoryAddress(), rStart, rEnd);
+ }
+
+ private static final int memcmp(final long laddr, int lStart, int lEnd, final long raddr, int rStart, final int rEnd) {
+ int lLen = lEnd - lStart;
+ int rLen = rEnd - rStart;
+ int n = Math.min(rLen, lLen);
+ long lPos = laddr + lStart;
+ long rPos = raddr + rStart;
+
+ while (n > 7) {
+ long leftLong = PlatformDependent.getLong(lPos);
+ long rightLong = PlatformDependent.getLong(rPos);
+ if (leftLong != rightLong) {
+ return UnsignedLongs.compare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ }
+ lPos += 8;
+ rPos += 8;
+ n -= 8;
+ }
+
+ while (n-- != 0) {
+ byte leftByte = PlatformDependent.getByte(lPos);
+ byte rightByte = PlatformDependent.getByte(rPos);
+ if (leftByte != rightByte) {
+ return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+ }
+ lPos++;
+ rPos++;
+ }
+
+ if (lLen == rLen) {
+ return 0;
+ }
+
+ return lLen > rLen ? 1 : -1;
+
+ }
+
+ /**
+ * Helper function to compare a set of bytes in DrillBuf to a ByteArray.
+ *
+ * @param left Left DrillBuf for comparison purposes
+ * @param lStart start offset in the buffer
+ * @param lEnd end offset in the buffer
+ * @param right second input to be compared
+ * @param rStart start offset in the byte array
+ * @param rEnd end offset in the byte array
+ * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
+ */
+ public static final int compare(final DrillBuf left, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
+ if(BOUNDS_CHECKING_ENABLED){
+ left.checkBytes(lStart, lEnd);
+ }
+ return memcmp(left.memoryAddress(), lStart, lEnd, right, rStart, rEnd);
+ }
+
+
+ private static final int memcmp(final long laddr, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
+ int lLen = lEnd - lStart;
+ int rLen = rEnd - rStart;
+ int n = Math.min(rLen, lLen);
+ long lPos = laddr + lStart;
+ int rPos = rStart;
+
+ while (n-- != 0) {
+ byte leftByte = PlatformDependent.getByte(lPos);
+ byte rightByte = right[rPos];
+ if (leftByte != rightByte) {
+ return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+ }
+ lPos++;
+ rPos++;
+ }
+
+ if (lLen == rLen) {
+ return 0;
+ }
+
+ return lLen > rLen ? 1 : -1;
+ }
+
+ /*
+ * Following are helper functions to interact with sparse decimal represented in a byte array.
+ */
+
+ // Get the integer ignore the sign
+ public static int getInteger(byte[] b, int index) {
+ return getInteger(b, index, true);
+ }
+ // Get the integer, ignore the sign
+ public static int getInteger(byte[] b, int index, boolean ignoreSign) {
+ int startIndex = index * DecimalUtility.INTEGER_SIZE;
+
+ if (index == 0 && ignoreSign == true) {
+ return (b[startIndex + 3] & 0xFF) |
+ (b[startIndex + 2] & 0xFF) << 8 |
+ (b[startIndex + 1] & 0xFF) << 16 |
+ (b[startIndex] & 0x7F) << 24;
+ }
+
+ return ((b[startIndex + 3] & 0xFF) |
+ (b[startIndex + 2] & 0xFF) << 8 |
+ (b[startIndex + 1] & 0xFF) << 16 |
+ (b[startIndex] & 0xFF) << 24);
+
+ }
+
+ // Set integer in the byte array
+ public static void setInteger(byte[] b, int index, int value) {
+ int startIndex = index * DecimalUtility.INTEGER_SIZE;
+ b[startIndex] = (byte) ((value >> 24) & 0xFF);
+ b[startIndex + 1] = (byte) ((value >> 16) & 0xFF);
+ b[startIndex + 2] = (byte) ((value >> 8) & 0xFF);
+ b[startIndex + 3] = (byte) ((value) & 0xFF);
+ }
+
+ // Set the sign in a sparse decimal representation
+ public static void setSign(byte[] b, boolean sign) {
+ int value = getInteger(b, 0);
+ if (sign == true) {
+ setInteger(b, 0, value | 0x80000000);
+ } else {
+ setInteger(b, 0, value & 0x7FFFFFFF);
+ }
+ }
+
+ // Get the sign
+ public static boolean getSign(byte[] b) {
+ return ((getInteger(b, 0, false) & 0x80000000) != 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java
new file mode 100644
index 0000000..98a0b61
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java
@@ -0,0 +1,682 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.joda.time.Period;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.DateTimeFormatterBuilder;
+import org.joda.time.format.DateTimeParser;
+
+import com.carrotsearch.hppc.ObjectIntOpenHashMap;
+
+// Utility class for Date, DateTime, TimeStamp, Interval data types
+public class DateUtility {
+
+
+ /* We have a hashmap that stores the timezone as the key and an index as the value
+ * While storing the timezone in value vectors, holders we only use this index. As we
+ * reconstruct the timestamp, we use this index to index through the array timezoneList
+ * and get the corresponding timezone and pass it to joda-time
+ */
+ public static ObjectIntOpenHashMap<String> timezoneMap = new ObjectIntOpenHashMap<String>();
+
+ public static String[] timezoneList = {"Africa/Abidjan",
+ "Africa/Accra",
+ "Africa/Addis_Ababa",
+ "Africa/Algiers",
+ "Africa/Asmara",
+ "Africa/Asmera",
+ "Africa/Bamako",
+ "Africa/Bangui",
+ "Africa/Banjul",
+ "Africa/Bissau",
+ "Africa/Blantyre",
+ "Africa/Brazzaville",
+ "Africa/Bujumbura",
+ "Africa/Cairo",
+ "Africa/Casablanca",
+ "Africa/Ceuta",
+ "Africa/Conakry",
+ "Africa/Dakar",
+ "Africa/Dar_es_Salaam",
+ "Africa/Djibouti",
+ "Africa/Douala",
+ "Africa/El_Aaiun",
+ "Africa/Freetown",
+ "Africa/Gaborone",
+ "Africa/Harare",
+ "Africa/Johannesburg",
+ "Africa/Juba",
+ "Africa/Kampala",
+ "Africa/Khartoum",
+ "Africa/Kigali",
+ "Africa/Kinshasa",
+ "Africa/Lagos",
+ "Africa/Libreville",
+ "Africa/Lome",
+ "Africa/Luanda",
+ "Africa/Lubumbashi",
+ "Africa/Lusaka",
+ "Africa/Malabo",
+ "Africa/Maputo",
+ "Africa/Maseru",
+ "Africa/Mbabane",
+ "Africa/Mogadishu",
+ "Africa/Monrovia",
+ "Africa/Nairobi",
+ "Africa/Ndjamena",
+ "Africa/Niamey",
+ "Africa/Nouakchott",
+ "Africa/Ouagadougou",
+ "Africa/Porto-Novo",
+ "Africa/Sao_Tome",
+ "Africa/Timbuktu",
+ "Africa/Tripoli",
+ "Africa/Tunis",
+ "Africa/Windhoek",
+ "America/Adak",
+ "America/Anchorage",
+ "America/Anguilla",
+ "America/Antigua",
+ "America/Araguaina",
+ "America/Argentina/Buenos_Aires",
+ "America/Argentina/Catamarca",
+ "America/Argentina/ComodRivadavia",
+ "America/Argentina/Cordoba",
+ "America/Argentina/Jujuy",
+ "America/Argentina/La_Rioja",
+ "America/Argentina/Mendoza",
+ "America/Argentina/Rio_Gallegos",
+ "America/Argentina/Salta",
+ "America/Argentina/San_Juan",
+ "America/Argentina/San_Luis",
+ "America/Argentina/Tucuman",
+ "America/Argentina/Ushuaia",
+ "America/Aruba",
+ "America/Asuncion",
+ "America/Atikokan",
+ "America/Atka",
+ "America/Bahia",
+ "America/Bahia_Banderas",
+ "America/Barbados",
+ "America/Belem",
+ "America/Belize",
+ "America/Blanc-Sablon",
+ "America/Boa_Vista",
+ "America/Bogota",
+ "America/Boise",
+ "America/Buenos_Aires",
+ "America/Cambridge_Bay",
+ "America/Campo_Grande",
+ "America/Cancun",
+ "America/Caracas",
+ "America/Catamarca",
+ "America/Cayenne",
+ "America/Cayman",
+ "America/Chicago",
+ "America/Chihuahua",
+ "America/Coral_Harbour",
+ "America/Cordoba",
+ "America/Costa_Rica",
+ "America/Cuiaba",
+ "America/Curacao",
+ "America/Danmarkshavn",
+ "America/Dawson",
+ "America/Dawson_Creek",
+ "America/Denver",
+ "America/Detroit",
+ "America/Dominica",
+ "America/Edmonton",
+ "America/Eirunepe",
+ "America/El_Salvador",
+ "America/Ensenada",
+ "America/Fort_Wayne",
+ "America/Fortaleza",
+ "America/Glace_Bay",
+ "America/Godthab",
+ "America/Goose_Bay",
+ "America/Grand_Turk",
+ "America/Grenada",
+ "America/Guadeloupe",
+ "America/Guatemala",
+ "America/Guayaquil",
+ "America/Guyana",
+ "America/Halifax",
+ "America/Havana",
+ "America/Hermosillo",
+ "America/Indiana/Indianapolis",
+ "America/Indiana/Knox",
+ "America/Indiana/Marengo",
+ "America/Indiana/Petersburg",
+ "America/Indiana/Tell_City",
+ "America/Indiana/Vevay",
+ "America/Indiana/Vincennes",
+ "America/Indiana/Winamac",
+ "America/Indianapolis",
+ "America/Inuvik",
+ "America/Iqaluit",
+ "America/Jamaica",
+ "America/Jujuy",
+ "America/Juneau",
+ "America/Kentucky/Louisville",
+ "America/Kentucky/Monticello",
+ "America/Knox_IN",
+ "America/Kralendijk",
+ "America/La_Paz",
+ "America/Lima",
+ "America/Los_Angeles",
+ "America/Louisville",
+ "America/Lower_Princes",
+ "America/Maceio",
+ "America/Managua",
+ "America/Manaus",
+ "America/Marigot",
+ "America/Martinique",
+ "America/Matamoros",
+ "America/Mazatlan",
+ "America/Mendoza",
+ "America/Menominee",
+ "America/Merida",
+ "America/Metlakatla",
+ "America/Mexico_City",
+ "America/Miquelon",
+ "America/Moncton",
+ "America/Monterrey",
+ "America/Montevideo",
+ "America/Montreal",
+ "America/Montserrat",
+ "America/Nassau",
+ "America/New_York",
+ "America/Nipigon",
+ "America/Nome",
+ "America/Noronha",
+ "America/North_Dakota/Beulah",
+ "America/North_Dakota/Center",
+ "America/North_Dakota/New_Salem",
+ "America/Ojinaga",
+ "America/Panama",
+ "America/Pangnirtung",
+ "America/Paramaribo",
+ "America/Phoenix",
+ "America/Port-au-Prince",
+ "America/Port_of_Spain",
+ "America/Porto_Acre",
+ "America/Porto_Velho",
+ "America/Puerto_Rico",
+ "America/Rainy_River",
+ "America/Rankin_Inlet",
+ "America/Recife",
+ "America/Regina",
+ "America/Resolute",
+ "America/Rio_Branco",
+ "America/Rosario",
+ "America/Santa_Isabel",
+ "America/Santarem",
+ "America/Santiago",
+ "America/Santo_Domingo",
+ "America/Sao_Paulo",
+ "America/Scoresbysund",
+ "America/Shiprock",
+ "America/Sitka",
+ "America/St_Barthelemy",
+ "America/St_Johns",
+ "America/St_Kitts",
+ "America/St_Lucia",
+ "America/St_Thomas",
+ "America/St_Vincent",
+ "America/Swift_Current",
+ "America/Tegucigalpa",
+ "America/Thule",
+ "America/Thunder_Bay",
+ "America/Tijuana",
+ "America/Toronto",
+ "America/Tortola",
+ "America/Vancouver",
+ "America/Virgin",
+ "America/Whitehorse",
+ "America/Winnipeg",
+ "America/Yakutat",
+ "America/Yellowknife",
+ "Antarctica/Casey",
+ "Antarctica/Davis",
+ "Antarctica/DumontDUrville",
+ "Antarctica/Macquarie",
+ "Antarctica/Mawson",
+ "Antarctica/McMurdo",
+ "Antarctica/Palmer",
+ "Antarctica/Rothera",
+ "Antarctica/South_Pole",
+ "Antarctica/Syowa",
+ "Antarctica/Vostok",
+ "Arctic/Longyearbyen",
+ "Asia/Aden",
+ "Asia/Almaty",
+ "Asia/Amman",
+ "Asia/Anadyr",
+ "Asia/Aqtau",
+ "Asia/Aqtobe",
+ "Asia/Ashgabat",
+ "Asia/Ashkhabad",
+ "Asia/Baghdad",
+ "Asia/Bahrain",
+ "Asia/Baku",
+ "Asia/Bangkok",
+ "Asia/Beirut",
+ "Asia/Bishkek",
+ "Asia/Brunei",
+ "Asia/Calcutta",
+ "Asia/Choibalsan",
+ "Asia/Chongqing",
+ "Asia/Chungking",
+ "Asia/Colombo",
+ "Asia/Dacca",
+ "Asia/Damascus",
+ "Asia/Dhaka",
+ "Asia/Dili",
+ "Asia/Dubai",
+ "Asia/Dushanbe",
+ "Asia/Gaza",
+ "Asia/Harbin",
+ "Asia/Hebron",
+ "Asia/Ho_Chi_Minh",
+ "Asia/Hong_Kong",
+ "Asia/Hovd",
+ "Asia/Irkutsk",
+ "Asia/Istanbul",
+ "Asia/Jakarta",
+ "Asia/Jayapura",
+ "Asia/Jerusalem",
+ "Asia/Kabul",
+ "Asia/Kamchatka",
+ "Asia/Karachi",
+ "Asia/Kashgar",
+ "Asia/Kathmandu",
+ "Asia/Katmandu",
+ "Asia/Kolkata",
+ "Asia/Krasnoyarsk",
+ "Asia/Kuala_Lumpur",
+ "Asia/Kuching",
+ "Asia/Kuwait",
+ "Asia/Macao",
+ "Asia/Macau",
+ "Asia/Magadan",
+ "Asia/Makassar",
+ "Asia/Manila",
+ "Asia/Muscat",
+ "Asia/Nicosia",
+ "Asia/Novokuznetsk",
+ "Asia/Novosibirsk",
+ "Asia/Omsk",
+ "Asia/Oral",
+ "Asia/Phnom_Penh",
+ "Asia/Pontianak",
+ "Asia/Pyongyang",
+ "Asia/Qatar",
+ "Asia/Qyzylorda",
+ "Asia/Rangoon",
+ "Asia/Riyadh",
+ "Asia/Saigon",
+ "Asia/Sakhalin",
+ "Asia/Samarkand",
+ "Asia/Seoul",
+ "Asia/Shanghai",
+ "Asia/Singapore",
+ "Asia/Taipei",
+ "Asia/Tashkent",
+ "Asia/Tbilisi",
+ "Asia/Tehran",
+ "Asia/Tel_Aviv",
+ "Asia/Thimbu",
+ "Asia/Thimphu",
+ "Asia/Tokyo",
+ "Asia/Ujung_Pandang",
+ "Asia/Ulaanbaatar",
+ "Asia/Ulan_Bator",
+ "Asia/Urumqi",
+ "Asia/Vientiane",
+ "Asia/Vladivostok",
+ "Asia/Yakutsk",
+ "Asia/Yekaterinburg",
+ "Asia/Yerevan",
+ "Atlantic/Azores",
+ "Atlantic/Bermuda",
+ "Atlantic/Canary",
+ "Atlantic/Cape_Verde",
+ "Atlantic/Faeroe",
+ "Atlantic/Faroe",
+ "Atlantic/Jan_Mayen",
+ "Atlantic/Madeira",
+ "Atlantic/Reykjavik",
+ "Atlantic/South_Georgia",
+ "Atlantic/St_Helena",
+ "Atlantic/Stanley",
+ "Australia/ACT",
+ "Australia/Adelaide",
+ "Australia/Brisbane",
+ "Australia/Broken_Hill",
+ "Australia/Canberra",
+ "Australia/Currie",
+ "Australia/Darwin",
+ "Australia/Eucla",
+ "Australia/Hobart",
+ "Australia/LHI",
+ "Australia/Lindeman",
+ "Australia/Lord_Howe",
+ "Australia/Melbourne",
+ "Australia/NSW",
+ "Australia/North",
+ "Australia/Perth",
+ "Australia/Queensland",
+ "Australia/South",
+ "Australia/Sydney",
+ "Australia/Tasmania",
+ "Australia/Victoria",
+ "Australia/West",
+ "Australia/Yancowinna",
+ "Brazil/Acre",
+ "Brazil/DeNoronha",
+ "Brazil/East",
+ "Brazil/West",
+ "CET",
+ "CST6CDT",
+ "Canada/Atlantic",
+ "Canada/Central",
+ "Canada/East-Saskatchewan",
+ "Canada/Eastern",
+ "Canada/Mountain",
+ "Canada/Newfoundland",
+ "Canada/Pacific",
+ "Canada/Saskatchewan",
+ "Canada/Yukon",
+ "Chile/Continental",
+ "Chile/EasterIsland",
+ "Cuba",
+ "EET",
+ "EST",
+ "EST5EDT",
+ "Egypt",
+ "Eire",
+ "Etc/GMT",
+ "Etc/GMT+0",
+ "Etc/GMT+1",
+ "Etc/GMT+10",
+ "Etc/GMT+11",
+ "Etc/GMT+12",
+ "Etc/GMT+2",
+ "Etc/GMT+3",
+ "Etc/GMT+4",
+ "Etc/GMT+5",
+ "Etc/GMT+6",
+ "Etc/GMT+7",
+ "Etc/GMT+8",
+ "Etc/GMT+9",
+ "Etc/GMT-0",
+ "Etc/GMT-1",
+ "Etc/GMT-10",
+ "Etc/GMT-11",
+ "Etc/GMT-12",
+ "Etc/GMT-13",
+ "Etc/GMT-14",
+ "Etc/GMT-2",
+ "Etc/GMT-3",
+ "Etc/GMT-4",
+ "Etc/GMT-5",
+ "Etc/GMT-6",
+ "Etc/GMT-7",
+ "Etc/GMT-8",
+ "Etc/GMT-9",
+ "Etc/GMT0",
+ "Etc/Greenwich",
+ "Etc/UCT",
+ "Etc/UTC",
+ "Etc/Universal",
+ "Etc/Zulu",
+ "Europe/Amsterdam",
+ "Europe/Andorra",
+ "Europe/Athens",
+ "Europe/Belfast",
+ "Europe/Belgrade",
+ "Europe/Berlin",
+ "Europe/Bratislava",
+ "Europe/Brussels",
+ "Europe/Bucharest",
+ "Europe/Budapest",
+ "Europe/Chisinau",
+ "Europe/Copenhagen",
+ "Europe/Dublin",
+ "Europe/Gibraltar",
+ "Europe/Guernsey",
+ "Europe/Helsinki",
+ "Europe/Isle_of_Man",
+ "Europe/Istanbul",
+ "Europe/Jersey",
+ "Europe/Kaliningrad",
+ "Europe/Kiev",
+ "Europe/Lisbon",
+ "Europe/Ljubljana",
+ "Europe/London",
+ "Europe/Luxembourg",
+ "Europe/Madrid",
+ "Europe/Malta",
+ "Europe/Mariehamn",
+ "Europe/Minsk",
+ "Europe/Monaco",
+ "Europe/Moscow",
+ "Europe/Nicosia",
+ "Europe/Oslo",
+ "Europe/Paris",
+ "Europe/Podgorica",
+ "Europe/Prague",
+ "Europe/Riga",
+ "Europe/Rome",
+ "Europe/Samara",
+ "Europe/San_Marino",
+ "Europe/Sarajevo",
+ "Europe/Simferopol",
+ "Europe/Skopje",
+ "Europe/Sofia",
+ "Europe/Stockholm",
+ "Europe/Tallinn",
+ "Europe/Tirane",
+ "Europe/Tiraspol",
+ "Europe/Uzhgorod",
+ "Europe/Vaduz",
+ "Europe/Vatican",
+ "Europe/Vienna",
+ "Europe/Vilnius",
+ "Europe/Volgograd",
+ "Europe/Warsaw",
+ "Europe/Zagreb",
+ "Europe/Zaporozhye",
+ "Europe/Zurich",
+ "GB",
+ "GB-Eire",
+ "GMT",
+ "GMT+0",
+ "GMT-0",
+ "GMT0",
+ "Greenwich",
+ "HST",
+ "Hongkong",
+ "Iceland",
+ "Indian/Antananarivo",
+ "Indian/Chagos",
+ "Indian/Christmas",
+ "Indian/Cocos",
+ "Indian/Comoro",
+ "Indian/Kerguelen",
+ "Indian/Mahe",
+ "Indian/Maldives",
+ "Indian/Mauritius",
+ "Indian/Mayotte",
+ "Indian/Reunion",
+ "Iran",
+ "Israel",
+ "Jamaica",
+ "Japan",
+ "Kwajalein",
+ "Libya",
+ "MET",
+ "MST",
+ "MST7MDT",
+ "Mexico/BajaNorte",
+ "Mexico/BajaSur",
+ "Mexico/General",
+ "NZ",
+ "NZ-CHAT",
+ "Navajo",
+ "PRC",
+ "PST8PDT",
+ "Pacific/Apia",
+ "Pacific/Auckland",
+ "Pacific/Chatham",
+ "Pacific/Chuuk",
+ "Pacific/Easter",
+ "Pacific/Efate",
+ "Pacific/Enderbury",
+ "Pacific/Fakaofo",
+ "Pacific/Fiji",
+ "Pacific/Funafuti",
+ "Pacific/Galapagos",
+ "Pacific/Gambier",
+ "Pacific/Guadalcanal",
+ "Pacific/Guam",
+ "Pacific/Honolulu",
+ "Pacific/Johnston",
+ "Pacific/Kiritimati",
+ "Pacific/Kosrae",
+ "Pacific/Kwajalein",
+ "Pacific/Majuro",
+ "Pacific/Marquesas",
+ "Pacific/Midway",
+ "Pacific/Nauru",
+ "Pacific/Niue",
+ "Pacific/Norfolk",
+ "Pacific/Noumea",
+ "Pacific/Pago_Pago",
+ "Pacific/Palau",
+ "Pacific/Pitcairn",
+ "Pacific/Pohnpei",
+ "Pacific/Ponape",
+ "Pacific/Port_Moresby",
+ "Pacific/Rarotonga",
+ "Pacific/Saipan",
+ "Pacific/Samoa",
+ "Pacific/Tahiti",
+ "Pacific/Tarawa",
+ "Pacific/Tongatapu",
+ "Pacific/Truk",
+ "Pacific/Wake",
+ "Pacific/Wallis",
+ "Pacific/Yap",
+ "Poland",
+ "Portugal",
+ "ROC",
+ "ROK",
+ "Singapore",
+ "Turkey",
+ "UCT",
+ "US/Alaska",
+ "US/Aleutian",
+ "US/Arizona",
+ "US/Central",
+ "US/East-Indiana",
+ "US/Eastern",
+ "US/Hawaii",
+ "US/Indiana-Starke",
+ "US/Michigan",
+ "US/Mountain",
+ "US/Pacific",
+ "US/Pacific-New",
+ "US/Samoa",
+ "UTC",
+ "Universal",
+ "W-SU",
+ "WET",
+ "Zulu"};
+
+ static {
+ for (int i = 0; i < timezoneList.length; i++) {
+ timezoneMap.put(timezoneList[i], i);
+ }
+ }
+
+ public static final DateTimeFormatter formatDate = DateTimeFormat.forPattern("yyyy-MM-dd");
+ public static final DateTimeFormatter formatTimeStamp = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+ public static final DateTimeFormatter formatTimeStampTZ = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ");
+ public static final DateTimeFormatter formatTime = DateTimeFormat.forPattern("HH:mm:ss.SSS");
+
+ public static DateTimeFormatter dateTimeTZFormat = null;
+ public static DateTimeFormatter timeFormat = null;
+
+ public static final int yearsToMonths = 12;
+ public static final int hoursToMillis = 60 * 60 * 1000;
+ public static final int minutesToMillis = 60 * 1000;
+ public static final int secondsToMillis = 1000;
+ public static final int monthToStandardDays = 30;
+ public static final long monthsToMillis = 2592000000L; // 30 * 24 * 60 * 60 * 1000
+ public static final int daysToStandardMillis = 24 * 60 * 60 * 1000;
+
+
+ public static int getIndex(String timezone) {
+ return timezoneMap.get(timezone);
+ }
+
+ public static String getTimeZone(int index) {
+ return timezoneList[index];
+ }
+
+ // Function returns the date time formatter used to parse date strings
+ public static DateTimeFormatter getDateTimeFormatter() {
+
+ if (dateTimeTZFormat == null) {
+ DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");
+ DateTimeParser optionalTime = DateTimeFormat.forPattern(" HH:mm:ss").getParser();
+ DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
+ DateTimeParser optionalZone = DateTimeFormat.forPattern(" ZZZ").getParser();
+
+ dateTimeTZFormat = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).appendOptional(optionalZone).toFormatter();
+ }
+
+ return dateTimeTZFormat;
+ }
+
+ // Function returns time formatter used to parse time strings
+ public static DateTimeFormatter getTimeFormatter() {
+ if (timeFormat == null) {
+ DateTimeFormatter timeFormatter = DateTimeFormat.forPattern("HH:mm:ss");
+ DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
+ timeFormat = new DateTimeFormatterBuilder().append(timeFormatter).appendOptional(optionalSec).toFormatter();
+ }
+ return timeFormat;
+ }
+
+ public static int monthsFromPeriod(Period period){
+ return (period.getYears() * yearsToMonths) + period.getMonths();
+ }
+
+ public static int millisFromPeriod(final Period period){
+ return (period.getHours() * hoursToMillis) +
+ (period.getMinutes() * minutesToMillis) +
+ (period.getSeconds() * secondsToMillis) +
+ (period.getMillis());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/memory/Accountor.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/Accountor.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/Accountor.java
new file mode 100644
index 0000000..7014a0b
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/memory/Accountor.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.DrillBuf;
+
+public interface Accountor extends AutoCloseable{
+
+ public boolean transferTo(Accountor target, DrillBuf buf, long size);
+ public boolean transferIn(DrillBuf buf, long size);
+ public long getAvailable();
+ public long getCapacity();
+ public long getAllocation();
+ public long getPeakMemoryAllocation();
+
+ public boolean reserve(long size);
+ public boolean forceAdditionalReservation(long size);
+
+ public void reserved(long expected, DrillBuf buf);
+
+ public void release(DrillBuf buf, long size);
+ public void releasePartial(DrillBuf buf, long size);
+ public long resetFragmentLimits();
+ public void close();
+
+ public void setFragmentLimit(long add);
+ public long getFragmentLimit();
+}
[42/45] drill git commit: DRILL-3987: (CLEANUP) Final cleanups to get
complete working build/distribution
Posted by ja...@apache.org.
DRILL-3987: (CLEANUP) Final cleanups to get complete working build/distribution
- small cleanups
- move Hook to drill-adbc
- update distribution assembly to include new modules
This closes #250
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/494492f1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/494492f1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/494492f1
Branch: refs/heads/master
Commit: 494492f189ff68480e12c6b40194e7a073d544e4
Parents: 44dea43
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Nov 8 10:37:05 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 18:32:54 2015 -0800
----------------------------------------------------------------------
common/pom.xml | 22 --------
.../drill/common/scanner/BuildTimeScan.java | 9 +++
.../apache/drill/exec/metrics/DrillMetrics.java | 6 +-
common/src/main/resources/drill-module.conf | 14 ++++-
.../common/exceptions/TestUserException.java | 3 +-
.../exec/store/hive/HiveFieldConverter.java | 4 +-
.../drill/exec/store/hive/HiveUtilities.java | 4 +-
distribution/pom.xml | 25 +++++++++
distribution/src/assemble/bin.xml | 5 ++
.../expr/fn/DrillComplexWriterFuncHolder.java | 4 +-
.../record/VectorAccessibleComplexWriter.java | 48 ++++++++++++++++
.../src/main/resources/drill-module.conf | 12 ----
.../apache/drill/TestAllocationException.java | 9 ++-
.../apache/drill/TestOutOfMemoryOutcome.java | 3 +-
.../exec/compile/TestEvaluationVisitor.java | 2 +-
...ll2130JavaExecHamcrestConfigurationTest.java | 12 ++--
.../fragment/TestFragmentExecutorCancel.java | 6 +-
.../src/test/resources/drill-oom-xsort.conf | 2 -
exec/jdbc/pom.xml | 5 ++
.../drill/jdbc/impl/DrillConnectionImpl.java | 2 +-
.../drill/jdbc/impl/DrillResultSetImpl.java | 10 +---
.../apache/drill/jdbc/DrillResultSetTest.java | 13 ++---
.../java/org/apache/drill/jdbc/test/Hook.java | 59 ++++++++++++++++++++
.../org/apache/drill/jdbc/test/JdbcAssert.java | 2 -
.../drill/exec/memory/BoundsChecking.java | 17 ++++++
.../drill/exec/memory/TopLevelAllocator.java | 8 ++-
.../impl/src/main/resources/drill-module.conf | 25 +++++++++
logical/pom.xml | 22 ++++++--
.../common/logical/FormatPluginConfigBase.java | 13 ++++-
logical/src/main/resources/drill-module.conf | 19 +++++++
.../common/expression/parser/TreeTest.java | 5 +-
pom.xml | 1 +
32 files changed, 305 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index db9c9a8..337d848 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -129,28 +129,6 @@
</arguments>
</configuration>
</plugin>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.15</version>
- <configuration>
- <useSystemClassLoader>false</useSystemClassLoader>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.antlr</groupId>
- <artifactId>antlr3-maven-plugin</artifactId>
- <version>3.4</version>
- <configuration>
-
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>antlr</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
index 1de819e..28f3704 100644
--- a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
+++ b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
@@ -87,6 +87,15 @@ public class BuildTimeScan {
}
}
if (result != null) {
+ if (logger.isInfoEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(format("Loaded prescanned packages %s from locations:\n", result.getScannedPackages()));
+ for (URL u : preScanned) {
+ sb.append('\t');
+ sb.append(u.toExternalForm());
+ sb.append('\n');
+ }
+ }
logger.info(format("Loaded prescanned packages %s from locations %s", result.getScannedPackages(), preScanned));
return result;
} else {
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java b/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
index 6fd3dd4..781c9d6 100644
--- a/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
+++ b/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
@@ -37,9 +37,9 @@ import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
public class DrillMetrics {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMetrics.class);
- public static final String METRICS_JMX_OUTPUT_ENABLED = "drill.exec.metrics.jmx.enabled";
- public static final String METRICS_LOG_OUTPUT_ENABLED = "drill.exec.metrics.log.enabled";
- public static final String METRICS_LOG_OUTPUT_INTERVAL = "drill.exec.metrics.log.interval";
+ public static final String METRICS_JMX_OUTPUT_ENABLED = "drill.metrics.jmx.enabled";
+ public static final String METRICS_LOG_OUTPUT_ENABLED = "drill.metrics.log.enabled";
+ public static final String METRICS_LOG_OUTPUT_INTERVAL = "drill.metrics.log.interval";
static final DrillConfig config = DrillConfig.create();
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/common/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/common/src/main/resources/drill-module.conf b/common/src/main/resources/drill-module.conf
index f8226f8..0f0e4f7 100644
--- a/common/src/main/resources/drill-module.conf
+++ b/common/src/main/resources/drill-module.conf
@@ -29,5 +29,17 @@ drill {
org.apache.drill.exec.store.mock,
org.apache.drill.common.logical
]
- }
+ },
+
+ metrics : {
+ context: "drillbit",
+ jmx: {
+ enabled : true
+ },
+ log: {
+ enabled : false,
+ interval : 60
+ }
+ },
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/common/src/test/java/org/apache/drill/common/exceptions/TestUserException.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/common/exceptions/TestUserException.java b/common/src/test/java/org/apache/drill/common/exceptions/TestUserException.java
index 8531b4f..46dfcfd 100644
--- a/common/src/test/java/org/apache/drill/common/exceptions/TestUserException.java
+++ b/common/src/test/java/org/apache/drill/common/exceptions/TestUserException.java
@@ -26,7 +26,8 @@ import org.junit.Test;
* Test various use cases when creating user exceptions
*/
public class TestUserException {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUserException.class);
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
+ .getLogger("--ignore.as.this.is.for.testing.exceptions--");
private Exception wrap(UserException uex, int numWraps) {
Exception ex = uex;
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
index 09c38a2..aff9e22 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
@@ -191,7 +191,7 @@ public abstract class HiveFieldConverter {
public Decimal28(int precision, int scale, FragmentContext context) {
holder.scale = scale;
holder.precision = precision;
- holder.buffer = context.getManagedBuffer(Decimal28SparseHolder.nDecimalDigits * DecimalUtility.integerSize);
+ holder.buffer = context.getManagedBuffer(Decimal28SparseHolder.nDecimalDigits * DecimalUtility.INTEGER_SIZE);
holder.start = 0;
}
@@ -210,7 +210,7 @@ public abstract class HiveFieldConverter {
public Decimal38(int precision, int scale, FragmentContext context) {
holder.scale = scale;
holder.precision = precision;
- holder.buffer = context.getManagedBuffer(Decimal38SparseHolder.nDecimalDigits * DecimalUtility.integerSize);
+ holder.buffer = context.getManagedBuffer(Decimal38SparseHolder.nDecimalDigits * DecimalUtility.INTEGER_SIZE);
holder.start = 0;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
index ce38261..05e51e3 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
@@ -231,7 +231,7 @@ public class HiveUtilities {
}
case DECIMAL28SPARSE: {
- final int needSpace = Decimal28SparseHolder.nDecimalDigits * DecimalUtility.integerSize;
+ final int needSpace = Decimal28SparseHolder.nDecimalDigits * DecimalUtility.INTEGER_SIZE;
Preconditions.checkArgument(managedBuffer.capacity() > needSpace,
String.format("Not sufficient space in given managed buffer. Need %d bytes, buffer has %d bytes",
needSpace, managedBuffer.capacity()));
@@ -252,7 +252,7 @@ public class HiveUtilities {
}
case DECIMAL38SPARSE: {
- final int needSpace = Decimal38SparseHolder.nDecimalDigits * DecimalUtility.integerSize;
+ final int needSpace = Decimal38SparseHolder.nDecimalDigits * DecimalUtility.INTEGER_SIZE;
Preconditions.checkArgument(managedBuffer.capacity() > needSpace,
String.format("Not sufficient space in given managed buffer. Need %d bytes, buffer has %d bytes",
needSpace, managedBuffer.capacity()));
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 896a6fc..3a4db67 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -30,6 +30,31 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-logical</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.exec</groupId>
+ <artifactId>vector</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.memory</groupId>
+ <artifactId>drill-memory-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.memory</groupId>
+ <artifactId>drill-memory-impl</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.exec</groupId>
+ <artifactId>drill-rpc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index acdeefd..69af353 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -86,6 +86,11 @@
<include>org.apache.drill.exec:drill-jdbc:jar</include>
<include>org.apache.drill:drill-protocol:jar</include>
<include>org.apache.drill:drill-common:jar</include>
+ <include>org.apache.drill:drill-logical:jar</include>
+ <include>org.apache.drill.exec:vector:jar</include>
+ <include>org.apache.drill.memory:drill-memory-base:jar</include>
+ <include>org.apache.drill.memory:drill-memory-impl:jar</include>
+ <include>org.apache.drill.exec:drill-rpc:jar</include>
<include>org.apache.drill.exec:drill-java-exec:jar</include>
<include>org.apache.drill.contrib.storage-hive:drill-storage-hive-core</include>
<include>org.apache.drill.contrib.storage-hive:drill-hive-exec-shaded</include>
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java
index f7ad1b3..747a08b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java
@@ -22,7 +22,7 @@ import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.physical.impl.project.ProjectRecordBatch;
-import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
+import org.apache.drill.exec.record.VectorAccessibleComplexWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import com.sun.codemodel.JBlock;
@@ -54,13 +54,13 @@ public class DrillComplexWriterFuncHolder extends DrillSimpleFuncHolder{
JVar complexWriter = g.declareClassField("complexWriter", g.getModel()._ref(ComplexWriter.class));
- JClass cwClass = g.getModel().ref(ComplexWriterImpl.class);
JInvocation container = g.getMappingSet().getOutgoing().invoke("getOutgoingContainer");
//Default name is "col", if not passed in a reference name for the output vector.
String refName = ref == null? "col" : ref.getRootSegment().getPath();
+ JClass cwClass = g.getModel().ref(VectorAccessibleComplexWriter.class);
g.getSetupBlock().assign(complexWriter, cwClass.staticInvoke("getWriter").arg(refName).arg(container));
JClass projBatchClass = g.getModel().ref(ProjectRecordBatch.class);
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorAccessibleComplexWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorAccessibleComplexWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorAccessibleComplexWriter.java
new file mode 100644
index 0000000..08ba9a1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorAccessibleComplexWriter.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.MapVector;
+import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+
+public class VectorAccessibleComplexWriter extends MapVector {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorAccessibleComplexWriter.class);
+
+ private final VectorContainer vc;
+
+ public VectorAccessibleComplexWriter(VectorContainer vc) {
+ super("", null, null);
+ this.vc = vc;
+ }
+
+ @Override
+ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+ final ValueVector v = vc.addOrGet(name, type, clazz);
+ putChild(name, v);
+ return this.typeify(v, clazz);
+
+ }
+
+ public static ComplexWriter getWriter(String name, VectorContainer container) {
+ VectorAccessibleComplexWriter vc = new VectorAccessibleComplexWriter(container);
+ return new ComplexWriterImpl(name, vc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 9bde6e0..6b5d9fe 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -82,16 +82,6 @@ drill.exec: {
}
}
},
- metrics : {
- context: "drillbit",
- jmx: {
- enabled : true
- },
- log: {
- enabled : false,
- interval : 60
- }
- },
zk: {
connect: "localhost:2181",
root: "drill",
@@ -164,7 +154,6 @@ drill.exec: {
}
},
memory: {
- top.max: 1000000000000,
operator: {
max: 20000000000,
initial: 10000000
@@ -175,7 +164,6 @@ drill.exec: {
}
},
debug: {
- error_on_leak: true,
return_error_for_failure_in_cancelled_fragments: false
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
index 67cabff..19613fe 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
@@ -17,21 +17,24 @@
*/
package org.apache.drill;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import org.apache.drill.exec.testing.Controls;
import org.apache.drill.exec.testing.ControlsInjectionUtil;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import org.junit.Ignore;
import org.junit.Test;
/**
* Run several tpch queries and inject an OutOfMemoryException in ScanBatch that will cause an OUT_OF_MEMORY outcome to
* be propagated downstream. Make sure the proper "memory error" message is sent to the client.
*/
+@Ignore("Need to add exception site in memory layer that doesn't depend on other Drill code.")
public class TestAllocationException extends BaseTestQuery {
private static final String SINGLE_MODE = "ALTER SESSION SET `planner.disable_exchanges` = true";
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java b/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
index b270a8b..9d78024 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
@@ -18,6 +18,7 @@
package org.apache.drill;
import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import org.apache.drill.exec.testing.ControlsInjectionUtil;
@@ -44,7 +45,7 @@ public class TestOutOfMemoryOutcome extends BaseTestQuery{
+ "\"desc\":\"" + "next-allocate" + "\","
+ "\"nSkip\":0,"
+ "\"nFire\":1,"
- + "\"exceptionClass\":\"" + "org.apache.drill.exec.memory.OutOfMemoryException" + "\""
+ + "\"exceptionClass\":\"" + OutOfMemoryException.class.getName() + "\""
+ "}]}";
ControlsInjectionUtil.setControls(client, controlsString);
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
index 2265347..2311a42 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
@@ -69,7 +69,7 @@ public class TestEvaluationVisitor {
ValueVectorWriteExpression e2 = new ValueVectorWriteExpression(outId, e, true);
v.addExpr(e2, g.getRoot());
- System.out.println(g.generateAndGet());
+ logger.debug(g.generateAndGet());
}
private LogicalExpression getExpr(String expr) throws Exception{
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/test/java/org/apache/drill/exec/test/Drill2130JavaExecHamcrestConfigurationTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/test/Drill2130JavaExecHamcrestConfigurationTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/test/Drill2130JavaExecHamcrestConfigurationTest.java
index 03d858c..9f1b7dc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/test/Drill2130JavaExecHamcrestConfigurationTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/test/Drill2130JavaExecHamcrestConfigurationTest.java
@@ -17,14 +17,17 @@
*/
package org.apache.drill.exec.test;
-import org.junit.Test;
+import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
-import static org.hamcrest.CoreMatchers.equalTo;
+
+import org.junit.Test;
public class Drill2130JavaExecHamcrestConfigurationTest {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
+ .getLogger(Drill2130JavaExecHamcrestConfigurationTest.class);
@SuppressWarnings("unused")
private org.hamcrest.MatcherAssert forCompileTimeCheckForNewEnoughHamcrest;
@@ -36,9 +39,8 @@ public class Drill2130JavaExecHamcrestConfigurationTest {
catch ( NoSuchMethodError e ) {
fail( "Class search path seems broken re new JUnit and old Hamcrest."
+ " Got NoSuchMethodError; e: " + e );
- }
- catch ( AssertionError e ) {
- System.out.println( "Class path seems fine re new JUnit vs. old Hamcrest."
+ } catch (AssertionError e) {
+ logger.debug("Class path seems fine re new JUnit vs. old Hamcrest."
+ " (Got AssertionError, not NoSuchMethodError.)" );
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
index eb5cc49..cdb703f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
@@ -17,11 +17,13 @@
*/
package org.apache.drill.exec.work.fragment;
+import static org.junit.Assert.fail;
+
import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.testing.ControlsInjectionUtil;
-import static org.junit.Assert.fail;
import org.junit.Test;
/**
@@ -44,7 +46,7 @@ public class TestFragmentExecutorCancel extends BaseTestQuery {
+ "\"desc\":\"" + "next-allocate" + "\","
+ "\"nSkip\":0,"
+ "\"nFire\":1,"
- + "\"exceptionClass\":\"" + "org.apache.drill.exec.memory.OutOfMemoryException" + "\""
+ + "\"exceptionClass\":\"" + OutOfMemoryException.class.getName() + "\""
+ "}]}";
ControlsInjectionUtil.setControls(client, controlsString);
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/java-exec/src/test/resources/drill-oom-xsort.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/drill-oom-xsort.conf b/exec/java-exec/src/test/resources/drill-oom-xsort.conf
index 2b5ba80..d91bdbb 100644
--- a/exec/java-exec/src/test/resources/drill-oom-xsort.conf
+++ b/exec/java-exec/src/test/resources/drill-oom-xsort.conf
@@ -6,7 +6,6 @@ drill: {
classpath.scanning: {
packages += "org.apache.drill.exec.expr.fn.impl"
},
- exec: {
memory: {
fragment: {
max: 50000000,
@@ -17,5 +16,4 @@ drill: {
initial: 2000000
}
}
- }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml
index 0b94c03..cae5538 100644
--- a/exec/jdbc/pom.xml
+++ b/exec/jdbc/pom.xml
@@ -32,6 +32,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-logical</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
index fc7b667..ab73a14 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
@@ -51,8 +51,8 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.memory.RootAllocatorFactory;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.server.Drillbit;
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java
index 4878251..a2083d3 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java
@@ -56,19 +56,14 @@ import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.jdbc.AlreadyClosedSqlException;
-import org.apache.drill.jdbc.DrillConnection;
import org.apache.drill.jdbc.DrillResultSet;
import org.apache.drill.jdbc.ExecutionCanceledSqlException;
import org.apache.drill.jdbc.SchemaChangeListener;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import org.slf4j.Logger;
-
import com.google.common.collect.Queues;
@@ -1935,7 +1930,8 @@ class DrillResultSetImpl extends AvaticaResultSet implements DrillResultSet {
// ResultsListener:
static class ResultsListener implements UserResultsListener {
- private static final Logger logger = getLogger( ResultsListener.class );
+ private static final org.slf4j.Logger logger =
+ org.slf4j.LoggerFactory.getLogger(ResultsListener.class);
private static volatile int nextInstanceId = 1;
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java
index 8574d28..a417140 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java
@@ -17,7 +17,12 @@
*/
package org.apache.drill.jdbc;
-import static org.junit.Assert.*;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.core.StringContains.containsString;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.ResultSet;
@@ -31,12 +36,6 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.core.StringContains.containsString;
-
-
public class DrillResultSetTest extends DrillTest {
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Hook.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Hook.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Hook.java
new file mode 100644
index 0000000..99c75cf
--- /dev/null
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Hook.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc.test;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import com.google.common.base.Function;
+
+public enum Hook {
+ /** Called with the logical plan. */
+ LOGICAL_PLAN;
+
+ private final List<Function<Object, Object>> handlers =
+ new CopyOnWriteArrayList<>();
+
+ public Closeable add(final Function handler) {
+ handlers.add(handler);
+ return new Closeable() {
+ public void close() {
+ remove(handler);
+ }
+ };
+ }
+
+ /** Removes a handler from this Hook. */
+ private boolean remove(Function handler) {
+ return handlers.remove(handler);
+ }
+
+ /** Runs all handlers registered for this Hook, with the given argument. */
+ public void run(Object arg) {
+ for (Function<Object, Object> handler : handlers) {
+ handler.apply(arg);
+ }
+ }
+
+ /** Removes a Hook after use. */
+ public interface Closeable extends AutoCloseable {
+ void close(); // override, removing "throws"
+ }
+}
+
+// End Hook.java
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
index b161e20..3f8514e 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
@@ -31,11 +31,9 @@ import java.util.SortedSet;
import java.util.TreeSet;
import org.apache.calcite.linq4j.Ord;
-
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.common.util.Hook;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory;
import org.apache.drill.jdbc.ConnectionFactory;
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java
index 7588b2c..a15e348 100644
--- a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BoundsChecking.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.memory;
public class BoundsChecking {
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index 1684970..1e96238 100644
--- a/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -37,8 +37,10 @@ public class TopLevelAllocator implements BufferAllocator {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopLevelAllocator.class);
public static final String CHILD_BUFFER_INJECTION_SITE = "child.buffer";
- public static final String TOP_LEVEL_MAX_ALLOC = "drill.exec.memory.top.max";
- public static final String ERROR_ON_MEMORY_LEAK = "drill.exec.debug.error_on_leak";
+ private static final PooledByteBufAllocatorL ALLOCATOR = new PooledByteBufAllocatorL(DrillMetrics.getInstance());
+
+ public static final String TOP_LEVEL_MAX_ALLOC = "drill.memory.top.max";
+ public static final String ERROR_ON_MEMORY_LEAK = "drill.memory.debug.error_on_leak";
public static long MAXIMUM_DIRECT_MEMORY;
@@ -52,7 +54,7 @@ public class TopLevelAllocator implements BufferAllocator {
private TopLevelAllocator(DrillConfig config, long maximumAllocation, boolean errorOnLeak){
MAXIMUM_DIRECT_MEMORY = maximumAllocation;
- innerAllocator = new PooledByteBufAllocatorL(DrillMetrics.getInstance());
+ innerAllocator = ALLOCATOR;
this.config=(config!=null) ? config : DrillConfig.create();
this.errorOnLeak = errorOnLeak;
this.acct = new AccountorImpl(config, errorOnLeak, null, null, maximumAllocation, 0, true);
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/exec/memory/impl/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/memory/impl/src/main/resources/drill-module.conf b/exec/memory/impl/src/main/resources/drill-module.conf
new file mode 100644
index 0000000..593ef8e
--- /dev/null
+++ b/exec/memory/impl/src/main/resources/drill-module.conf
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file tells Drill to consider this module when class path scanning.
+// This file can also include any supplementary configuration information.
+// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+drill: {
+ memory: {
+ debug.error_on_leak: true,
+ top.max: 1000000000000
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/logical/pom.xml
----------------------------------------------------------------------
diff --git a/logical/pom.xml b/logical/pom.xml
index f592657..fbae5bc 100644
--- a/logical/pom.xml
+++ b/logical/pom.xml
@@ -40,6 +40,13 @@
<artifactId>drill-common</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
<dependency>
<!-- add as provided scope so that we can compile TestTools. Should only be ever used in a test scenario where someone else is bringing JUnit in. -->
@@ -116,20 +123,27 @@
<build>
<plugins>
- <plugin>
+ <plugin> <!-- classpath scanning -->
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+ </dependencies>
<executions>
<execution>
<phase>process-classes</phase>
- <goals>
- <goal>java</goal>
- </goals>
+ <goals><goal>java</goal></goals>
</execution>
</executions>
<configuration>
<mainClass>org.apache.drill.common.scanner.BuildTimeScan</mainClass>
+ <includePluginDependencies>true</includePluginDependencies>
<arguments>
<argument>${project.build.outputDirectory}</argument>
</arguments>
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java b/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
index 50de963..d65520f 100644
--- a/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
+++ b/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
@@ -34,7 +34,18 @@ public abstract class FormatPluginConfigBase implements FormatPluginConfig{
*/
public static Set<Class<? extends FormatPluginConfig>> getSubTypes(final ScanResult classpathScan) {
final Set<Class<? extends FormatPluginConfig>> pluginClasses = classpathScan.getImplementations(FormatPluginConfig.class);
- logger.debug("Found {} format plugin configuration classes: {}.", pluginClasses.size(), pluginClasses);
+ if (logger.isDebugEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Found ");
+ sb.append(pluginClasses.size());
+ sb.append("format plugin configuration classes:\n");
+ for (Class<?> c : pluginClasses) {
+ sb.append('\t');
+ sb.append(c.getName());
+ sb.append('\n');
+ }
+ logger.debug(sb.toString());
+ }
return pluginClasses;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/logical/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/logical/src/main/resources/drill-module.conf b/logical/src/main/resources/drill-module.conf
new file mode 100644
index 0000000..b6cb266
--- /dev/null
+++ b/logical/src/main/resources/drill-module.conf
@@ -0,0 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http:// www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file tells Drill to consider this module when class path scanning.
+// This file can also include any supplementary configuration information.
+// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
----------------------------------------------------------------------
diff --git a/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
index 7cf1477..c8f6219 100644
--- a/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
+++ b/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
@@ -29,6 +29,7 @@ import org.apache.drill.test.DrillTest;
import org.junit.Test;
public class TreeTest extends DrillTest {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TreeTest.class);
@Test
public void testIfWithCase() throws Exception{
@@ -103,11 +104,11 @@ public class TreeTest extends DrillTest {
* @throws IOException
*/
private void testExpressionParsing(String expr) throws RecognitionException, IOException{
- System.out.println("-----" + expr + "-----");
+ logger.debug("-----" + expr + "-----");
LogicalExpression e = parseExpression(expr);
String newStringExpr = serializeExpression(e);
- System.out.println(newStringExpr);
+ logger.debug(newStringExpr);
LogicalExpression e2 = parseExpression(newStringExpr);
//Assert.assertEquals(e, e2);
http://git-wip-us.apache.org/repos/asf/drill/blob/494492f1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1dde3ad..ae6f402 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1914,6 +1914,7 @@
<module>tools</module>
<module>protocol</module>
<module>common</module>
+ <module>logical</module>
<module>exec</module>
<module>distribution</module>
</modules>
[09/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/EmptyValuePopulator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/EmptyValuePopulator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/EmptyValuePopulator.java
deleted file mode 100644
index fa1de66..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/EmptyValuePopulator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import com.google.common.base.Preconditions;
-import org.apache.drill.exec.vector.UInt4Vector;
-
-/**
- * A helper class that is used to track and populate empty values in repeated value vectors.
- */
-public class EmptyValuePopulator {
- private final UInt4Vector offsets;
-
- public EmptyValuePopulator(UInt4Vector offsets) {
- this.offsets = Preconditions.checkNotNull(offsets, "offsets cannot be null");
- }
-
- /**
- * Marks all values since the last set as empty. The last set value is obtained from underlying offsets vector.
- *
- * @param lastIndex the last index (inclusive) in the offsets vector until which empty population takes place
- * @throws java.lang.IndexOutOfBoundsException if lastIndex is negative or greater than offsets capacity.
- */
- public void populate(int lastIndex) {
- if (lastIndex < 0) {
- throw new IndexOutOfBoundsException("index cannot be negative");
- }
- final UInt4Vector.Accessor accessor = offsets.getAccessor();
- final UInt4Vector.Mutator mutator = offsets.getMutator();
- final int lastSet = Math.max(accessor.getValueCount() - 1, 0);
- final int previousEnd = accessor.get(lastSet);//0 ? 0 : accessor.get(lastSet);
- for (int i = lastSet; i < lastIndex; i++) {
- mutator.setSafe(i + 1, previousEnd);
- }
- mutator.setValueCount(lastIndex+1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
deleted file mode 100644
index 468a7f3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.record.TypedFieldId;
-import org.apache.drill.exec.vector.ValueVector;
-
-public class FieldIdUtil {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FieldIdUtil.class);
- public static TypedFieldId getFieldIdIfMatches(ValueVector vector, TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
- if (seg == null) {
- if (addToBreadCrumb) {
- builder.intermediateType(vector.getField().getType());
- }
- return builder.finalType(vector.getField().getType()).build();
- }
-
- if (seg.isArray()) {
- if (seg.isLastPath()) {
- MajorType type;
- if (vector instanceof AbstractContainerVector) {
- type = ((AbstractContainerVector) vector).getLastPathType();
- } else if (vector instanceof ListVector) {
- type = ((ListVector) vector).getDataVector().getField().getType();
- builder.listVector();
- } else {
- throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
- }
- builder //
- .withIndex() //
- .finalType(type);
-
- // remainder starts with the 1st array segment in SchemaPath.
- // only set remainder when it's the only array segment.
- if (addToBreadCrumb) {
- addToBreadCrumb = false;
- builder.remainder(seg);
- }
- return builder.build();
- } else {
- if (addToBreadCrumb) {
- addToBreadCrumb = false;
- builder.remainder(seg);
- }
- }
- } else {
- if (vector instanceof ListVector) {
- return null;
- }
- }
-
- ValueVector v;
- if (vector instanceof AbstractContainerVector) {
- VectorWithOrdinal vord = ((AbstractContainerVector) vector).getChildVectorWithOrdinal(seg.isArray() ? null : seg.getNameSegment().getPath());
- if (vord == null) {
- return null;
- }
- v = vord.vector;
- if (addToBreadCrumb) {
- builder.intermediateType(v.getField().getType());
- builder.addId(vord.ordinal);
- }
- } else if (vector instanceof ListVector) {
- v = ((ListVector) vector).getDataVector();
- } else {
- throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
- }
-
- if (v instanceof AbstractContainerVector) {
- // we're looking for a multi path.
- AbstractContainerVector c = (AbstractContainerVector) v;
- return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
- } else if(v instanceof ListVector) {
- ListVector list = (ListVector) v;
- return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
- } else if (v instanceof UnionVector) {
- return ((UnionVector) v).getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
- } else {
- if (seg.isNamed()) {
- if(addToBreadCrumb) {
- builder.intermediateType(v.getField().getType());
- }
- builder.finalType(v.getField().getType());
- } else {
- builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
- }
-
- if (seg.isLastPath()) {
- return builder.build();
- } else {
- PathSegment child = seg.getChild();
- if (child.isLastPath() && child.isArray()) {
- if (addToBreadCrumb) {
- builder.remainder(child);
- }
- builder.withIndex();
- builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
- return builder.build();
- } else {
- logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong.");
- return null;
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
deleted file mode 100644
index e5d9443..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
+++ /dev/null
@@ -1,327 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.vector.complex;
-
-import com.google.common.collect.ObjectArrays;
-import io.netty.buffer.DrillBuf;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.record.TypedFieldId;
-import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.drill.exec.vector.AddOrGetResult;
-import org.apache.drill.exec.vector.UInt1Vector;
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VectorDescriptor;
-import org.apache.drill.exec.vector.ZeroVector;
-import org.apache.drill.exec.vector.complex.impl.ComplexCopier;
-import org.apache.drill.exec.vector.complex.impl.UnionListReader;
-import org.apache.drill.exec.vector.complex.impl.UnionListWriter;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.FieldWriter;
-
-import java.util.List;
-
-public class ListVector extends BaseRepeatedValueVector {
-
- private UInt4Vector offsets;
- private final UInt1Vector bits;
- private Mutator mutator = new Mutator();
- private Accessor accessor = new Accessor();
- private UnionListWriter writer;
- private UnionListReader reader;
- private CallBack callBack;
-
- public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- super(field, allocator);
- this.bits = new UInt1Vector(MaterializedField.create("$bits$", Types.required(MinorType.UINT1)), allocator);
- offsets = getOffsetVector();
- this.field.addChild(getDataVector().getField());
- this.writer = new UnionListWriter(this);
- this.reader = new UnionListReader(this);
- this.callBack = callBack;
- }
-
- public UnionListWriter getWriter() {
- return writer;
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
- super.allocateNewSafe();
- }
-
- public void transferTo(ListVector target) {
- offsets.makeTransferPair(target.offsets).transfer();
- bits.makeTransferPair(target.bits).transfer();
- if (target.getDataVector() instanceof ZeroVector) {
- target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
- }
- getDataVector().makeTransferPair(target.getDataVector()).transfer();
- }
-
- public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
- copyFrom(inIndex, outIndex, from);
- }
-
- public void copyFrom(int inIndex, int outIndex, ListVector from) {
- FieldReader in = from.getReader();
- in.setPosition(inIndex);
- FieldWriter out = getWriter();
- out.setPosition(outIndex);
- ComplexCopier.copy(in, out);
- }
-
- @Override
- public ValueVector getDataVector() {
- return vector;
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- return new TransferImpl(field.withPath(ref));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector target) {
- return new TransferImpl((ListVector) target);
- }
-
- private class TransferImpl implements TransferPair {
-
- ListVector to;
-
- public TransferImpl(MaterializedField field) {
- to = new ListVector(field, allocator, null);
- to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
- }
-
- public TransferImpl(ListVector to) {
- this.to = to;
- to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
- }
-
- @Override
- public void transfer() {
- transferTo(to);
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- to.allocateNew();
- for (int i = 0; i < length; i++) {
- copyValueSafe(startIndex + i, i);
- }
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- this.to.copyFrom(from, to, ListVector.this);
- }
- }
-
- @Override
- public Accessor getAccessor() {
- return accessor;
- }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- @Override
- public FieldReader getReader() {
- return reader;
- }
-
- @Override
- public boolean allocateNewSafe() {
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- if (!offsets.allocateNewSafe()) {
- return false;
- }
- success = vector.allocateNewSafe();
- success = success && bits.allocateNewSafe();
- } finally {
- if (!success) {
- clear();
- }
- }
- if (success) {
- offsets.zeroVector();
- bits.zeroVector();
- }
- return success;
- }
-
- @Override
- protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
- return getField().getAsBuilder()
- .setValueCount(getAccessor().getValueCount())
- .setBufferLength(getBufferSize())
- .addChild(offsets.getMetadata())
- .addChild(bits.getMetadata())
- .addChild(vector.getMetadata());
- }
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- AddOrGetResult<T> result = super.addOrGetVector(descriptor);
- reader = new UnionListReader(this);
- return result;
- }
-
- @Override
- public int getBufferSize() {
- if (getAccessor().getValueCount() == 0) {
- return 0;
- }
- return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize();
- }
-
- @Override
- public void clear() {
- offsets.clear();
- vector.clear();
- bits.clear();
- lastSet = 0;
- super.clear();
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false),
- vector.getBuffers(false), DrillBuf.class), DrillBuf.class);
- if (clear) {
- for (DrillBuf buffer:buffers) {
- buffer.retain();
- }
- clear();
- }
- return buffers;
- }
-
- @Override
- public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
- final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
- offsets.load(offsetMetadata, buffer);
-
- final int offsetLength = offsetMetadata.getBufferLength();
- final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
- final int bitLength = bitMetadata.getBufferLength();
- bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
-
- final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
- if (getDataVector() == DEFAULT_DATA_VECTOR) {
- addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
- }
-
- final int vectorLength = vectorMetadata.getBufferLength();
- vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
- }
-
- public UnionVector promoteToUnion() {
- MaterializedField newField = MaterializedField.create(getField().getPath(), Types.optional(MinorType.UNION));
- UnionVector vector = new UnionVector(newField, allocator, null);
- replaceDataVector(vector);
- reader = new UnionListReader(this);
- return vector;
- }
-
- public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
- return FieldIdUtil.getFieldIdIfMatches(this, builder, addToBreadCrumb, seg);
- }
-
- private int lastSet;
-
- public class Accessor extends BaseRepeatedAccessor {
-
- @Override
- public Object getObject(int index) {
- if (isNull(index)) {
- return null;
- }
- final List<Object> vals = new JsonStringArrayList<>();
- final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
- final int start = offsetsAccessor.get(index);
- final int end = offsetsAccessor.get(index + 1);
- final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor();
- for(int i = start; i < end; i++) {
- vals.add(valuesAccessor.getObject(i));
- }
- return vals;
- }
-
- @Override
- public boolean isNull(int index) {
- return bits.getAccessor().get(index) == 0;
- }
- }
-
- public class Mutator extends BaseRepeatedMutator {
- public void setNotNull(int index) {
- bits.getMutator().setSafe(index, 1);
- lastSet = index + 1;
- }
-
- @Override
- public void startNewValue(int index) {
- for (int i = lastSet; i <= index; i++) {
- offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
- }
- setNotNull(index);
- lastSet = index + 1;
- }
-
- @Override
- public void setValueCount(int valueCount) {
- // TODO: populate offset end points
- if (valueCount == 0) {
- offsets.getMutator().setValueCount(0);
- } else {
- for (int i = lastSet; i < valueCount; i++) {
- offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
- }
- offsets.getMutator().setValueCount(valueCount + 1);
- }
- final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
- vector.getMutator().setValueCount(childValueCount);
- bits.getMutator().setValueCount(valueCount);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
deleted file mode 100644
index 048358c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
-
-import io.netty.buffer.DrillBuf;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import javax.annotation.Nullable;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.holders.ComplexHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.exec.util.JsonStringHashMap;
-import org.apache.drill.exec.vector.BaseValueVector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier;
-import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-import com.google.common.base.Preconditions;
-
-public class MapVector extends AbstractMapVector {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
-
- public final static MajorType TYPE = Types.required(MinorType.MAP);
-
- private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
- private final Accessor accessor = new Accessor();
- private final Mutator mutator = new Mutator();
- private int valueCount;
-
- public MapVector(String path, BufferAllocator allocator, CallBack callBack){
- this(MaterializedField.create(SchemaPath.getSimplePath(path), TYPE), allocator, callBack);
- }
-
- public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- super(field, allocator, callBack);
- }
-
- @Override
- public FieldReader getReader() {
- //return new SingleMapReaderImpl(MapVector.this);
- return reader;
- }
-
- transient private MapTransferPair ephPair;
- transient private MapSingleCopier ephPair2;
-
- public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
- if(ephPair == null || ephPair.from != from) {
- ephPair = (MapTransferPair) from.makeTransferPair(this);
- }
- ephPair.copyValueSafe(fromIndex, thisIndex);
- }
-
- public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
- if(ephPair2 == null || ephPair2.from != from) {
- ephPair2 = from.makeSingularCopier(this);
- }
- ephPair2.copySafe(fromSubIndex, thisIndex);
- }
-
- @Override
- protected boolean supportsDirectRead() {
- return true;
- }
-
- public Iterator<String> fieldNameIterator() {
- return getChildFieldNames().iterator();
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- v.setInitialCapacity(numRecords);
- }
- }
-
- @Override
- public int getBufferSize() {
- if (valueCount == 0 || size() == 0) {
- return 0;
- }
- long buffer = 0;
- for (final ValueVector v : (Iterable<ValueVector>)this) {
- buffer += v.getBufferSize();
- }
-
- return (int) buffer;
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- long bufferSize = 0;
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSizeFor(valueCount);
- }
-
- return (int) bufferSize;
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- int expectedSize = getBufferSize();
- int actualSize = super.getBufferSize();
-
- Preconditions.checkArgument(expectedSize == actualSize);
- return super.getBuffers(clear);
- }
-
- @Override
- public TransferPair getTransferPair() {
- return new MapTransferPair(this, getField().getPath());
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new MapTransferPair(this, (MapVector) to);
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- return new MapTransferPair(this, ref);
- }
-
- protected static class MapTransferPair implements TransferPair{
- private final TransferPair[] pairs;
- private final MapVector from;
- private final MapVector to;
-
- public MapTransferPair(MapVector from, SchemaPath path) {
- this(from, new MapVector(MaterializedField.create(path, TYPE), from.allocator, from.callBack), false);
- }
-
- public MapTransferPair(MapVector from, MapVector to) {
- this(from, to, true);
- }
-
- protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
- this.from = from;
- this.to = to;
- this.pairs = new TransferPair[from.size()];
- this.to.ephPair = null;
- this.to.ephPair2 = null;
-
- int i = 0;
- ValueVector vector;
- for (String child:from.getChildFieldNames()) {
- int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
- //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
- // the child fields may be nested fields of the top level child. For example if the structure
- // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
- // But the children member of a Materialized field is a HashSet. If the fields are added in the
- // children HashSet, and the hashCode of the Materialized field includes the hash code of the
- // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
- // (This is similar to what happens in ScanBatch where the children cannot be added till they are
- // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
- // include the hashCode of the children but is based only on MaterializedField$key.
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (allocate && to.size() != preSize) {
- newVector.allocateNew();
- }
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
- @Override
- public void transfer() {
- for (final TransferPair p : pairs) {
- p.transfer();
- }
- to.valueCount = from.valueCount;
- from.clear();
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(from, to);
- }
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- for (TransferPair p : pairs) {
- p.splitAndTransfer(startIndex, length);
- }
- to.getMutator().setValueCount(length);
- }
- }
-
- @Override
- public int getValueCapacity() {
- if (size() == 0) {
- return 0;
- }
-
- final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
- @Override
- public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
- return Ints.compare(
- Preconditions.checkNotNull(left).getValueCapacity(),
- Preconditions.checkNotNull(right).getValueCapacity()
- );
- }
- };
-
- return natural.min(getChildren()).getValueCapacity();
- }
-
- @Override
- public Accessor getAccessor() {
- return accessor;
- }
-
- @Override
- public void load(SerializedField metadata, DrillBuf buf) {
- final List<SerializedField> fields = metadata.getChildList();
- valueCount = metadata.getValueCount();
-
- int bufOffset = 0;
- for (final SerializedField child : fields) {
- final MaterializedField fieldDef = MaterializedField.create(child);
-
- ValueVector vector = getChild(fieldDef.getLastName());
- if (vector == null) {
- // if we arrive here, we didn't have a matching vector.
- vector = TypeHelper.getNewVector(fieldDef, allocator);
- putChild(fieldDef.getLastName(), vector);
- }
- if (child.getValueCount() == 0) {
- vector.clear();
- } else {
- vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
- }
- bufOffset += child.getBufferLength();
- }
-
- assert bufOffset == buf.capacity();
- }
-
- @Override
- public SerializedField getMetadata() {
- SerializedField.Builder b = getField() //
- .getAsBuilder() //
- .setBufferLength(getBufferSize()) //
- .setValueCount(valueCount);
-
-
- for(ValueVector v : getChildren()) {
- b.addChild(v.getMetadata());
- }
- return b.build();
- }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- public class Accessor extends BaseValueVector.BaseAccessor {
-
- @Override
- public Object getObject(int index) {
- Map<String, Object> vv = new JsonStringHashMap<>();
- for (String child:getChildFieldNames()) {
- ValueVector v = getChild(child);
- // TODO(DRILL-4001): Resolve this hack:
- // The index/value count check in the following if statement is a hack
- // to work around the current fact that RecordBatchLoader.load and
- // MapVector.load leave child vectors with a length of zero (as opposed
- // to matching the lengths of siblings and the parent map vector)
- // because they don't remove (or set the lengths of) vectors from
- // previous batches that aren't in the current batch.
- if (v != null && index < v.getAccessor().getValueCount()) {
- Object value = v.getAccessor().getObject(index);
- if (value != null) {
- vv.put(child, value);
- }
- }
- }
- return vv;
- }
-
- public void get(int index, ComplexHolder holder) {
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- @Override
- public int getValueCount() {
- return valueCount;
- }
- }
-
- public ValueVector getVectorById(int id) {
- return getChildByOrdinal(id);
- }
-
- public class Mutator extends BaseValueVector.BaseMutator {
-
- @Override
- public void setValueCount(int valueCount) {
- for (final ValueVector v : getChildren()) {
- v.getMutator().setValueCount(valueCount);
- }
- MapVector.this.valueCount = valueCount;
- }
-
- @Override
- public void reset() { }
-
- @Override
- public void generateTestData(int values) { }
- }
-
- @Override
- public void clear() {
- for (final ValueVector v : getChildren()) {
- v.clear();
- }
- valueCount = 0;
- }
-
- @Override
- public void close() {
- final Collection<ValueVector> vectors = getChildren();
- for (final ValueVector v : vectors) {
- v.close();
- }
- vectors.clear();
- valueCount = 0;
-
- super.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/Positionable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/Positionable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/Positionable.java
deleted file mode 100644
index 6d86a64..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/Positionable.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-public interface Positionable {
- public void setPosition(int index);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
deleted file mode 100644
index a6967f3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-/**
- * A {@link org.apache.drill.exec.vector.ValueVector} mix-in that can be used in conjunction with
- * {@link RepeatedValueVector} subtypes.
- */
-public interface RepeatedFixedWidthVectorLike {
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount Number of separate repeating groupings.
- * @param innerValueCount Number of supported values in the vector.
- */
- void allocateNew(int valueCount, int innerValueCount);
-
- /**
- * Load the records in the provided buffer based on the given number of values.
- * @param valueCount Number of separate repeating groupings.
- * @param innerValueCount Number atomic values the buffer contains.
- * @param buf Incoming buffer.
- * @return The number of bytes of the buffer that were consumed.
- */
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
deleted file mode 100644
index 330eed4..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ /dev/null
@@ -1,429 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import io.netty.buffer.DrillBuf;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.holders.ComplexHolder;
-import org.apache.drill.exec.expr.holders.RepeatedListHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.drill.exec.vector.AddOrGetResult;
-import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VectorDescriptor;
-import org.apache.drill.exec.vector.complex.impl.NullReader;
-import org.apache.drill.exec.vector.complex.impl.RepeatedListReaderImpl;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-public class RepeatedListVector extends AbstractContainerVector
- implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
-
- public final static MajorType TYPE = Types.repeated(MinorType.LIST);
- private final RepeatedListReaderImpl reader = new RepeatedListReaderImpl(null, this);
- private final DelegateRepeatedVector delegate;
-
- protected static class DelegateRepeatedVector extends BaseRepeatedValueVector {
-
- private final RepeatedListAccessor accessor = new RepeatedListAccessor();
- private final RepeatedListMutator mutator = new RepeatedListMutator();
- private final EmptyValuePopulator emptyPopulator;
- private transient DelegateTransferPair ephPair;
-
- public class RepeatedListAccessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
-
- @Override
- public Object getObject(int index) {
- final List<Object> list = new JsonStringArrayList();
- final int start = offsets.getAccessor().get(index);
- final int until = offsets.getAccessor().get(index+1);
- for (int i = start; i < until; i++) {
- list.add(vector.getAccessor().getObject(i));
- }
- return list;
- }
-
- public void get(int index, RepeatedListHolder holder) {
- assert index <= getValueCapacity();
- holder.start = getOffsetVector().getAccessor().get(index);
- holder.end = getOffsetVector().getAccessor().get(index+1);
- }
-
- public void get(int index, ComplexHolder holder) {
- final FieldReader reader = getReader();
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- public void get(int index, int arrayIndex, ComplexHolder holder) {
- final RepeatedListHolder listHolder = new RepeatedListHolder();
- get(index, listHolder);
- int offset = listHolder.start + arrayIndex;
- if (offset >= listHolder.end) {
- holder.reader = NullReader.INSTANCE;
- } else {
- FieldReader r = getDataVector().getReader();
- r.setPosition(offset);
- holder.reader = r;
- }
- }
- }
-
- public class RepeatedListMutator extends BaseRepeatedValueVector.BaseRepeatedMutator {
-
- public int add(int index) {
- final int curEnd = getOffsetVector().getAccessor().get(index+1);
- getOffsetVector().getMutator().setSafe(index + 1, curEnd + 1);
- return curEnd;
- }
-
- @Override
- public void startNewValue(int index) {
- emptyPopulator.populate(index+1);
- super.startNewValue(index);
- }
-
- @Override
- public void setValueCount(int valueCount) {
- emptyPopulator.populate(valueCount);
- super.setValueCount(valueCount);
- }
- }
-
-
- public class DelegateTransferPair implements TransferPair {
- private final DelegateRepeatedVector target;
- private final TransferPair[] children;
-
- public DelegateTransferPair(DelegateRepeatedVector target) {
- this.target = Preconditions.checkNotNull(target);
- if (target.getDataVector() == DEFAULT_DATA_VECTOR) {
- target.addOrGetVector(VectorDescriptor.create(getDataVector().getField()));
- target.getDataVector().allocateNew();
- }
- this.children = new TransferPair[] {
- getOffsetVector().makeTransferPair(target.getOffsetVector()),
- getDataVector().makeTransferPair(target.getDataVector())
- };
- }
-
- @Override
- public void transfer() {
- for (TransferPair child:children) {
- child.transfer();
- }
- }
-
- @Override
- public ValueVector getTo() {
- return target;
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- target.allocateNew();
- for (int i = 0; i < length; i++) {
- copyValueSafe(startIndex + i, i);
- }
- }
-
- @Override
- public void copyValueSafe(int srcIndex, int destIndex) {
- final RepeatedListHolder holder = new RepeatedListHolder();
- getAccessor().get(srcIndex, holder);
- target.emptyPopulator.populate(destIndex+1);
- final TransferPair vectorTransfer = children[1];
- int newIndex = target.getOffsetVector().getAccessor().get(destIndex);
- //todo: make this a bulk copy.
- for (int i = holder.start; i < holder.end; i++, newIndex++) {
- vectorTransfer.copyValueSafe(i, newIndex);
- }
- target.getOffsetVector().getMutator().setSafe(destIndex + 1, newIndex);
- }
- }
-
- public DelegateRepeatedVector(SchemaPath path, BufferAllocator allocator) {
- this(MaterializedField.create(path, TYPE), allocator);
- }
-
- public DelegateRepeatedVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- emptyPopulator = new EmptyValuePopulator(getOffsetVector());
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
- if (!allocateNewSafe()) {
- throw new OutOfMemoryRuntimeException();
- }
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- return makeTransferPair(new DelegateRepeatedVector(ref, allocator));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector target) {
- return new DelegateTransferPair(DelegateRepeatedVector.class.cast(target));
- }
-
- @Override
- public RepeatedListAccessor getAccessor() {
- return accessor;
- }
-
- @Override
- public RepeatedListMutator getMutator() {
- return mutator;
- }
-
- @Override
- public FieldReader getReader() {
- throw new UnsupportedOperationException();
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) {
- if(ephPair == null || ephPair.target != from) {
- ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this));
- }
- ephPair.copyValueSafe(fromIndex, thisIndex);
- }
-
- }
-
- protected class RepeatedListTransferPair implements TransferPair {
- private final TransferPair delegate;
-
- public RepeatedListTransferPair(TransferPair delegate) {
- this.delegate = delegate;
- }
-
- public void transfer() {
- delegate.transfer();
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- delegate.splitAndTransfer(startIndex, length);
- }
-
- @Override
- public ValueVector getTo() {
- final DelegateRepeatedVector delegateVector = DelegateRepeatedVector.class.cast(delegate.getTo());
- return new RepeatedListVector(getField(), allocator, callBack, delegateVector);
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- delegate.copyValueSafe(from, to);
- }
- }
-
- public RepeatedListVector(SchemaPath path, BufferAllocator allocator, CallBack callBack) {
- this(MaterializedField.create(path, TYPE), allocator, callBack);
- }
-
- public RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- this(field, allocator, callBack, new DelegateRepeatedVector(field, allocator));
- }
-
- protected RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack, DelegateRepeatedVector delegate) {
- super(field, allocator, callBack);
- this.delegate = Preconditions.checkNotNull(delegate);
-
- final List<MaterializedField> children = Lists.newArrayList(field.getChildren());
- final int childSize = children.size();
- assert childSize < 3;
- final boolean hasChild = childSize > 0;
- if (hasChild) {
- // the last field is data field
- final MaterializedField child = children.get(childSize-1);
- addOrGetVector(VectorDescriptor.create(child));
- }
- }
-
-
- @Override
- public RepeatedListReaderImpl getReader() {
- return reader;
- }
-
- @Override
- public DelegateRepeatedVector.RepeatedListAccessor getAccessor() {
- return delegate.getAccessor();
- }
-
- @Override
- public DelegateRepeatedVector.RepeatedListMutator getMutator() {
- return delegate.getMutator();
- }
-
- @Override
- public UInt4Vector getOffsetVector() {
- return delegate.getOffsetVector();
- }
-
- @Override
- public ValueVector getDataVector() {
- return delegate.getDataVector();
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
- delegate.allocateNew();
- }
-
- @Override
- public boolean allocateNewSafe() {
- return delegate.allocateNewSafe();
- }
-
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- final AddOrGetResult<T> result = delegate.addOrGetVector(descriptor);
- if (result.isCreated() && callBack != null) {
- callBack.doWork();
- }
- return result;
- }
-
- @Override
- public int size() {
- return delegate.size();
- }
-
- @Override
- public int getBufferSize() {
- return delegate.getBufferSize();
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- return delegate.getBufferSizeFor(valueCount);
- }
-
- @Override
- public void close() {
- delegate.close();
- }
-
- @Override
- public void clear() {
- delegate.clear();
- }
-
- @Override
- public TransferPair getTransferPair() {
- return new RepeatedListTransferPair(delegate.getTransferPair());
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- return new RepeatedListTransferPair(delegate.getTransferPair(ref));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- final RepeatedListVector target = RepeatedListVector.class.cast(to);
- return new RepeatedListTransferPair(delegate.makeTransferPair(target.delegate));
- }
-
- @Override
- public int getValueCapacity() {
- return delegate.getValueCapacity();
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- return delegate.getBuffers(clear);
- }
-
-
- @Override
- public void load(SerializedField metadata, DrillBuf buf) {
- delegate.load(metadata, buf);
- }
-
- @Override
- public SerializedField getMetadata() {
- return delegate.getMetadata();
- }
-
- @Override
- public Iterator<ValueVector> iterator() {
- return delegate.iterator();
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- delegate.setInitialCapacity(numRecords);
- }
-
- /**
- * @deprecated
- * prefer using {@link #addOrGetVector(org.apache.drill.exec.vector.VectorDescriptor)} instead.
- */
- @Override
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
- final AddOrGetResult<T> result = addOrGetVector(VectorDescriptor.create(type));
- return result.getVector();
- }
-
- @Override
- public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
- if (name != null) {
- return null;
- }
- return typeify(delegate.getDataVector(), clazz);
- }
-
- @Override
- public void allocateNew(int valueCount, int innerValueCount) {
- clear();
- getOffsetVector().allocateNew(valueCount + 1);
- getMutator().reset();
- }
-
- @Override
- public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
- if (name != null) {
- return null;
- }
- return new VectorWithOrdinal(delegate.getDataVector(), 0);
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) {
- delegate.copyFromSafe(fromIndex, thisIndex, from.delegate);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
deleted file mode 100644
index 618e3a3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import io.netty.buffer.DrillBuf;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.holders.ComplexHolder;
-import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.record.TypedFieldId;
-import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.drill.exec.vector.AddOrGetResult;
-import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VectorDescriptor;
-import org.apache.drill.exec.vector.complex.impl.NullReader;
-import org.apache.drill.exec.vector.complex.impl.RepeatedMapReaderImpl;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-public class RepeatedMapVector extends AbstractMapVector
- implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
-
- public final static MajorType TYPE = MajorType.newBuilder().setMinorType(MinorType.MAP).setMode(DataMode.REPEATED).build();
-
- private final UInt4Vector offsets; // offsets to start of each record (considering record indices are 0-indexed)
- private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
- private final RepeatedMapAccessor accessor = new RepeatedMapAccessor();
- private final Mutator mutator = new Mutator();
- private final EmptyValuePopulator emptyPopulator;
-
- public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- super(field, allocator, callBack);
- this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator);
- this.emptyPopulator = new EmptyValuePopulator(offsets);
- }
-
- @Override
- public UInt4Vector getOffsetVector() {
- return offsets;
- }
-
- @Override
- public ValueVector getDataVector() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- offsets.setInitialCapacity(numRecords + 1);
- for(final ValueVector v : (Iterable<ValueVector>) this) {
- v.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
- }
- }
-
- @Override
- public RepeatedMapReaderImpl getReader() {
- return reader;
- }
-
- @Override
- public void allocateNew(int groupCount, int innerValueCount) {
- clear();
- try {
- offsets.allocateNew(groupCount + 1);
- for (ValueVector v : getChildren()) {
- AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
- }
- } catch (OutOfMemoryRuntimeException e){
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- public Iterator<String> fieldNameIterator() {
- return getChildFieldNames().iterator();
- }
-
- @Override
- public List<ValueVector> getPrimitiveVectors() {
- final List<ValueVector> primitiveVectors = super.getPrimitiveVectors();
- primitiveVectors.add(offsets);
- return primitiveVectors;
- }
-
- @Override
- public int getBufferSize() {
- if (getAccessor().getValueCount() == 0) {
- return 0;
- }
- long bufferSize = offsets.getBufferSize();
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSize();
- }
- return (int) bufferSize;
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- long bufferSize = 0;
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSizeFor(valueCount);
- }
-
- return (int) bufferSize;
- }
-
- @Override
- public void close() {
- offsets.close();
- super.close();
- }
-
- @Override
- public TransferPair getTransferPair() {
- return new RepeatedMapTransferPair(this, getField().getPath());
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new RepeatedMapTransferPair(this, (RepeatedMapVector)to);
- }
-
- MapSingleCopier makeSingularCopier(MapVector to) {
- return new MapSingleCopier(this, to);
- }
-
- protected static class MapSingleCopier {
- private final TransferPair[] pairs;
- public final RepeatedMapVector from;
-
- public MapSingleCopier(RepeatedMapVector from, MapVector to) {
- this.from = from;
- this.pairs = new TransferPair[from.size()];
-
- int i = 0;
- ValueVector vector;
- for (final String child:from.getChildFieldNames()) {
- int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (to.size() != preSize) {
- newVector.allocateNew();
- }
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
- public void copySafe(int fromSubIndex, int toIndex) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(fromSubIndex, toIndex);
- }
- }
- }
-
- @Override
- public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
- if (seg != null && seg.isArray() && !seg.isLastPath()) {
- if (addToBreadCrumb) {
- addToBreadCrumb = false;
- builder.remainder(seg);
- }
- // skip the first array segment as there is no corresponding child vector.
- seg = seg.getChild();
-
- // multi-level numbered access to a repeated map is not possible so return if the next part is also an array
- // segment.
- if (seg.isArray()) {
- return null;
- }
- }
-
- return super.getFieldIdIfMatches(builder, addToBreadCrumb, seg);
- }
-
- public TransferPair getTransferPairToSingleMap(FieldReference reference) {
- return new SingleMapTransferPair(this, reference);
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- return new RepeatedMapTransferPair(this, ref);
- }
-
- @Override
- public boolean allocateNewSafe() {
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- if (!offsets.allocateNewSafe()) {
- return false;
- }
- success = super.allocateNewSafe();
- } finally {
- if (!success) {
- clear();
- }
- }
- offsets.zeroVector();
- return success;
- }
-
- protected static class SingleMapTransferPair implements TransferPair {
- private final TransferPair[] pairs;
- private final RepeatedMapVector from;
- private final MapVector to;
- private static final MajorType MAP_TYPE = Types.required(MinorType.MAP);
-
- public SingleMapTransferPair(RepeatedMapVector from, SchemaPath path) {
- this(from, new MapVector(MaterializedField.create(path, MAP_TYPE), from.allocator, from.callBack), false);
- }
-
- public SingleMapTransferPair(RepeatedMapVector from, MapVector to) {
- this(from, to, true);
- }
-
- public SingleMapTransferPair(RepeatedMapVector from, MapVector to, boolean allocate) {
- this.from = from;
- this.to = to;
- this.pairs = new TransferPair[from.size()];
- int i = 0;
- ValueVector vector;
- for (final String child : from.getChildFieldNames()) {
- int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (allocate && to.size() != preSize) {
- newVector.allocateNew();
- }
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
-
- @Override
- public void transfer() {
- for (TransferPair p : pairs) {
- p.transfer();
- }
- to.getMutator().setValueCount(from.getAccessor().getValueCount());
- from.clear();
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(from, to);
- }
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- for (TransferPair p : pairs) {
- p.splitAndTransfer(startIndex, length);
- }
- to.getMutator().setValueCount(length);
- }
- }
-
- private static class RepeatedMapTransferPair implements TransferPair{
-
- private final TransferPair[] pairs;
- private final RepeatedMapVector to;
- private final RepeatedMapVector from;
-
- public RepeatedMapTransferPair(RepeatedMapVector from, SchemaPath path) {
- this(from, new RepeatedMapVector(MaterializedField.create(path, TYPE), from.allocator, from.callBack), false);
- }
-
- public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to) {
- this(from, to, true);
- }
-
- public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to, boolean allocate) {
- this.from = from;
- this.to = to;
- this.pairs = new TransferPair[from.size()];
- this.to.ephPair = null;
-
- int i = 0;
- ValueVector vector;
- for (final String child : from.getChildFieldNames()) {
- final int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
-
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (to.size() != preSize) {
- newVector.allocateNew();
- }
-
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
- @Override
- public void transfer() {
- from.offsets.transferTo(to.offsets);
- for (TransferPair p : pairs) {
- p.transfer();
- }
- from.clear();
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int srcIndex, int destIndex) {
- RepeatedMapHolder holder = new RepeatedMapHolder();
- from.getAccessor().get(srcIndex, holder);
- to.emptyPopulator.populate(destIndex + 1);
- int newIndex = to.offsets.getAccessor().get(destIndex);
- //todo: make these bulk copies
- for (int i = holder.start; i < holder.end; i++, newIndex++) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(i, newIndex);
- }
- }
- to.offsets.getMutator().setSafe(destIndex + 1, newIndex);
- }
-
- @Override
- public void splitAndTransfer(final int groupStart, final int groups) {
- final UInt4Vector.Accessor a = from.offsets.getAccessor();
- final UInt4Vector.Mutator m = to.offsets.getMutator();
-
- final int startPos = a.get(groupStart);
- final int endPos = a.get(groupStart + groups);
- final int valuesToCopy = endPos - startPos;
-
- to.offsets.clear();
- to.offsets.allocateNew(groups + 1);
-
- int normalizedPos;
- for (int i = 0; i < groups + 1; i++) {
- normalizedPos = a.get(groupStart + i) - startPos;
- m.set(i, normalizedPos);
- }
-
- m.setValueCount(groups + 1);
- to.emptyPopulator.populate(groups);
-
- for (final TransferPair p : pairs) {
- p.splitAndTransfer(startPos, valuesToCopy);
- }
- }
- }
-
-
- transient private RepeatedMapTransferPair ephPair;
-
- public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) {
- if (ephPair == null || ephPair.from != from) {
- ephPair = (RepeatedMapTransferPair) from.makeTransferPair(this);
- }
- ephPair.copyValueSafe(fromIndex, thisIndex);
- }
-
- @Override
- public int getValueCapacity() {
- return Math.max(offsets.getValueCapacity() - 1, 0);
- }
-
- @Override
- public RepeatedMapAccessor getAccessor() {
- return accessor;
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- final int expectedBufferSize = getBufferSize();
- final int actualBufferSize = super.getBufferSize();
-
- Preconditions.checkArgument(expectedBufferSize == actualBufferSize + offsets.getBufferSize());
- return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
- }
-
-
- @Override
- public void load(SerializedField metadata, DrillBuf buffer) {
- final List<SerializedField> children = metadata.getChildList();
-
- final SerializedField offsetField = children.get(0);
- offsets.load(offsetField, buffer);
- int bufOffset = offsetField.getBufferLength();
-
- for (int i = 1; i < children.size(); i++) {
- final SerializedField child = children.get(i);
- final MaterializedField fieldDef = MaterializedField.create(child);
- ValueVector vector = getChild(fieldDef.getLastName());
- if (vector == null) {
- // if we arrive here, we didn't have a matching vector.
- vector = TypeHelper.getNewVector(fieldDef, allocator);
- putChild(fieldDef.getLastName(), vector);
- }
- final int vectorLength = child.getBufferLength();
- vector.load(child, buffer.slice(bufOffset, vectorLength));
- bufOffset += vectorLength;
- }
-
- assert bufOffset == buffer.capacity();
- }
-
-
- @Override
- public SerializedField getMetadata() {
- SerializedField.Builder builder = getField() //
- .getAsBuilder() //
- .setBufferLength(getBufferSize()) //
- // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
- .setValueCount(accessor.getValueCount());
- builder.addChild(offsets.getMetadata());
- for (final ValueVector child : getChildren()) {
- builder.addChild(child.getMetadata());
- }
- return builder.build();
- }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- public class RepeatedMapAccessor implements RepeatedAccessor {
- @Override
- public Object getObject(int index) {
- final List<Object> list = new JsonStringArrayList<>();
- final int end = offsets.getAccessor().get(index+1);
- String fieldName;
- for (int i = offsets.getAccessor().get(index); i < end; i++) {
- final Map<String, Object> vv = Maps.newLinkedHashMap();
- for (final MaterializedField field : getField().getChildren()) {
- if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
- fieldName = field.getLastName();
- final Object value = getChild(fieldName).getAccessor().getObject(i);
- if (value != null) {
- vv.put(fieldName, value);
- }
- }
- }
- list.add(vv);
- }
- return list;
- }
-
- @Override
- public int getValueCount() {
- return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
- }
-
- @Override
- public int getInnerValueCount() {
- final int valueCount = getValueCount();
- if (valueCount == 0) {
- return 0;
- }
- return offsets.getAccessor().get(valueCount);
- }
-
- @Override
- public int getInnerValueCountAt(int index) {
- return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
- }
-
- @Override
- public boolean isEmpty(int index) {
- return false;
- }
-
- @Override
- public boolean isNull(int index) {
- return false;
- }
-
- public void get(int index, RepeatedMapHolder holder) {
- assert index < getValueCapacity() :
- String.format("Attempted to access index %d when value capacity is %d",
- index, getValueCapacity());
- final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
- holder.start = offsetsAccessor.get(index);
- holder.end = offsetsAccessor.get(index + 1);
- }
-
- public void get(int index, ComplexHolder holder) {
- final FieldReader reader = getReader();
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- public void get(int index, int arrayIndex, ComplexHolder holder) {
- final RepeatedMapHolder h = new RepeatedMapHolder();
- get(index, h);
- final int offset = h.start + arrayIndex;
-
- if (offset >= h.end) {
- holder.reader = NullReader.INSTANCE;
- } else {
- reader.setSinglePosition(index, arrayIndex);
- holder.reader = reader;
- }
- }
- }
-
- public class Mutator implements RepeatedMutator {
- @Override
- public void startNewValue(int index) {
- emptyPopulator.populate(index + 1);
- offsets.getMutator().setSafe(index + 1, offsets.getAccessor().get(index));
- }
-
- @Override
- public void setValueCount(int topLevelValueCount) {
- emptyPopulator.populate(topLevelValueCount);
- offsets.getMutator().setValueCount(topLevelValueCount == 0 ? 0 : topLevelValueCount + 1);
- int childValueCount = offsets.getAccessor().get(topLevelValueCount);
- for (final ValueVector v : getChildren()) {
- v.getMutator().setValueCount(childValueCount);
- }
- }
-
- @Override
- public void reset() {}
-
- @Override
- public void generateTestData(int values) {}
-
- public int add(int index) {
- final int prevEnd = offsets.getAccessor().get(index + 1);
- offsets.getMutator().setSafe(index + 1, prevEnd + 1);
- return prevEnd;
- }
- }
-
- @Override
- public void clear() {
- getMutator().reset();
-
- offsets.clear();
- for(final ValueVector vector : getChildren()) {
- vector.clear();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
deleted file mode 100644
index 0fba292..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.ContainerVectorLike;
-
-/**
- * An abstraction representing repeated value vectors.
- *
- * A repeated vector contains values that may either be flat or nested. A value consists of zero or more cells(inner values).
- * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector
- * uses the offset vector to determine the sequence of cells pertaining to an individual value.
- *
- */
-public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
-
- final static int DEFAULT_REPEAT_PER_RECORD = 5;
-
- /**
- * Returns the underlying offset vector or null if none exists.
- *
- * TODO(DRILL-2995): eliminate exposing low-level interfaces.
- */
- UInt4Vector getOffsetVector();
-
- /**
- * Returns the underlying data vector or null if none exists.
- */
- ValueVector getDataVector();
-
- @Override
- RepeatedAccessor getAccessor();
-
- @Override
- RepeatedMutator getMutator();
-
- interface RepeatedAccessor extends ValueVector.Accessor {
- /**
- * Returns total number of cells that vector contains.
- *
- * The result includes empty, null valued cells.
- */
- int getInnerValueCount();
-
-
- /**
- * Returns number of cells that the value at the given index contains.
- */
- int getInnerValueCountAt(int index);
-
- /**
- * Returns true if the value at the given index is empty, false otherwise.
- *
- * @param index value index
- */
- boolean isEmpty(int index);
- }
-
- interface RepeatedMutator extends ValueVector.Mutator {
- /**
- * Starts a new value that is a container of cells.
- *
- * @param index index of new value to start
- */
- void startNewValue(int index);
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
deleted file mode 100644
index 67954e7..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-public interface RepeatedVariableWidthVectorLike {
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param totalBytes Desired size of the underlying data buffer.
- * @param parentValueCount Number of separate repeating groupings.
- * @param childValueCount Number of supported values in the vector.
- */
- void allocateNew(int totalBytes, int parentValueCount, int childValueCount);
-
- /**
- * Provide the maximum amount of variable width bytes that can be stored int his vector.
- * @return
- */
- int getByteCapacity();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java
deleted file mode 100644
index f5ed3a0..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import java.util.Arrays;
-
-public class StateTool {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StateTool.class);
-
- public static <T extends Enum<?>> void check(T currentState, T... expectedStates) {
- for (T s : expectedStates) {
- if (s == currentState) {
- return;
- }
- }
- throw new IllegalArgumentException(String.format("Expected to be in one of these states %s but was actuall in state %s", Arrays.toString(expectedStates), currentState));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/VectorWithOrdinal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/VectorWithOrdinal.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/VectorWithOrdinal.java
deleted file mode 100644
index a0c8dd8..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/VectorWithOrdinal.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import org.apache.drill.exec.vector.ValueVector;
-
-public class VectorWithOrdinal {
- public final ValueVector vector;
- public final int ordinal;
-
- public VectorWithOrdinal(ValueVector v, int ordinal) {
- this.vector = v;
- this.ordinal = ordinal;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
deleted file mode 100644
index 05fe63f..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex.impl;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.holders.UnionHolder;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.drill.exec.vector.complex.writer.FieldWriter;
-
-
-abstract class AbstractBaseReader implements FieldReader{
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseReader.class);
-
- private int index;
-
- public AbstractBaseReader() {
- super();
- }
-
- public void setPosition(int index){
- this.index = index;
- }
-
- int idx(){
- return index;
- }
-
- @Override
- public void reset() {
- index = 0;
- }
-
- @Override
- public Iterator<String> iterator() {
- throw new IllegalStateException("The current reader doesn't support reading as a map.");
- }
-
- public MajorType getType(){
- throw new IllegalStateException("The current reader doesn't support getting type information.");
- }
-
- @Override
- public MaterializedField getField() {
- return MaterializedField.create("unknown", Types.LATE_BIND_TYPE);
- }
-
- @Override
- public boolean next() {
- throw new IllegalStateException("The current reader doesn't support getting next information.");
- }
-
- @Override
- public int size() {
- throw new IllegalStateException("The current reader doesn't support getting size information.");
- }
-
- @Override
- public void read(UnionHolder holder) {
- holder.reader = this;
- holder.isSet = this.isSet() ? 1 : 0;
- }
-
- @Override
- public void read(int index, UnionHolder holder) {
- throw new IllegalStateException("The current reader doesn't support reading union type");
- }
-
- @Override
- public void copyAsValue(UnionWriter writer) {
- throw new IllegalStateException("The current reader doesn't support reading union type");
- }
-
- @Override
- public void copyAsValue(ListWriter writer) {
- ComplexCopier.copy(this, (FieldWriter)writer);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseWriter.java
deleted file mode 100644
index bd16223..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseWriter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex.impl;
-
-import org.apache.drill.exec.vector.complex.writer.FieldWriter;
-
-
-abstract class AbstractBaseWriter implements FieldWriter {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseWriter.class);
-
- final FieldWriter parent;
- private int index;
-
- public AbstractBaseWriter(FieldWriter parent) {
- this.parent = parent;
- }
-
- @Override
- public String toString() {
- return super.toString() + "[index = " + index + ", parent = " + parent + "]";
- }
-
- @Override
- public FieldWriter getParent() {
- return parent;
- }
-
- public boolean isRoot() {
- return parent == null;
- }
-
- int idx() {
- return index;
- }
-
- @Override
- public void setPosition(int index) {
- this.index = index;
- }
-
- @Override
- public void end() {
- }
-}
[10/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
deleted file mode 100644
index eb5dbcd..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.DrillBuf;
-
-import java.util.Iterator;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class BaseValueVector implements ValueVector {
- private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
-
- public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
- public static final int INITIAL_VALUE_ALLOCATION = 4096;
-
- protected final BufferAllocator allocator;
- protected final MaterializedField field;
-
- protected BaseValueVector(MaterializedField field, BufferAllocator allocator) {
- this.field = Preconditions.checkNotNull(field, "field cannot be null");
- this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
- }
-
- @Override
- public String toString() {
- return super.toString() + "[field = " + field + ", ...]";
- }
-
- @Override
- public void clear() {
- getMutator().reset();
- }
-
- @Override
- public void close() {
- clear();
- }
-
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- public MaterializedField getField(FieldReference ref){
- return getField().withPath(ref);
- }
-
- @Override
- public TransferPair getTransferPair() {
- return getTransferPair(new FieldReference(getField().getPath()));
- }
-
- @Override
- public SerializedField getMetadata() {
- return getMetadataBuilder().build();
- }
-
- protected SerializedField.Builder getMetadataBuilder() {
- return getField().getAsBuilder()
- .setValueCount(getAccessor().getValueCount())
- .setBufferLength(getBufferSize());
- }
-
- public abstract static class BaseAccessor implements ValueVector.Accessor {
- protected BaseAccessor() { }
-
- @Override
- public boolean isNull(int index) {
- return false;
- }
- }
-
- public abstract static class BaseMutator implements ValueVector.Mutator {
- protected BaseMutator() { }
-
- @Override
- public void generateTestData(int values) {}
-
- //TODO: consider making mutator stateless(if possible) on another issue.
- public void reset() {}
- }
-
- @Override
- public Iterator<ValueVector> iterator() {
- return Iterators.emptyIterator();
- }
-
- public static boolean checkBufRefs(final ValueVector vv) {
- for(final DrillBuf buffer : vv.getBuffers(false)) {
- if (buffer.refCnt() <= 0) {
- throw new IllegalStateException("zero refcount");
- }
- }
-
- return true;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
deleted file mode 100644
index 67ba5c4..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ /dev/null
@@ -1,448 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import com.google.common.base.Preconditions;
-import io.netty.buffer.DrillBuf;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.exec.exception.OversizedAllocationException;
-import org.apache.drill.exec.expr.holders.BitHolder;
-import org.apache.drill.exec.expr.holders.NullableBitHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.complex.impl.BitReaderImpl;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-/**
- * Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start
- * of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0'
- * or '1'.
- */
-public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
-
- private final FieldReader reader = new BitReaderImpl(BitVector.this);
- private final Accessor accessor = new Accessor();
- private final Mutator mutator = new Mutator();
-
- private int valueCount;
- private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
- private int allocationMonitor = 0;
-
- public BitVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- @Override
- public FieldReader getReader() {
- return reader;
- }
-
- @Override
- public int getBufferSize() {
- return getSizeFromCount(valueCount);
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- return getSizeFromCount(valueCount);
- }
-
- private int getSizeFromCount(int valueCount) {
- return (int) Math.ceil(valueCount / 8.0);
- }
-
- @Override
- public int getValueCapacity() {
- return (int)Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L);
- }
-
- private int getByteIndex(int index) {
- return (int) Math.floor(index / 8.0);
- }
-
- @Override
- public void setInitialCapacity(final int valueCount) {
- allocationSizeInBytes = getSizeFromCount(valueCount);
- }
-
- @Override
- public void allocateNew() {
- if (!allocateNewSafe()) {
- throw new OutOfMemoryRuntimeException();
- }
- }
-
- @Override
- public boolean allocateNewSafe() {
- long curAllocationSize = allocationSizeInBytes;
- if (allocationMonitor > 10) {
- curAllocationSize = Math.max(8, allocationSizeInBytes / 2);
- allocationMonitor = 0;
- } else if (allocationMonitor < -2) {
- curAllocationSize = allocationSizeInBytes * 2L;
- allocationMonitor = 0;
- }
-
- try {
- allocateBytes(curAllocationSize);
- } catch (OutOfMemoryRuntimeException ex) {
- return false;
- }
- return true;
- }
-
- @Override
- public void reset() {
- valueCount = 0;
- allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
- allocationMonitor = 0;
- zeroVector();
- super.reset();
- }
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount
- * The number of values which can be contained within this vector.
- */
- @Override
- public void allocateNew(int valueCount) {
- final int size = getSizeFromCount(valueCount);
- allocateBytes(size);
- }
-
- private void allocateBytes(final long size) {
- if (size > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
- }
-
- final int curSize = (int) size;
- clear();
- data = allocator.buffer(curSize);
- zeroVector();
- allocationSizeInBytes = curSize;
- }
-
- /**
- * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
- */
- public void reAlloc() {
- final long newAllocationSize = allocationSizeInBytes * 2L;
- if (newAllocationSize > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
- }
-
- final int curSize = (int)newAllocationSize;
- final DrillBuf newBuf = allocator.buffer(curSize);
- newBuf.setZero(0, newBuf.capacity());
- newBuf.setBytes(0, data, 0, data.capacity());
- data.release();
- data = newBuf;
- allocationSizeInBytes = curSize;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void zeroVector() {
- data.setZero(0, data.capacity());
- }
-
- public void copyFrom(int inIndex, int outIndex, BitVector from) {
- this.mutator.set(outIndex, from.accessor.get(inIndex));
- }
-
- public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) {
- if (outIndex >= this.getValueCapacity()) {
- decrementAllocationMonitor();
- return false;
- }
- copyFrom(inIndex, outIndex, from);
- return true;
- }
-
- @Override
- public void load(SerializedField metadata, DrillBuf buffer) {
- Preconditions.checkArgument(this.field.matches(metadata), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
- final int valueCount = metadata.getValueCount();
- final int expectedLength = getSizeFromCount(valueCount);
- final int actualLength = metadata.getBufferLength();
- assert expectedLength == actualLength: "expected and actual buffer sizes do not match";
-
- clear();
- data = buffer.slice(0, actualLength);
- data.retain();
- this.valueCount = valueCount;
- }
-
- @Override
- public Mutator getMutator() {
- return new Mutator();
- }
-
- @Override
- public Accessor getAccessor() {
- return new Accessor();
- }
-
- @Override
- public TransferPair getTransferPair() {
- return new TransferImpl(getField());
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- return new TransferImpl(getField().withPath(ref));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new TransferImpl((BitVector) to);
- }
-
-
- public void transferTo(BitVector target) {
- target.clear();
- if (target.data != null) {
- target.data.release();
- }
- target.data = data;
- target.data.retain(1);
- target.valueCount = valueCount;
- clear();
- }
-
- public void splitAndTransferTo(int startIndex, int length, BitVector target) {
- assert startIndex + length <= valueCount;
- int firstByte = getByteIndex(startIndex);
- int byteSize = getSizeFromCount(length);
- int offset = startIndex % 8;
- if (offset == 0) {
- target.clear();
- // slice
- if (target.data != null) {
- target.data.release();
- }
- target.data = (DrillBuf) data.slice(firstByte, byteSize);
- target.data.retain(1);
- } else {
- // Copy data
- // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
- // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
- // The last byte copied to target is a bit tricky :
- // 1) if length requires partly byte (length % 8 !=0), copy the remaining bits only.
- // 2) otherwise, copy the last byte in the same way as to the prior bytes.
- target.clear();
- target.allocateNew(length);
- // TODO maybe do this one word at a time, rather than byte?
- for(int i = 0; i < byteSize - 1; i++) {
- target.data.setByte(i, (((this.data.getByte(firstByte + i) & 0xFF) >>> offset) + (this.data.getByte(firstByte + i + 1) << (8 - offset))));
- }
- if (length % 8 != 0) {
- target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
- } else {
- target.data.setByte(byteSize - 1,
- (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) << (8 - offset))));
- }
- }
- target.getMutator().setValueCount(length);
- }
-
- private class TransferImpl implements TransferPair {
- BitVector to;
-
- public TransferImpl(MaterializedField field) {
- this.to = new BitVector(field, allocator);
- }
-
- public TransferImpl(BitVector to) {
- this.to = to;
- }
-
- @Override
- public BitVector getTo() {
- return to;
- }
-
- @Override
- public void transfer() {
- transferTo(to);
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- splitAndTransferTo(startIndex, length, to);
- }
-
- @Override
- public void copyValueSafe(int fromIndex, int toIndex) {
- to.copyFromSafe(fromIndex, toIndex, BitVector.this);
- }
- }
-
- private void decrementAllocationMonitor() {
- if (allocationMonitor > 0) {
- allocationMonitor = 0;
- }
- --allocationMonitor;
- }
-
- private void incrementAllocationMonitor() {
- ++allocationMonitor;
- }
-
- public class Accessor extends BaseAccessor {
-
- /**
- * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the bit was not set.
- *
- * @param index
- * position of the bit in the vector
- * @return 1 if set, otherwise 0
- */
- public final int get(int index) {
- int byteIndex = index >> 3;
- byte b = data.getByte(byteIndex);
- int bitIndex = index & 7;
- return Long.bitCount(b & (1L << bitIndex));
- }
-
- @Override
- public boolean isNull(int index) {
- return false;
- }
-
- @Override
- public final Boolean getObject(int index) {
- return new Boolean(get(index) != 0);
- }
-
- @Override
- public final int getValueCount() {
- return valueCount;
- }
-
- public final void get(int index, BitHolder holder) {
- holder.value = get(index);
- }
-
- public final void get(int index, NullableBitHolder holder) {
- holder.isSet = 1;
- holder.value = get(index);
- }
- }
-
- /**
- * MutableBit implements a vector of bit-width values. Elements in the vector are accessed by position from the
- * logical start of the vector. Values should be pushed onto the vector sequentially, but may be randomly accessed.
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public class Mutator extends BaseMutator {
-
- private Mutator() {
- }
-
- /**
- * Set the bit at the given index to the specified value.
- *
- * @param index
- * position of the bit to set
- * @param value
- * value to set (either 1 or 0)
- */
- public final void set(int index, int value) {
- int byteIndex = index >> 3;
- int bitIndex = index & 7;
- byte currentByte = data.getByte(byteIndex);
- byte bitMask = (byte) (1L << bitIndex);
- if (value != 0) {
- currentByte |= bitMask;
- } else {
- currentByte -= (bitMask & currentByte);
- }
-
- data.setByte(byteIndex, currentByte);
- }
-
- public final void set(int index, BitHolder holder) {
- set(index, holder.value);
- }
-
- final void set(int index, NullableBitHolder holder) {
- set(index, holder.value);
- }
-
- public void setSafe(int index, int value) {
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, value);
- }
-
- public void setSafe(int index, BitHolder holder) {
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, holder.value);
- }
-
- public void setSafe(int index, NullableBitHolder holder) {
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, holder.value);
- }
-
- @Override
- public final void setValueCount(int valueCount) {
- int currentValueCapacity = getValueCapacity();
- BitVector.this.valueCount = valueCount;
- int idx = getSizeFromCount(valueCount);
- while(valueCount > getValueCapacity()) {
- reAlloc();
- }
- if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
- incrementAllocationMonitor();
- } else if (allocationMonitor > 0) {
- allocationMonitor = 0;
- }
- VectorTrimmer.trim(data, idx);
- }
-
- @Override
- public final void generateTestData(int values) {
- boolean even = true;
- for(int i = 0; i < values; i++, even = !even) {
- if (even) {
- set(i, 1);
- }
- }
- setValueCount(values);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
deleted file mode 100644
index c2781eb..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-
-public interface FixedWidthVector extends ValueVector{
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount Number of values in the vector.
- */
- void allocateNew(int valueCount);
-
-/**
- * Zero out the underlying buffer backing this vector.
- */
- void zeroVector();
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java
deleted file mode 100644
index 8091c4c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-public interface NullableVector extends ValueVector{
-
- ValueVector getValuesVector();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
deleted file mode 100644
index c418751..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-public interface NullableVectorDefinitionSetter {
-
- public void setIndexDefined(int index);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
deleted file mode 100644
index a7a3609..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.DrillBuf;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.exec.expr.holders.ObjectHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-public class ObjectVector extends BaseValueVector {
- private final Accessor accessor = new Accessor();
- private final Mutator mutator = new Mutator();
- private int maxCount = 0;
- private int count = 0;
- private int allocationSize = 4096;
-
- private List<Object[]> objectArrayList = new ArrayList<>();
-
- public ObjectVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- public void addNewArray() {
- objectArrayList.add(new Object[allocationSize]);
- maxCount += allocationSize;
- }
-
- @Override
- public FieldReader getReader() {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- public final class Mutator implements ValueVector.Mutator {
-
- public void set(int index, Object obj) {
- int listOffset = index / allocationSize;
- if (listOffset >= objectArrayList.size()) {
- addNewArray();
- }
- objectArrayList.get(listOffset)[index % allocationSize] = obj;
- }
-
- public boolean setSafe(int index, long value) {
- set(index, value);
- return true;
- }
-
- protected void set(int index, ObjectHolder holder) {
- set(index, holder.obj);
- }
-
- public boolean setSafe(int index, ObjectHolder holder){
- set(index, holder);
- return true;
- }
-
- @Override
- public void setValueCount(int valueCount) {
- count = valueCount;
- }
-
- @Override
- public void reset() {
- count = 0;
- maxCount = 0;
- objectArrayList = new ArrayList<>();
- addNewArray();
- }
-
- @Override
- public void generateTestData(int values) {
- }
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- // NoOp
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
- addNewArray();
- }
-
- public void allocateNew(int valueCount) throws OutOfMemoryRuntimeException {
- while (maxCount < valueCount) {
- addNewArray();
- }
- }
-
- @Override
- public boolean allocateNewSafe() {
- allocateNew();
- return true;
- }
-
- @Override
- public int getBufferSize() {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public void close() {
- clear();
- }
-
- @Override
- public void clear() {
- objectArrayList.clear();
- maxCount = 0;
- count = 0;
- }
-
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- @Override
- public TransferPair getTransferPair() {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public int getValueCapacity() {
- return maxCount;
- }
-
- @Override
- public Accessor getAccessor() {
- return accessor;
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public UserBitShared.SerializedField getMetadata() {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- @Override
- public Iterator<ValueVector> iterator() {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- public final class Accessor extends BaseAccessor {
- @Override
- public Object getObject(int index) {
- int listOffset = index / allocationSize;
- if (listOffset >= objectArrayList.size()) {
- addNewArray();
- }
- return objectArrayList.get(listOffset)[index % allocationSize];
- }
-
- @Override
- public int getValueCount() {
- return count;
- }
-
- public Object get(int index) {
- return getObject(index);
- }
-
- public void get(int index, ObjectHolder holder){
- holder.obj = getObject(index);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
deleted file mode 100644
index 4c2491c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.vector;
-
-import org.apache.drill.exec.util.CallBack;
-
-
-public class SchemaChangeCallBack implements CallBack {
- private boolean schemaChanged = false;
-
- /**
- * Constructs a schema-change callback with the schema-changed state set to
- * {@code false}.
- */
- public SchemaChangeCallBack() {
- }
-
- /**
- * Sets the schema-changed state to {@code true}.
- */
- @Override
- public void doWork() {
- schemaChanged = true;
- }
-
- /**
- * Returns the value of schema-changed state, <strong>resetting</strong> the
- * schema-changed state to {@code false}.
- */
- public boolean getSchemaChangedAndReset() {
- final boolean current = schemaChanged;
- schemaChanged = false;
- return current;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
deleted file mode 100644
index 42384cb..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.DrillBuf;
-
-import java.math.BigDecimal;
-
-import org.apache.drill.exec.expr.holders.BigIntHolder;
-import org.apache.drill.exec.expr.holders.BitHolder;
-import org.apache.drill.exec.expr.holders.DateHolder;
-import org.apache.drill.exec.expr.holders.Decimal18Holder;
-import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
-import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
-import org.apache.drill.exec.expr.holders.Decimal9Holder;
-import org.apache.drill.exec.expr.holders.Float4Holder;
-import org.apache.drill.exec.expr.holders.Float8Holder;
-import org.apache.drill.exec.expr.holders.IntHolder;
-import org.apache.drill.exec.expr.holders.IntervalDayHolder;
-import org.apache.drill.exec.expr.holders.IntervalYearHolder;
-import org.apache.drill.exec.expr.holders.NullableBitHolder;
-import org.apache.drill.exec.expr.holders.TimeHolder;
-import org.apache.drill.exec.expr.holders.TimeStampHolder;
-import org.apache.drill.exec.expr.holders.VarCharHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.util.DecimalUtility;
-
-import com.google.common.base.Charsets;
-
-
-public class ValueHolderHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
-
- public static IntHolder getIntHolder(int value) {
- IntHolder holder = new IntHolder();
- holder.value = value;
-
- return holder;
- }
-
- public static BigIntHolder getBigIntHolder(long value) {
- BigIntHolder holder = new BigIntHolder();
- holder.value = value;
-
- return holder;
- }
-
- public static Float4Holder getFloat4Holder(float value) {
- Float4Holder holder = new Float4Holder();
- holder.value = value;
-
- return holder;
- }
-
- public static Float8Holder getFloat8Holder(double value) {
- Float8Holder holder = new Float8Holder();
- holder.value = value;
-
- return holder;
- }
-
- public static DateHolder getDateHolder(long value) {
- DateHolder holder = new DateHolder();
- holder.value = value;
- return holder;
- }
-
- public static TimeHolder getTimeHolder(int value) {
- TimeHolder holder = new TimeHolder();
- holder.value = value;
- return holder;
- }
-
- public static TimeStampHolder getTimeStampHolder(long value) {
- TimeStampHolder holder = new TimeStampHolder();
- holder.value = value;
- return holder;
- }
-
- public static BitHolder getBitHolder(int value) {
- BitHolder holder = new BitHolder();
- holder.value = value;
-
- return holder;
- }
-
- public static NullableBitHolder getNullableBitHolder(boolean isNull, int value) {
- NullableBitHolder holder = new NullableBitHolder();
- holder.isSet = isNull? 0 : 1;
- if (! isNull) {
- holder.value = value;
- }
-
- return holder;
- }
-
- public static VarCharHolder getVarCharHolder(DrillBuf buf, String s){
- VarCharHolder vch = new VarCharHolder();
-
- byte[] b = s.getBytes(Charsets.UTF_8);
- vch.start = 0;
- vch.end = b.length;
- vch.buffer = buf.reallocIfNeeded(b.length);
- vch.buffer.setBytes(0, b);
- return vch;
- }
-
- public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
- VarCharHolder vch = new VarCharHolder();
-
- byte[] b = s.getBytes(Charsets.UTF_8);
- vch.start = 0;
- vch.end = b.length;
- vch.buffer = a.buffer(b.length); //
- vch.buffer.setBytes(0, b);
- return vch;
- }
-
-
- public static IntervalYearHolder getIntervalYearHolder(int intervalYear) {
- IntervalYearHolder holder = new IntervalYearHolder();
-
- holder.value = intervalYear;
- return holder;
- }
-
- public static IntervalDayHolder getIntervalDayHolder(int days, int millis) {
- IntervalDayHolder dch = new IntervalDayHolder();
-
- dch.days = days;
- dch.milliseconds = millis;
- return dch;
- }
-
- public static Decimal9Holder getDecimal9Holder(int decimal, int scale, int precision) {
- Decimal9Holder dch = new Decimal9Holder();
-
- dch.scale = scale;
- dch.precision = precision;
- dch.value = decimal;
-
- return dch;
- }
-
- public static Decimal18Holder getDecimal18Holder(long decimal, int scale, int precision) {
- Decimal18Holder dch = new Decimal18Holder();
-
- dch.scale = scale;
- dch.precision = precision;
- dch.value = decimal;
-
- return dch;
- }
-
- public static Decimal28SparseHolder getDecimal28Holder(DrillBuf buf, String decimal) {
-
- Decimal28SparseHolder dch = new Decimal28SparseHolder();
-
- BigDecimal bigDecimal = new BigDecimal(decimal);
-
- dch.scale = bigDecimal.scale();
- dch.precision = bigDecimal.precision();
- Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
- dch.start = 0;
- dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.integerSize);
- DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
-
- return dch;
- }
-
- public static Decimal38SparseHolder getDecimal38Holder(DrillBuf buf, String decimal) {
-
- Decimal38SparseHolder dch = new Decimal38SparseHolder();
-
- BigDecimal bigDecimal = new BigDecimal(decimal);
-
- dch.scale = bigDecimal.scale();
- dch.precision = bigDecimal.precision();
- Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
- dch.start = 0;
- dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.integerSize);
- DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
-
- return dch;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
deleted file mode 100644
index 8627d1a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import java.io.Closeable;
-
-import io.netty.buffer.DrillBuf;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-/**
- * An abstraction that is used to store a sequence of values in an individual column.
- *
- * A {@link ValueVector value vector} stores underlying data in-memory in a columnar fashion that is compact and
- * efficient. The column whose data is stored, is referred by {@link #getField()}.
- *
- * A vector when instantiated, relies on a {@link org.apache.drill.exec.record.DeadBuf dead buffer}. It is important
- * that vector is allocated before attempting to read or write.
- *
- * There are a few "rules" around vectors:
- *
- * <ul>
- * <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
- * <li>null vectors start with all values as null before writing anything</li>
- * <li>for variable width types, the offset vector should be all zeros before writing</li>
- * <li>you must call setValueCount before a vector can be read</li>
- * <li>you should never write to a vector once it has been read.</li>
- * </ul>
- *
- * Please note that the current implementation doesn't enfore those rules, hence we may find few places that
- * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)
- *
- * This interface "should" strive to guarantee this order of operation:
- * <blockquote>
- * allocate > mutate > setvaluecount > access > clear (or allocate to start the process over).
- * </blockquote>
- */
-public interface ValueVector extends Closeable, Iterable<ValueVector> {
- /**
- * Allocate new buffers. ValueVector implements logic to determine how much to allocate.
- * @throws OutOfMemoryRuntimeException Thrown if no memory can be allocated.
- */
- void allocateNew() throws OutOfMemoryRuntimeException;
-
- /**
- * Allocates new buffers. ValueVector implements logic to determine how much to allocate.
- * @return Returns true if allocation was succesful.
- */
- boolean allocateNewSafe();
-
- /**
- * Set the initial record capacity
- * @param numRecords
- */
- void setInitialCapacity(int numRecords);
-
- /**
- * Returns the maximum number of values that can be stored in this vector instance.
- */
- int getValueCapacity();
-
- /**
- * Alternative to clear(). Allows use as an AutoCloseable in try-with-resources.
- */
- @Override
- void close();
-
- /**
- * Release the underlying DrillBuf and reset the ValueVector to empty.
- */
- void clear();
-
- /**
- * Get information about how this field is materialized.
- */
- MaterializedField getField();
-
- /**
- * Returns a {@link org.apache.drill.exec.record.TransferPair transfer pair}, creating a new target vector of
- * the same type.
- */
- TransferPair getTransferPair();
-
- TransferPair getTransferPair(FieldReference ref);
-
- /**
- * Returns a new {@link org.apache.drill.exec.record.TransferPair transfer pair} that is used to transfer underlying
- * buffers into the target vector.
- */
- TransferPair makeTransferPair(ValueVector target);
-
- /**
- * Returns an {@link org.apache.drill.exec.vector.ValueVector.Accessor accessor} that is used to read from this vector
- * instance.
- */
- Accessor getAccessor();
-
- /**
- * Returns an {@link org.apache.drill.exec.vector.ValueVector.Mutator mutator} that is used to write to this vector
- * instance.
- */
- Mutator getMutator();
-
- /**
- * Returns a {@link org.apache.drill.exec.vector.complex.reader.FieldReader field reader} that supports reading values
- * from this vector.
- */
- FieldReader getReader();
-
- /**
- * Get the metadata for this field. Used in serialization
- *
- * @return FieldMetadata for this field.
- */
- SerializedField getMetadata();
-
- /**
- * Returns the number of bytes that is used by this vector instance.
- */
- int getBufferSize();
-
- /**
- * Returns the number of bytes that is used by this vector if it holds the given number
- * of values. The result will be the same as if Mutator.setValueCount() were called, followed
- * by calling getBufferSize(), but without any of the closing side-effects that setValueCount()
- * implies wrt finishing off the population of a vector. Some operations might wish to use
- * this to determine how much memory has been used by a vector so far, even though it is
- * not finished being populated.
- *
- * @param valueCount the number of values to assume this vector contains
- * @return the buffer size if this vector is holding valueCount values
- */
- int getBufferSizeFor(int valueCount);
-
- /**
- * Return the underlying buffers associated with this vector. Note that this doesn't impact the reference counts for
- * this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus
- * external classes shouldn't hold a reference to it (unless they change it).
- * @param clear Whether to clear vector before returning; the buffers will still be refcounted;
- * but the returned array will be the only reference to them
- *
- * @return The underlying {@link io.netty.buffer.DrillBuf buffers} that is used by this vector instance.
- */
- DrillBuf[] getBuffers(boolean clear);
-
- /**
- * Load the data provided in the buffer. Typically used when deserializing from the wire.
- *
- * @param metadata
- * Metadata used to decode the incoming buffer.
- * @param buffer
- * The buffer that contains the ValueVector.
- */
- void load(SerializedField metadata, DrillBuf buffer);
-
- /**
- * An abstraction that is used to read from this vector instance.
- */
- interface Accessor {
- /**
- * Get the Java Object representation of the element at the specified position. Useful for testing.
- *
- * @param index
- * Index of the value to get
- */
- Object getObject(int index);
-
- /**
- * Returns the number of values that is stored in this vector.
- */
- int getValueCount();
-
- /**
- * Returns true if the value at the given index is null, false otherwise.
- */
- boolean isNull(int index);
- }
-
- /**
- * An abstractiong that is used to write into this vector instance.
- */
- interface Mutator {
- /**
- * Sets the number of values that is stored in this vector to the given value count.
- *
- * @param valueCount value count to set.
- */
- void setValueCount(int valueCount);
-
- /**
- * Resets the mutator to pristine state.
- */
- void reset();
-
- /**
- * @deprecated this has nothing to do with value vector abstraction and should be removed.
- */
- @Deprecated
- void generateTestData(int values);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
deleted file mode 100644
index ee9c039..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.DrillBuf;
-
-public interface VariableWidthVector extends ValueVector{
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param totalBytes Desired size of the underlying data buffer.
- * @param valueCount Number of values in the vector.
- */
- void allocateNew(int totalBytes, int valueCount);
-
- /**
- * Provide the maximum amount of variable width bytes that can be stored int his vector.
- * @return
- */
- int getByteCapacity();
-
- VariableWidthMutator getMutator();
-
- VariableWidthAccessor getAccessor();
-
- interface VariableWidthAccessor extends Accessor {
- int getValueLength(int index);
- }
-
- int getCurrentSizeInBytes();
-
- interface VariableWidthMutator extends Mutator {
- void setValueLengthSafe(int index, int length);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
deleted file mode 100644
index 2ecaef6..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import java.util.Collection;
-
-import com.google.common.base.Preconditions;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class VectorDescriptor {
- private static final String DEFAULT_NAME = "NONE";
-
- private final MaterializedField field;
-
- public VectorDescriptor(final TypeProtos.MajorType type) {
- this(DEFAULT_NAME, type);
- }
-
- public VectorDescriptor(final String name, final TypeProtos.MajorType type) {
- this(MaterializedField.create(name, type));
- }
-
- public VectorDescriptor(final MaterializedField field) {
- this.field = Preconditions.checkNotNull(field, "field cannot be null");
- }
-
- public MaterializedField getField() {
- return field;
- }
-
- public TypeProtos.MajorType getType() {
- return field.getType();
- }
-
- public String getName() {
- return field.getLastName();
- }
-
- public Collection<MaterializedField> getChildren() {
- return field.getChildren();
- }
-
- public boolean hasName() {
- return getName() != DEFAULT_NAME;
- }
-
- public VectorDescriptor withName(final String name) {
- return new VectorDescriptor(field.withPath(new FieldReference(name)));
- }
-
- public VectorDescriptor withType(final TypeProtos.MajorType type) {
- return new VectorDescriptor(field.withType(type));
- }
-
- public static VectorDescriptor create(final String name, final TypeProtos.MajorType type) {
- return new VectorDescriptor(name, type);
- }
-
- public static VectorDescriptor create(final TypeProtos.MajorType type) {
- return new VectorDescriptor(type);
- }
-
- public static VectorDescriptor create(final MaterializedField field) {
- return new VectorDescriptor(field);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
deleted file mode 100644
index 7f977f2..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.DrillBuf;
-
-public class VectorTrimmer {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorTrimmer.class);
-
- public static void trim(ByteBuf data, int idx) {
- data.writerIndex(idx);
- if (data instanceof DrillBuf) {
- // data.capacity(idx);
- data.writerIndex(idx);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ZeroVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
deleted file mode 100644
index 9fb151f..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector;
-
-import java.util.Iterator;
-
-import com.google.common.collect.Iterators;
-import io.netty.buffer.DrillBuf;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.complex.impl.NullReader;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-public class ZeroVector implements ValueVector {
- public final static ZeroVector INSTANCE = new ZeroVector();
-
- private final MaterializedField field = MaterializedField.create("[DEFAULT]", Types.LATE_BIND_TYPE);
-
- private final TransferPair defaultPair = new TransferPair() {
- @Override
- public void transfer() { }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) { }
-
- @Override
- public ValueVector getTo() {
- return ZeroVector.this;
- }
-
- @Override
- public void copyValueSafe(int from, int to) { }
- };
-
- private final Accessor defaultAccessor = new Accessor() {
- @Override
- public Object getObject(int index) {
- return null;
- }
-
- @Override
- public int getValueCount() {
- return 0;
- }
-
- @Override
- public boolean isNull(int index) {
- return true;
- }
- };
-
- private final Mutator defaultMutator = new Mutator() {
- @Override
- public void setValueCount(int valueCount) { }
-
- @Override
- public void reset() { }
-
- @Override
- public void generateTestData(int values) { }
- };
-
- public ZeroVector() { }
-
- @Override
- public void close() { }
-
- @Override
- public void clear() { }
-
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- @Override
- public TransferPair getTransferPair() {
- return defaultPair;
- }
-
- @Override
- public UserBitShared.SerializedField getMetadata() {
- return getField()
- .getAsBuilder()
- .setBufferLength(getBufferSize())
- .setValueCount(getAccessor().getValueCount())
- .build();
- }
-
- @Override
- public Iterator iterator() {
- return Iterators.emptyIterator();
- }
-
- @Override
- public int getBufferSize() {
- return 0;
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- return 0;
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- return new DrillBuf[0];
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
- allocateNewSafe();
- }
-
- @Override
- public boolean allocateNewSafe() {
- return true;
- }
-
- @Override
- public void setInitialCapacity(int numRecords) { }
-
- @Override
- public int getValueCapacity() {
- return 0;
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- return defaultPair;
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector target) {
- return defaultPair;
- }
-
- @Override
- public Accessor getAccessor() {
- return defaultAccessor;
- }
-
- @Override
- public Mutator getMutator() {
- return defaultMutator;
- }
-
- @Override
- public FieldReader getReader() {
- return NullReader.INSTANCE;
- }
-
- @Override
- public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
deleted file mode 100644
index 5f6d0d2..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.DecimalMin.List;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TypedFieldId;
-import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.exec.vector.ValueVector;
-
-/**
- * Base class for composite vectors.
- *
- * This class implements common functionality of composite vectors.
- */
-public abstract class AbstractContainerVector implements ValueVector {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
-
- protected final MaterializedField field;
- protected final BufferAllocator allocator;
- protected final CallBack callBack;
-
- protected AbstractContainerVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- this.field = Preconditions.checkNotNull(field);
- this.allocator = allocator;
- this.callBack = callBack;
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
- if (!allocateNewSafe()) {
- throw new OutOfMemoryRuntimeException();
- }
- }
-
- /**
- * Returns the field definition of this instance.
- */
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- /**
- * Returns a {@link org.apache.drill.exec.vector.ValueVector} corresponding to the given field name if exists or null.
- */
- public ValueVector getChild(String name) {
- return getChild(name, ValueVector.class);
- }
-
- /**
- * Returns a sequence of field names in the order that they show up in the schema.
- */
- protected Collection<String> getChildFieldNames() {
- return Sets.newLinkedHashSet(Iterables.transform(field.getChildren(), new Function<MaterializedField, String>() {
- @Nullable
- @Override
- public String apply(MaterializedField field) {
- return Preconditions.checkNotNull(field).getLastName();
- }
- }));
- }
-
- /**
- * Clears out all underlying child vectors.
- */
- @Override
- public void close() {
- for (ValueVector vector:(Iterable<ValueVector>)this) {
- vector.close();
- }
- }
-
- protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
- if (clazz.isAssignableFrom(v.getClass())) {
- return (T) v;
- }
- throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
- }
-
- public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
- return FieldIdUtil.getFieldIdIfMatches(this, builder, addToBreadCrumb, seg);
- }
-
- MajorType getLastPathType() {
- if((this.getField().getType().getMinorType() == MinorType.LIST &&
- this.getField().getType().getMode() == DataMode.REPEATED)) { // Use Repeated scalar type instead of Required List.
- VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
- ValueVector v = vord.vector;
- if (! (v instanceof AbstractContainerVector)) {
- return v.getField().getType();
- }
- } else if (this.getField().getType().getMinorType() == MinorType.MAP &&
- this.getField().getType().getMode() == DataMode.REPEATED) { // Use Required Map
- return this.getField().getType().toBuilder().setMode(DataMode.REQUIRED).build();
- }
-
- return this.getField().getType();
- }
-
- protected boolean supportsDirectRead() {
- return false;
- }
-
- // return the number of child vectors
- public abstract int size();
-
- // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
- public abstract <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz);
-
- // return the child vector with the input name
- public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
-
- // return the child vector's ordinal in the composite container
- public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
deleted file mode 100644
index 35df691..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import io.netty.buffer.DrillBuf;
-import org.apache.drill.common.collections.MapWithOrdinal;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.exec.vector.ValueVector;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-/*
- * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
- */
-public abstract class AbstractMapVector extends AbstractContainerVector {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
-
- // Maintains a map with key as field name and value is the vector itself
- private final MapWithOrdinal<String, ValueVector> vectors = new MapWithOrdinal<>();
-
- protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- super(field.clone(), allocator, callBack);
- MaterializedField clonedField = field.clone();
- // create the hierarchy of the child vectors based on the materialized field
- for (MaterializedField child : clonedField.getChildren()) {
- if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
- final String fieldName = child.getLastName();
- final ValueVector v = TypeHelper.getNewVector(child, allocator, callBack);
- putVector(fieldName, v);
- }
- }
- }
-
- @Override
- public void close() {
- for(final ValueVector valueVector : vectors.values()) {
- valueVector.close();
- }
- vectors.clear();
-
- super.close();
- }
-
- @Override
- public boolean allocateNewSafe() {
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- for (final ValueVector v : vectors.values()) {
- if (!v.allocateNewSafe()) {
- return false;
- }
- }
- success = true;
- } finally {
- if (!success) {
- clear();
- }
- }
- return true;
- }
-
- /**
- * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
- * {@link org.apache.drill.exec.vector.ValueVector}.
- *
- * Execution takes place in the following order:
- * <ul>
- * <li>
- * if field is new, create and insert a new vector of desired type.
- * </li>
- * <li>
- * if field exists and existing vector is of desired vector type, return the vector.
- * </li>
- * <li>
- * if field exists and null filled, clear the existing vector; create and insert a new vector of desired type.
- * </li>
- * <li>
- * otherwise, throw an {@link java.lang.IllegalStateException}
- * </li>
- * </ul>
- *
- * @param name name of the field
- * @param type type of the field
- * @param clazz class of expected vector type
- * @param <T> class type of expected vector type
- * @throws java.lang.IllegalStateException raised if there is a hard schema change
- *
- * @return resultant {@link org.apache.drill.exec.vector.ValueVector}
- */
- @Override
- public <T extends ValueVector> T addOrGet(String name, TypeProtos.MajorType type, Class<T> clazz) {
- final ValueVector existing = getChild(name);
- boolean create = false;
- if (existing == null) {
- create = true;
- } else if (clazz.isAssignableFrom(existing.getClass())) {
- return (T) existing;
- } else if (nullFilled(existing)) {
- existing.clear();
- create = true;
- }
- if (create) {
- final T vector = (T) TypeHelper.getNewVector(field.getPath(), name, allocator, type, callBack);
- putChild(name, vector);
- if (callBack!=null) {
- callBack.doWork();
- }
- return vector;
- }
- final String message = "Drill does not support schema change yet. Existing[{}] and desired[{}] vector types mismatch";
- throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(), clazz.getSimpleName()));
- }
-
- private boolean nullFilled(ValueVector vector) {
- for (int r = 0; r < vector.getAccessor().getValueCount(); r++) {
- if (!vector.getAccessor().isNull(r)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Returns a {@link org.apache.drill.exec.vector.ValueVector} corresponding to the given ordinal identifier.
- */
- public ValueVector getChildByOrdinal(int id) {
- return vectors.getByOrdinal(id);
- }
-
- /**
- * Returns a {@link org.apache.drill.exec.vector.ValueVector} instance of subtype of <T> corresponding to the given
- * field name if exists or null.
- */
- @Override
- public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
- final ValueVector v = vectors.get(name.toLowerCase());
- if (v == null) {
- return null;
- }
- return typeify(v, clazz);
- }
-
- /**
- * Inserts the vector with the given name if it does not exist else replaces it with the new value.
- *
- * Note that this method does not enforce any vector type check nor throws a schema change exception.
- */
- protected void putChild(String name, ValueVector vector) {
- putVector(name, vector);
- field.addChild(vector.getField());
- }
-
- /**
- * Inserts the input vector into the map if it does not exist, replaces if it exists already
- * @param name field name
- * @param vector vector to be inserted
- */
- protected void putVector(String name, ValueVector vector) {
- final ValueVector old = vectors.put(
- Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
- Preconditions.checkNotNull(vector, "vector cannot be null")
- );
- if (old != null && old != vector) {
- logger.debug("Field [{}] mutated from [{}] to [{}]", name, old.getClass().getSimpleName(),
- vector.getClass().getSimpleName());
- }
- }
-
- /**
- * Returns a sequence of underlying child vectors.
- */
- protected Collection<ValueVector> getChildren() {
- return vectors.values();
- }
-
- /**
- * Returns the number of underlying child vectors.
- */
- @Override
- public int size() {
- return vectors.size();
- }
-
- @Override
- public Iterator<ValueVector> iterator() {
- return vectors.values().iterator();
- }
-
- /**
- * Returns a list of scalar child vectors recursing the entire vector hierarchy.
- */
- public List<ValueVector> getPrimitiveVectors() {
- final List<ValueVector> primitiveVectors = Lists.newArrayList();
- for (final ValueVector v : vectors.values()) {
- if (v instanceof AbstractMapVector) {
- AbstractMapVector mapVector = (AbstractMapVector) v;
- primitiveVectors.addAll(mapVector.getPrimitiveVectors());
- } else {
- primitiveVectors.add(v);
- }
- }
- return primitiveVectors;
- }
-
- /**
- * Returns a vector with its corresponding ordinal mapping if field exists or null.
- */
- @Override
- public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
- final int ordinal = vectors.getOrdinal(name.toLowerCase());
- if (ordinal < 0) {
- return null;
- }
- final ValueVector vector = vectors.getByOrdinal(ordinal);
- return new VectorWithOrdinal(vector, ordinal);
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- final List<DrillBuf> buffers = Lists.newArrayList();
-
- for (final ValueVector vector : vectors.values()) {
- for (final DrillBuf buf : vector.getBuffers(false)) {
- buffers.add(buf);
- if (clear) {
- buf.retain(1);
- }
- }
- if (clear) {
- vector.clear();
- }
- }
-
- return buffers.toArray(new DrillBuf[buffers.size()]);
- }
-
- @Override
- public int getBufferSize() {
- int actualBufSize = 0 ;
-
- for (final ValueVector v : vectors.values()) {
- for (final DrillBuf buf : v.getBuffers(false)) {
- actualBufSize += buf.writerIndex();
- }
- }
- return actualBufSize;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
deleted file mode 100644
index de4ad95..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import java.util.Collections;
-import java.util.Iterator;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-import io.netty.buffer.DrillBuf;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.exception.SchemaChangeRuntimeException;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.vector.AddOrGetResult;
-import org.apache.drill.exec.vector.BaseValueVector;
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VectorDescriptor;
-import org.apache.drill.exec.vector.ZeroVector;
-
-public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
-
- public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
- public final static String OFFSETS_VECTOR_NAME = "$offsets$";
- public final static String DATA_VECTOR_NAME = "$data$";
-
- public final static MaterializedField OFFSETS_FIELD =
- MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(TypeProtos.MinorType.UINT4));
-
- protected final UInt4Vector offsets;
- protected ValueVector vector;
-
- protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
- this(field, allocator, DEFAULT_DATA_VECTOR);
- }
-
- protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
- super(field, allocator);
- this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
- this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
- }
-
- @Override
- public boolean allocateNewSafe() {
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- if (!offsets.allocateNewSafe()) {
- return false;
- }
- success = vector.allocateNewSafe();
- } finally {
- if (!success) {
- clear();
- }
- }
- offsets.zeroVector();
- return success;
- }
-
-
- @Override
- public UInt4Vector getOffsetVector() {
- return offsets;
- }
-
- @Override
- public ValueVector getDataVector() {
- return vector;
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- offsets.setInitialCapacity(numRecords + 1);
- vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
- }
-
- @Override
- public int getValueCapacity() {
- final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
- if (vector == DEFAULT_DATA_VECTOR) {
- return offsetValueCapacity;
- }
- return Math.min(vector.getValueCapacity(), offsetValueCapacity);
- }
-
- @Override
- protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
- return super.getMetadataBuilder()
- .addChild(offsets.getMetadata())
- .addChild(vector.getMetadata());
- }
-
- @Override
- public int getBufferSize() {
- if (getAccessor().getValueCount() == 0) {
- return 0;
- }
- return offsets.getBufferSize() + vector.getBufferSize();
- }
-
- @Override
- public int getBufferSizeFor(int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount);
- }
-
- @Override
- public Iterator<ValueVector> iterator() {
- return Collections.singleton(getDataVector()).iterator();
- }
-
- @Override
- public void clear() {
- offsets.clear();
- vector.clear();
- super.clear();
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), DrillBuf.class);
- if (clear) {
- for (DrillBuf buffer:buffers) {
- buffer.retain();
- }
- clear();
- }
- return buffers;
- }
-
- @Override
- public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
- final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
- offsets.load(offsetMetadata, buffer);
-
- final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
- if (getDataVector() == DEFAULT_DATA_VECTOR) {
- addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
- }
-
- final int offsetLength = offsetMetadata.getBufferLength();
- final int vectorLength = vectorMetadata.getBufferLength();
- vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
- }
-
- /**
- * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
- *
- * @see {@link ContainerVectorLike#size}
- */
- @Override
- public int size() {
- return vector == DEFAULT_DATA_VECTOR ? 0:1;
- }
-
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- boolean created = false;
- if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) {
- final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
- vector = TypeHelper.getNewVector(field, allocator);
- // returned vector must have the same field
- assert field.equals(vector.getField());
- getField().addChild(field);
- created = true;
- }
-
- final TypeProtos.MajorType actual = vector.getField().getType();
- if (!actual.equals(descriptor.getType())) {
- final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
- descriptor.getType(), actual);
- throw new SchemaChangeRuntimeException(msg);
- }
-
- return new AddOrGetResult<>((T)vector, created);
- }
-
- protected void replaceDataVector(ValueVector v) {
- vector.clear();
- vector = v;
- }
-
- public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
-
- @Override
- public int getValueCount() {
- return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
- }
-
- @Override
- public int getInnerValueCount() {
- return vector.getAccessor().getValueCount();
- }
-
- @Override
- public int getInnerValueCountAt(int index) {
- return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
- }
-
- @Override
- public boolean isNull(int index) {
- return false;
- }
-
- @Override
- public boolean isEmpty(int index) {
- return false;
- }
- }
-
- public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
-
- @Override
- public void startNewValue(int index) {
- while (offsets.getValueCapacity() <= index) {
- offsets.reAlloc();
- }
- offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
- setValueCount(index+1);
- }
-
- @Override
- public void setValueCount(int valueCount) {
- // TODO: populate offset end points
- offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
- final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
- vector.getMutator().setValueCount(childValueCount);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
deleted file mode 100644
index 999b47d..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex;
-
-import org.apache.drill.exec.vector.AddOrGetResult;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VectorDescriptor;
-
-/**
- * A mix-in used for introducing container vector-like behaviour.
- */
-public interface ContainerVectorLike {
-
- /**
- * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
- *
- * @param descriptor vector descriptor
- * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
- * @throws org.apache.drill.common.exceptions.DrillRuntimeException
- * if schema change is not permissible between the given and existing data vector types.
- */
- <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
-
- /**
- * Returns the number of child vectors in this container vector-like instance.
- */
- int size();
-}
[16/45] drill git commit: DRILL-3987: (REFACTOR) Common and Vector
modules building.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
index b01534e..a708e92 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -19,11 +19,10 @@ package org.apache.drill.exec.memory;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.DrillBuf;
-import io.netty.buffer.UnsafeDirectLittleEndian;
import java.io.Closeable;
-import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.util.Pointer;
/**
@@ -39,7 +38,7 @@ public interface BufferAllocator extends Closeable {
* @param size
* The size in bytes.
* @return A new ByteBuf.
- * @throws OutOfMemoryRuntimeException if buffer cannot be allocated
+ * @throws OutOfMemoryException if buffer cannot be allocated
*/
public abstract DrillBuf buffer(int size);
@@ -50,7 +49,7 @@ public interface BufferAllocator extends Closeable {
* @param minSize The minimum size in bytes.
* @param maxSize The maximum size in bytes.
* @return A new ByteBuf.
- * @throws OutOfMemoryRuntimeException if buffer cannot be allocated
+ * @throws OutOfMemoryException if buffer cannot be allocated
*/
public abstract DrillBuf buffer(int minSize, int maxSize);
@@ -59,14 +58,19 @@ public interface BufferAllocator extends Closeable {
/**
* Create a child allocator nested below this one.
*
- * @param context - owning fragment for this allocator
- * @param initialReservation - specified in bytes
- * @param maximumReservation - specified in bytes
- * @param applyFragmentLimit - flag to conditionally enable fragment memory limits
+ * @param context
+ * - BufferManager associated with the new child allocator
+ * @param initialReservation
+ * - specified in bytes
+ * @param maximumReservation
+ * - specified in bytes
+ * @param applyFragmentLimit
+ * - flag to conditionally enable fragment memory limits
* @return - a new buffer allocator owned by the parent it was spawned from
- * @throws OutOfMemoryException - when off-heap memory has been exhausted
+ * @throws OutOfMemoryException
+ * - when off-heap memory has been exhausted
*/
- public abstract BufferAllocator getChildAllocator(FragmentContext context, long initialReservation,
+ public abstract BufferAllocator getChildAllocator(LimitConsumer limitListener, long initialReservation,
long maximumReservation, boolean applyFragmentLimit) throws OutOfMemoryException;
/**
@@ -90,15 +94,15 @@ public interface BufferAllocator extends Closeable {
/**
* For Top Level Allocators. Reset the fragment limits for all allocators
*/
- public void resetFragmentLimits();
+ public void resetLimits();
/**
* For Child allocators to set the Fragment limit for the corresponding fragment allocator.
* @param l the new fragment limit
*/
- public void setFragmentLimit(long l);
+ public void setLimit(long l);
- public long getFragmentLimit();
+ public long getLimit();
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
new file mode 100644
index 0000000..777d9d2
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+public interface LimitConsumer {
+
+ public String getIdentifier();
+ public long getAllocated();
+
+ public long getLimit();
+
+ public void setLimit(long limit);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
deleted file mode 100644
index 305eabd..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-
-public class OutOfMemoryRuntimeException extends DrillRuntimeException{
-
- public OutOfMemoryRuntimeException() {
- super();
-
- }
-
- public OutOfMemoryRuntimeException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
-
- }
-
- public OutOfMemoryRuntimeException(String message, Throwable cause) {
- super(message, cause);
-
- }
-
- public OutOfMemoryRuntimeException(String message) {
- super(message);
-
- }
-
- public OutOfMemoryRuntimeException(Throwable cause) {
- super(cause);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/ops/BufferManager.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/ops/BufferManager.java b/exec/vector/src/main/java/org/apache/drill/exec/ops/BufferManager.java
new file mode 100644
index 0000000..4df8f0e
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/ops/BufferManager.java
@@ -0,0 +1,66 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ops;
+
+import io.netty.buffer.DrillBuf;
+
+/**
+ * Manages a list of {@link DrillBuf}s that can be reallocated as needed. Upon
+ * re-allocation the old buffer will be freed. Managing a list of these buffers
+ * prevents some parts of the system from needing to define a correct location
+ * to place the final call to free them.
+ *
+ * The current uses of these types of buffers are within the pluggable components of Drill.
+ * In UDFs, memory management should not be a concern. We provide access to re-allocatable
+ * DrillBufs to give UDF writers general purpose buffers we can account for. To prevent the need
+ * for UDFs to contain boilerplate to close all of the buffers they request, this list
+ * is tracked at a higher level and all of the buffers are freed once we are sure that
+ * the code depending on them is done executing (currently {@link FragmentContext}
+ * and {@link QueryContext}.
+ */
+public interface BufferManager extends AutoCloseable {
+
+ /**
+ * Replace an old buffer with a new version at least of the provided size. Does not copy data.
+ *
+ * @param old
+ * Old Buffer that the user is no longer going to use.
+ * @param newSize
+ * Size of new replacement buffer.
+ * @return
+ */
+ public DrillBuf replace(DrillBuf old, int newSize);
+
+ /**
+ * Get a managed buffer of indeterminate size.
+ *
+ * @return A buffer.
+ */
+ public DrillBuf getManagedBuffer();
+
+ /**
+ * Get a managed buffer of at least a certain size.
+ *
+ * @param size
+ * The desired size
+ * @return A buffer
+ */
+ public DrillBuf getManagedBuffer(int size);
+
+ public void close();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index 26e1257..f8fb71c 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -26,7 +26,7 @@ import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
@@ -196,7 +196,7 @@ public class MaterializedField {
}
public Class<?> getValueClass() {
- return TypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
+ return BasicTypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
}
public boolean matches(SchemaPath path) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java b/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
index 931f981..a322f72 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
@@ -22,7 +22,7 @@ import java.util.Arrays;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.exec.vector.ValueVector;
import com.carrotsearch.hppc.IntArrayList;
@@ -105,7 +105,8 @@ public class TypedFieldId {
}
public Class<? extends ValueVector> getIntermediateClass() {
- return (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(intermediateType.getMinorType(), intermediateType.getMode());
+ return (Class<? extends ValueVector>) BasicTypeHelper.getValueVectorClass(intermediateType.getMinorType(),
+ intermediateType.getMode());
}
public MajorType getFinalType() {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/AssertionUtil.java b/exec/vector/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
new file mode 100644
index 0000000..2348e62
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+public class AssertionUtil {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
+
+
+ public static final boolean ASSERT_ENABLED;
+ public static final boolean BOUNDS_CHECKING_ENABLED;
+
+ static{
+ boolean isAssertEnabled = false;
+ assert isAssertEnabled = true;
+ ASSERT_ENABLED = isAssertEnabled;
+ BOUNDS_CHECKING_ENABLED = ASSERT_ENABLED || !"true".equals(System.getProperty("drill.enable_unsafe_memory_access"));
+ }
+
+ public static boolean isAssertionsEnabled(){
+ return ASSERT_ENABLED;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
new file mode 100644
index 0000000..0922b22
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
@@ -0,0 +1,730 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import io.netty.buffer.DrillBuf;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.drill.common.util.CoreDecimalUtility;
+import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
+import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
+
+public class DecimalUtility extends CoreDecimalUtility{
+
+ public final static int MAX_DIGITS = 9;
+ public final static int DIGITS_BASE = 1000000000;
+ public final static int DIGITS_MAX = 999999999;
+ public final static int INTEGER_SIZE = (Integer.SIZE/8);
+
+ public final static String[] decimalToString = {"",
+ "0",
+ "00",
+ "000",
+ "0000",
+ "00000",
+ "000000",
+ "0000000",
+ "00000000",
+ "000000000"};
+
+ public final static long[] scale_long_constants = {
+ 1,
+ 10,
+ 100,
+ 1000,
+ 10000,
+ 100000,
+ 1000000,
+ 10000000,
+ 100000000,
+ 1000000000,
+ 10000000000l,
+ 100000000000l,
+ 1000000000000l,
+ 10000000000000l,
+ 100000000000000l,
+ 1000000000000000l,
+ 10000000000000000l,
+ 100000000000000000l,
+ 1000000000000000000l};
+
+ /*
+ * Simple function that returns the static precomputed
+ * power of ten, instead of using Math.pow
+ */
+ public static long getPowerOfTen(int power) {
+ assert power >= 0 && power < scale_long_constants.length;
+ return scale_long_constants[(power)];
+ }
+
+ /*
+ * Math.pow returns a double and while multiplying with large digits
+ * in the decimal data type we encounter noise. So instead of multiplying
+ * with Math.pow we use the static constants to perform the multiplication
+ */
+ public static long adjustScaleMultiply(long input, int factor) {
+ int index = Math.abs(factor);
+ assert index >= 0 && index < scale_long_constants.length;
+ if (factor >= 0) {
+ return input * scale_long_constants[index];
+ } else {
+ return input / scale_long_constants[index];
+ }
+ }
+
+ public static long adjustScaleDivide(long input, int factor) {
+ int index = Math.abs(factor);
+ assert index >= 0 && index < scale_long_constants.length;
+ if (factor >= 0) {
+ return input / scale_long_constants[index];
+ } else {
+ return input * scale_long_constants[index];
+ }
+ }
+
+ /* Given the number of actual digits this function returns the
+ * number of indexes it will occupy in the array of integers
+ * which are stored in base 1 billion
+ */
+ public static int roundUp(int ndigits) {
+ return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+ }
+
+ /* Returns a string representation of the given integer
+ * If the length of the given integer is less than the
+ * passed length, this function will prepend zeroes to the string
+ */
+ public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
+ String value = ((Integer) number).toString();
+ int length = value.length();
+
+ StringBuilder str = new StringBuilder();
+ str.append(decimalToString[desiredLength - length]);
+ str.append(value);
+
+ return str;
+ }
+
+ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
+ String value = ((Long) number).toString();
+ int length = value.length();
+
+ StringBuilder str = new StringBuilder();
+
+ // Desired length can be > MAX_DIGITS
+ int zeroesLength = desiredLength - length;
+ while (zeroesLength > MAX_DIGITS) {
+ str.append(decimalToString[MAX_DIGITS]);
+ zeroesLength -= MAX_DIGITS;
+ }
+ str.append(decimalToString[zeroesLength]);
+ str.append(value);
+
+ return str;
+ }
+
+ public static BigDecimal getBigDecimalFromIntermediate(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
+
+ // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
+ return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false);
+ }
+
+ public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
+
+ // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
+ return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true);
+ }
+
+ public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) {
+ byte[] value = new byte[length];
+ bytebuf.getBytes(start, value, 0, length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
+
+ public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
+ byte[] value = new byte[length];
+ bytebuf.get(value);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
+
+ /* Create a BigDecimal object using the data in the DrillBuf.
+ * This function assumes that data is provided in a non-dense format
+ * It works on both sparse and intermediate representations.
+ */
+ public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) {
+
+ // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
+ int actualDigits;
+
+ // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out
+ BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
+
+ BigInteger base = BigInteger.valueOf(DIGITS_BASE);
+
+ for (int i = 1; i < nDecimalDigits; i++) {
+
+ BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
+ decimalDigits = decimalDigits.multiply(base);
+ decimalDigits = decimalDigits.add(temp);
+ }
+
+ // Truncate any additional padding we might have added
+ if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
+ BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
+ decimalDigits = decimalDigits.divide(truncate);
+ }
+
+ // set the sign
+ if ((data.getInt(startIndex) & 0x80000000) != 0) {
+ decimalDigits = decimalDigits.negate();
+ }
+
+ BigDecimal decimal = new BigDecimal(decimalDigits, scale);
+
+ return decimal;
+ }
+
+ /* This function returns a BigDecimal object from the dense decimal representation.
+ * First step is to convert the dense representation into an intermediate representation
+ * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object
+ */
+ public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
+
+ /* This method converts the dense representation to
+ * an intermediate representation. The intermediate
+ * representation has one more integer than the dense
+ * representation.
+ */
+ byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
+
+ // Start storing from the least significant byte of the first integer
+ int intermediateIndex = 3;
+
+ int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
+ int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
+
+ int maskIndex;
+ int shiftOrder;
+ byte shiftBits;
+
+ // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
+ if (maxPrecision == 38) {
+ maskIndex = 0;
+ shiftOrder = 6;
+ shiftBits = 0x00;
+ intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
+ } else if (maxPrecision == 28) {
+ maskIndex = 1;
+ shiftOrder = 4;
+ shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
+ intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
+ } else {
+ throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
+ }
+
+ int inputIndex = 1;
+ boolean sign = false;
+
+ if ((data.getByte(startIndex) & 0x80) != 0) {
+ sign = true;
+ }
+
+ while (inputIndex < width) {
+
+ intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
+
+ shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
+
+ inputIndex++;
+ intermediateIndex++;
+
+ if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
+ shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
+ maskIndex++;
+ shiftOrder -= 2;
+ }
+
+ }
+ /* copy the last byte */
+ intermediateBytes[intermediateIndex] = shiftBits;
+
+ if (sign == true) {
+ intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
+ }
+ DrillBuf intermediate = data.getAllocator().buffer(intermediateBytes.length);
+ intermediate.setBytes(0, intermediateBytes);
+
+ BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
+ intermediate.release();
+ return ret;
+ }
+
+ /*
+ * Function converts the BigDecimal and stores it in out internal sparse representation
+ */
+ public static void getSparseFromBigDecimal(BigDecimal input, DrillBuf data, int startIndex, int scale, int precision, int nDecimalDigits) {
+
+ // Initialize the buffer
+ for (int i = 0; i < nDecimalDigits; i++) {
+ data.setInt(startIndex + (i * INTEGER_SIZE), 0);
+ }
+
+ boolean sign = false;
+
+ if (input.signum() == -1) {
+ // negative input
+ sign = true;
+ input = input.abs();
+ }
+
+ // Truncate the input as per the scale provided
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+ // Separate out the integer part
+ BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
+
+ int destIndex = nDecimalDigits - roundUp(scale) - 1;
+
+ // we use base 1 billion integer digits for out integernal representation
+ BigDecimal base = new BigDecimal(DIGITS_BASE);
+
+ while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
+ // store the modulo as the integer value
+ data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
+ destIndex--;
+ // Divide by base 1 billion
+ integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
+ }
+
+ /* Sparse representation contains padding of additional zeroes
+ * so each digit contains MAX_DIGITS for ease of arithmetic
+ */
+ int actualDigits;
+ if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
+ // Pad additional zeroes
+ scale = scale + (MAX_DIGITS - actualDigits);
+ input = input.setScale(scale, BigDecimal.ROUND_DOWN);
+ }
+
+ //separate out the fractional part
+ BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
+
+ destIndex = nDecimalDigits - 1;
+
+ while (scale > 0) {
+ // Get next set of MAX_DIGITS (9) store it in the DrillBuf
+ fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
+ BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
+
+ data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
+ destIndex--;
+
+ fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
+ scale -= MAX_DIGITS;
+ }
+
+ // Set the negative sign
+ if (sign == true) {
+ data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
+ }
+
+ }
+
+
+ public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+ // Truncate or pad to set the input to the correct scale
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+ return (input.unscaledValue().longValue());
+ }
+
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
+ return BigDecimal.valueOf(input, scale);
+ }
+
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
+ return BigDecimal.valueOf(input, scale);
+ }
+
+
+ public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) {
+
+ int invert = 1;
+
+ /* If signs are different then simply look at the
+ * sign of the two inputs and determine which is greater
+ */
+ if (leftSign != rightSign) {
+
+ return((leftSign == true) ? -1 : 1);
+ } else if(leftSign == true) {
+ /* Both inputs are negative, at the end we will
+ * have to invert the comparison
+ */
+ invert = -1;
+ }
+
+ int cmp = 0;
+
+ for (int i = 0; i < width; i++) {
+ byte leftByte = left.getByte(leftStart + i);
+ byte rightByte = right.getByte(rightStart + i);
+ // Unsigned byte comparison
+ if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
+ cmp = 1;
+ break;
+ } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
+ cmp = -1;
+ break;
+ }
+ }
+ cmp *= invert; // invert the comparison if both were negative values
+
+ return cmp;
+ }
+
+ public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) {
+ int value = buffer.getInt(start + (index * 4));
+
+ if (index == 0) {
+ /* the first byte contains sign bit, return value without it */
+ value = (value & 0x7FFFFFFF);
+ }
+ return value;
+ }
+
+ public static void setInteger(DrillBuf buffer, int start, int index, int value) {
+ buffer.setInt(start + (index * 4), value);
+ }
+
+ public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
+
+ int invert = 1;
+
+ if (absCompare == false) {
+ if (leftSign != rightSign) {
+ return (leftSign == true) ? -1 : 1;
+ }
+
+ // Both values are negative invert the outcome of the comparison
+ if (leftSign == true) {
+ invert = -1;
+ }
+ }
+
+ int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
+ return cmp * invert;
+ }
+ public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
+ /* compute the number of integer digits in each decimal */
+ int leftInt = leftPrecision - leftScale;
+ int rightInt = rightPrecision - rightScale;
+
+ /* compute the number of indexes required for storing integer digits */
+ int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt);
+ int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt);
+
+ /* compute number of indexes required for storing scale */
+ int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale);
+ int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale);
+
+ /* compute index of the most significant integer digits */
+ int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+ int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+
+ int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
+ int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
+
+ /* Discard the zeroes in the integer part */
+ while (leftIndex1 < leftStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+ break;
+ }
+
+ /* Digit in this location is zero, decrement the actual number
+ * of integer digits
+ */
+ leftIntRoundedUp--;
+ leftIndex1++;
+ }
+
+ /* If we reached the stop index then the number of integers is zero */
+ if (leftIndex1 == leftStopIndex) {
+ leftIntRoundedUp = 0;
+ }
+
+ while (rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+ break;
+ }
+
+ /* Digit in this location is zero, decrement the actual number
+ * of integer digits
+ */
+ rightIntRoundedUp--;
+ rightIndex1++;
+ }
+
+ if (rightIndex1 == rightStopIndex) {
+ rightIntRoundedUp = 0;
+ }
+
+ /* We have the accurate number of non-zero integer digits,
+ * if the number of integer digits are different then we can determine
+ * which decimal is larger and needn't go down to comparing individual values
+ */
+ if (leftIntRoundedUp > rightIntRoundedUp) {
+ return 1;
+ }
+ else if (rightIntRoundedUp > leftIntRoundedUp) {
+ return -1;
+ }
+
+ /* The number of integer digits are the same, set the each index
+ * to the first non-zero integer and compare each digit
+ */
+ leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+ rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+
+ while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+ return 1;
+ }
+ else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+ return -1;
+ }
+
+ leftIndex1++;
+ rightIndex1++;
+ }
+
+ /* The integer part of both the decimal's are equal, now compare
+ * each individual fractional part. Set the index to be at the
+ * beginning of the fractional part
+ */
+ leftIndex1 = leftStopIndex;
+ rightIndex1 = rightStopIndex;
+
+ /* Stop indexes will be the end of the array */
+ leftStopIndex = nDecimalDigits;
+ rightStopIndex = nDecimalDigits;
+
+ /* compare the two fractional parts of the decimal */
+ while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+ return 1;
+ }
+ else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+ return -1;
+ }
+
+ leftIndex1++;
+ rightIndex1++;
+ }
+
+ /* Till now the fractional part of the decimals are equal, check
+ * if one of the decimal has fractional part that is remaining
+ * and is non-zero
+ */
+ while (leftIndex1 < leftStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+ return 1;
+ }
+ leftIndex1++;
+ }
+
+ while(rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+ return -1;
+ }
+ rightIndex1++;
+ }
+
+ /* Both decimal values are equal */
+ return 0;
+ }
+
+ public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
+ byte[] value = Arrays.copyOfRange(bytes, start, start + length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
+
+ public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
+ int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale);
+ int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale);
+
+ if (desiredScale < currentScale) {
+
+ boolean roundUp = false;
+
+ //Extract the first digit to be truncated to check if we need to round up
+ int truncatedScaleIndex = desiredScale + 1;
+ if (truncatedScaleIndex <= currentScale) {
+ int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
+ extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex);
+ int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
+ int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
+ if (temp != 0) {
+ extractDigit = extractDigit / (int) (Math.pow(10, temp));
+ }
+ if ((extractDigit % 10) > 4) {
+ roundUp = true;
+ }
+ }
+
+ // Get the source index beyond which we will truncate
+ int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
+ int srcIndex = srcIntIndex + newScaleRoundedUp;
+
+ // Truncate the remaining fractional part, move the integer part
+ int destIndex = nDecimalDigits - 1;
+ if (srcIndex != destIndex) {
+ while (srcIndex >= 0) {
+ setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
+ }
+
+ // Set the remaining portion of the decimal to be zeroes
+ while (destIndex >= 0) {
+ setInteger(result, start, destIndex--, 0);
+ }
+ srcIndex = nDecimalDigits - 1;
+ }
+
+ // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
+ int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
+ if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
+ truncateFactor = (int) Math.pow(10, truncateFactor);
+ int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
+ fractionalDigits /= truncateFactor;
+ setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
+ }
+
+ // Finally round up the digit if needed
+ if (roundUp == true) {
+ srcIndex = nDecimalDigits - 1;
+ int carry;
+ if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
+ carry = truncateFactor;
+ } else {
+ carry = 1;
+ }
+
+ while (srcIndex >= 0) {
+ int value = getIntegerFromSparseBuffer(result, start, srcIndex);
+ value += carry;
+
+ if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
+ carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
+ } else {
+ setInteger(result, start, srcIndex--, value);
+ carry = 0;
+ break;
+ }
+ }
+ }
+ } else if (desiredScale > currentScale) {
+ // Add fractional digits to the decimal
+
+ // Check if we need to shift the decimal digits to the left
+ if (newScaleRoundedUp > origScaleRoundedUp) {
+ int srcIndex = 0;
+ int destIndex = newScaleRoundedUp - origScaleRoundedUp;
+
+ // Check while extending scale, we are not overwriting integer part
+ while (srcIndex < destIndex) {
+ if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
+ throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
+ }
+ }
+
+ srcIndex = 0;
+ while (destIndex < nDecimalDigits) {
+ setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
+ }
+
+ // Clear the remaining part
+ while (srcIndex < nDecimalDigits) {
+ setInteger(result, start, srcIndex++, 0);
+ }
+ }
+ }
+ }
+
+ public static int getFirstFractionalDigit(int decimal, int scale) {
+ if (scale == 0) {
+ return 0;
+ }
+ int temp = (int) adjustScaleDivide(decimal, scale - 1);
+ return Math.abs(temp % 10);
+ }
+
+ public static int getFirstFractionalDigit(long decimal, int scale) {
+ if (scale == 0) {
+ return 0;
+ }
+ long temp = adjustScaleDivide(decimal, scale - 1);
+ return (int) (Math.abs(temp % 10));
+ }
+
+ public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) {
+ if (scale == 0) {
+ return 0;
+ }
+
+ int index = nDecimalDigits - roundUp(scale);
+ return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1));
+ }
+
+ public static int compareSparseSamePrecScale(DrillBuf left, int lStart, byte[] right, int length) {
+ // check the sign first
+ boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
+ boolean rSign = ByteFunctionHelpers.getSign(right);
+ int cmp = 0;
+
+ if (lSign != rSign) {
+ return (lSign == false) ? 1 : -1;
+ }
+
+ // invert the comparison if we are comparing negative numbers
+ int invert = (lSign == true) ? -1 : 1;
+
+ // compare byte by byte
+ int n = 0;
+ int lPos = lStart;
+ int rPos = 0;
+ while (n < length/4) {
+ int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
+ int rightInt = ByteFunctionHelpers.getInteger(right, n);
+ if (leftInt != rightInt) {
+ cmp = (leftInt - rightInt ) > 0 ? 1 : -1;
+ break;
+ }
+ n++;
+ }
+ return cmp * invert;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
index 622e2d2..77d457b 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
@@ -49,7 +49,7 @@ public class AllocationHelper {
* Allocates the exact amount if v is fixed width, otherwise falls back to dynamic allocation
* @param v value vector we are trying to allocate
* @param valueCount size we are trying to allocate
- * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the memory
+ * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the memory
*/
public static void allocateNew(ValueVector v, int valueCount) {
if (v instanceof FixedWidthVector) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 67ba5c4..856393d 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -25,7 +25,7 @@ import org.apache.drill.exec.exception.OversizedAllocationException;
import org.apache.drill.exec.expr.holders.BitHolder;
import org.apache.drill.exec.expr.holders.NullableBitHolder;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
@@ -88,7 +88,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
@Override
public void allocateNew() {
if (!allocateNewSafe()) {
- throw new OutOfMemoryRuntimeException();
+ throw new OutOfMemoryException();
}
}
@@ -105,7 +105,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
try {
allocateBytes(curAllocationSize);
- } catch (OutOfMemoryRuntimeException ex) {
+ } catch (OutOfMemoryException ex) {
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
index a7a3609..9ca4410 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.exec.expr.holders.ObjectHolder;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
@@ -103,11 +103,11 @@ public class ObjectVector extends BaseValueVector {
}
@Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
+ public void allocateNew() throws OutOfMemoryException {
addNewArray();
}
- public void allocateNew(int valueCount) throws OutOfMemoryRuntimeException {
+ public void allocateNew(int valueCount) throws OutOfMemoryException {
while (maxCount < valueCount) {
addNewArray();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
index 42384cb..54897d3 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
@@ -177,8 +177,9 @@ public class ValueHolderHelper {
dch.precision = bigDecimal.precision();
Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
dch.start = 0;
- dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.integerSize);
- DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+ dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE);
+ DecimalUtility
+ .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
return dch;
}
@@ -193,8 +194,9 @@ public class ValueHolderHelper {
dch.precision = bigDecimal.precision();
Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
dch.start = 0;
- dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.integerSize);
- DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+ dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE);
+ DecimalUtility
+ .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
return dch;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 8627d1a..b39fcfe 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -22,7 +22,7 @@ import java.io.Closeable;
import io.netty.buffer.DrillBuf;
import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
@@ -58,9 +58,9 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
public interface ValueVector extends Closeable, Iterable<ValueVector> {
/**
* Allocate new buffers. ValueVector implements logic to determine how much to allocate.
- * @throws OutOfMemoryRuntimeException Thrown if no memory can be allocated.
+ * @throws OutOfMemoryException Thrown if no memory can be allocated.
*/
- void allocateNew() throws OutOfMemoryRuntimeException;
+ void allocateNew() throws OutOfMemoryException;
/**
* Allocates new buffers. ValueVector implements logic to determine how much to allocate.
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
index 9fb151f..c5326f6 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
@@ -24,7 +24,7 @@ import io.netty.buffer.DrillBuf;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
@@ -128,7 +128,7 @@ public class ZeroVector implements ValueVector {
}
@Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
+ public void allocateNew() throws OutOfMemoryException {
allocateNewSafe();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
index 5f6d0d2..290b237 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -34,7 +34,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.util.CallBack;
@@ -59,9 +59,9 @@ public abstract class AbstractContainerVector implements ValueVector {
}
@Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
+ public void allocateNew() throws OutOfMemoryException {
if (!allocateNewSafe()) {
- throw new OutOfMemoryRuntimeException();
+ throw new OutOfMemoryException();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
index 35df691..581a54d 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
@@ -17,20 +17,22 @@
*/
package org.apache.drill.exec.vector.complex;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
import io.netty.buffer.DrillBuf;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.drill.common.collections.MapWithOrdinal;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.util.CallBack;
import org.apache.drill.exec.vector.ValueVector;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
/*
* Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
@@ -48,7 +50,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
for (MaterializedField child : clonedField.getChildren()) {
if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
final String fieldName = child.getLastName();
- final ValueVector v = TypeHelper.getNewVector(child, allocator, callBack);
+ final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
putVector(fieldName, v);
}
}
@@ -128,7 +130,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
create = true;
}
if (create) {
- final T vector = (T) TypeHelper.getNewVector(field.getPath(), name, allocator, type, callBack);
+ final T vector = (T) BasicTypeHelper.getNewVector(field.getPath(), name, allocator, type, callBack);
putChild(name, vector);
if (callBack!=null) {
callBack.doWork();
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
index de4ad95..bc90eda 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
@@ -17,16 +17,15 @@
*/
package org.apache.drill.exec.vector.complex;
+import io.netty.buffer.DrillBuf;
+
import java.util.Collections;
import java.util.Iterator;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-import io.netty.buffer.DrillBuf;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.SchemaChangeRuntimeException;
-import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.MaterializedField;
@@ -37,6 +36,9 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VectorDescriptor;
import org.apache.drill.exec.vector.ZeroVector;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -185,7 +187,7 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
boolean created = false;
if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) {
final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
- vector = TypeHelper.getNewVector(field, allocator);
+ vector = BasicTypeHelper.getNewVector(field, allocator);
// returned vector must have the same field
assert field.equals(vector.getField());
getField().addChild(field);
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
index e5d9443..b780d1a 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
@@ -26,7 +26,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
@@ -72,7 +72,7 @@ public class ListVector extends BaseRepeatedValueVector {
}
@Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
+ public void allocateNew() throws OutOfMemoryException {
super.allocateNewSafe();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 048358c..60d74c1 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -17,9 +17,6 @@
*/
package org.apache.drill.exec.vector.complex;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
-
import io.netty.buffer.DrillBuf;
import java.util.Collection;
@@ -34,7 +31,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.exec.expr.holders.ComplexHolder;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
@@ -49,6 +46,8 @@ import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
public class MapVector extends AbstractMapVector {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
@@ -269,7 +268,7 @@ public class MapVector extends AbstractMapVector {
ValueVector vector = getChild(fieldDef.getLastName());
if (vector == null) {
// if we arrive here, we didn't have a matching vector.
- vector = TypeHelper.getNewVector(fieldDef, allocator);
+ vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
putChild(fieldDef.getLastName(), vector);
}
if (child.getValueCount() == 0) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index 330eed4..cbc61f8 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -32,7 +32,7 @@ import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.holders.ComplexHolder;
import org.apache.drill.exec.expr.holders.RepeatedListHolder;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
@@ -182,9 +182,9 @@ public class RepeatedListVector extends AbstractContainerVector
}
@Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
+ public void allocateNew() throws OutOfMemoryException {
if (!allocateNewSafe()) {
- throw new OutOfMemoryRuntimeException();
+ throw new OutOfMemoryException();
}
}
@@ -300,7 +300,7 @@ public class RepeatedListVector extends AbstractContainerVector
}
@Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
+ public void allocateNew() throws OutOfMemoryException {
delegate.allocateNew();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 618e3a3..cb597be 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -31,11 +31,11 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.exec.expr.holders.ComplexHolder;
import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
@@ -108,7 +108,7 @@ public class RepeatedMapVector extends AbstractMapVector
for (ValueVector v : getChildren()) {
AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
}
- } catch (OutOfMemoryRuntimeException e){
+ } catch (OutOfMemoryException e){
clear();
throw e;
}
@@ -458,7 +458,7 @@ public class RepeatedMapVector extends AbstractMapVector
ValueVector vector = getChild(fieldDef.getLastName());
if (vector == null) {
// if we arrive here, we didn't have a matching vector.
- vector = TypeHelper.getNewVector(fieldDef, allocator);
+ vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
putChild(fieldDef.getLastName(), vector);
}
final int vectorLength = child.getBufferLength();
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
index 5575b5e..468c0d5 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
@@ -17,12 +17,9 @@
*/
package org.apache.drill.exec.vector.complex.impl;
-import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.VectorContainer;
-import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.MapVector;
import org.apache.drill.exec.vector.complex.StateTool;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
@@ -192,27 +189,5 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
return listRoot;
}
- private static class VectorAccessibleFacade extends MapVector {
- private final VectorContainer vc;
-
- public VectorAccessibleFacade(VectorContainer vc) {
- super("", null, null);
- this.vc = vc;
- }
-
- @Override
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
- final ValueVector v = vc.addOrGet(name, type, clazz);
- putChild(name, v);
- return this.typeify(v, clazz);
-
- }
-
- }
-
- public static ComplexWriter getWriter(String name, VectorContainer container){
- VectorAccessibleFacade vc = new VectorAccessibleFacade(container);
- return new ComplexWriterImpl(name, vc);
- }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
index 5f57c01..894e60e 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
@@ -17,10 +17,12 @@
*/
package org.apache.drill.exec.vector.complex.impl;
+import java.lang.reflect.Constructor;
+
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.vector.ValueVector;
@@ -31,8 +33,6 @@ import org.apache.drill.exec.vector.complex.ListVector;
import org.apache.drill.exec.vector.complex.UnionVector;
import org.apache.drill.exec.vector.complex.writer.FieldWriter;
-import java.lang.reflect.Constructor;
-
/**
* This FieldWriter implementation delegates all FieldWriter API calls to an inner FieldWriter. This inner field writer
* can start as a specific type, and this class will promote the writer to a UnionWriter if a call is made that the specifically
@@ -85,11 +85,13 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
state = State.SINGLE;
vector = v;
type = v.getField().getType().getMinorType();
- Class writerClass = TypeHelper.getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
+ Class writerClass = BasicTypeHelper
+ .getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
if (writerClass.equals(SingleListWriter.class)) {
writerClass = UnionListWriter.class;
}
- Class vectorClass = TypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField().getDataMode());
+ Class vectorClass = BasicTypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField()
+ .getDataMode());
try {
Constructor constructor = null;
for (Constructor c : writerClass.getConstructors()) {
[25/45] drill git commit: DRILL-3987: (MOVE) Extract RPC,
memory-base and memory-impl as separate modules.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
new file mode 100644
index 0000000..38eb622
--- /dev/null
+++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
@@ -0,0 +1,749 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.nio.charset.Charset;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.drill.exec.memory.Accountor;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.BufferManager;
+import org.apache.drill.exec.util.AssertionUtil;
+
+import com.google.common.base.Preconditions;
+
+public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillBuf.class);
+
+ private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
+
+ private final ByteBuf b;
+ private final long addr;
+ private final int offset;
+ private final boolean rootBuffer;
+ private final AtomicLong rootRefCnt = new AtomicLong(1);
+ private volatile BufferAllocator allocator;
+ private volatile Accountor acct;
+ private volatile int length;
+
+ // TODO - cleanup
+ // The code is partly shared and partly copy-pasted between
+ // these three types. They should be unified under one interface
+ // to share code and to remove the hacky code here to use only
+ // one of these types at a time and use null checks to find out
+ // which.
+ private BufferManager bufManager;
+
+ public DrillBuf(BufferAllocator allocator, Accountor a, UnsafeDirectLittleEndian b) {
+ super(b.maxCapacity());
+ this.b = b;
+ this.addr = b.memoryAddress();
+ this.acct = a;
+ this.length = b.capacity();
+ this.offset = 0;
+ this.rootBuffer = true;
+ this.allocator = allocator;
+ }
+
+ private DrillBuf(BufferAllocator allocator, Accountor a) {
+ super(0);
+ this.b = new EmptyByteBuf(allocator.getUnderlyingAllocator()).order(ByteOrder.LITTLE_ENDIAN);
+ this.allocator = allocator;
+ this.acct = a;
+ this.length = 0;
+ this.addr = 0;
+ this.rootBuffer = false;
+ this.offset = 0;
+ }
+
+ /**
+ * Special constructor used for RPC ownership transfer. Takes a snapshot slice of the current buf
+ * but points directly to the underlying UnsafeLittleEndian buffer. Does this by calling unwrap()
+ * twice on the provided DrillBuf and expecting an UnsafeDirectLittleEndian buffer. This operation
+ * includes taking a new reference count on the underlying buffer and maintaining returning with a
+ * current reference count for itself (masking the underlying reference count).
+ * @param allocator
+ * @param a Allocator used when users try to receive allocator from buffer.
+ * @param b Accountor used for accounting purposes.
+ */
+ public DrillBuf(BufferAllocator allocator, Accountor a, DrillBuf b) {
+ this(allocator, a, getUnderlying(b), b, 0, b.length, true);
+ assert b.unwrap().unwrap() instanceof UnsafeDirectLittleEndian;
+ b.unwrap().unwrap().retain();
+ }
+
+
+ private DrillBuf(DrillBuf buffer, int index, int length) {
+ this(buffer.allocator, null, buffer, buffer, index, length, false);
+ }
+
+ private static ByteBuf getUnderlying(DrillBuf b){
+ ByteBuf underlying = b.unwrap().unwrap();
+ return underlying.slice((int) (b.memoryAddress() - underlying.memoryAddress()), b.length);
+ }
+ private DrillBuf(BufferAllocator allocator, Accountor a, ByteBuf replacement, DrillBuf buffer, int index, int length, boolean root) {
+ super(length);
+ if (index < 0 || index > buffer.capacity() - length) {
+ throw new IndexOutOfBoundsException(buffer.toString() + ".slice(" + index + ", " + length + ')');
+ }
+
+ this.length = length;
+ writerIndex(length);
+
+ this.b = replacement;
+ this.addr = buffer.memoryAddress() + index;
+ this.offset = index;
+ this.acct = a;
+ this.length = length;
+ this.rootBuffer = root;
+ this.allocator = allocator;
+ }
+
+ public void setBufferManager(BufferManager bufManager) {
+ this.bufManager = bufManager;
+ }
+
+ public BufferAllocator getAllocator() {
+ return allocator;
+ }
+
+ public DrillBuf reallocIfNeeded(int size) {
+ if (this.capacity() >= size) {
+ return this;
+ }
+
+ if (bufManager != null) {
+ return bufManager.replace(this, size);
+ } else {
+ throw new UnsupportedOperationException("Realloc is only available in the context of an operator's UDFs");
+ }
+
+ }
+
+ @Override
+ public int refCnt() {
+ if(rootBuffer){
+ return (int) this.rootRefCnt.get();
+ }else{
+ return b.refCnt();
+ }
+
+ }
+
+ private long addr(int index) {
+ return addr + index;
+ }
+
+ private final void checkIndexD(int index) {
+ ensureAccessible();
+ if (index < 0 || index >= capacity()) {
+ throw new IndexOutOfBoundsException(String.format(
+ "index: %d (expected: range(0, %d))", index, capacity()));
+ }
+ }
+
+ private final void checkIndexD(int index, int fieldLength) {
+ ensureAccessible();
+ if (fieldLength < 0) {
+ throw new IllegalArgumentException("length: " + fieldLength + " (expected: >= 0)");
+ }
+ if (index < 0 || index > capacity() - fieldLength) {
+ throw new IndexOutOfBoundsException(String.format(
+ "index: %d, length: %d (expected: range(0, %d))", index, fieldLength, capacity()));
+ }
+ }
+
+ /**
+ * Allows a function to determine whether not reading a particular string of bytes is valid.
+ *
+ * Will throw an exception if the memory is not readable for some reason. Only doesn't something in the
+ * case that AssertionUtil.BOUNDS_CHECKING_ENABLED is true.
+ *
+ * @param start The starting position of the bytes to be read.
+ * @param end The exclusive endpoint of the bytes to be read.
+ */
+ public void checkBytes(int start, int end){
+ if (BOUNDS_CHECKING_ENABLED) {
+ checkIndexD(start, end - start);
+ }
+ }
+
+ private void chk(int index, int width) {
+ if (BOUNDS_CHECKING_ENABLED) {
+ checkIndexD(index, width);
+ }
+ }
+
+ private void chk(int index) {
+ if (BOUNDS_CHECKING_ENABLED) {
+ checkIndexD(index);
+ }
+ }
+
+ private void ensure(int width) {
+ if (BOUNDS_CHECKING_ENABLED) {
+ ensureWritable(width);
+ }
+ }
+
+ public boolean transferAccounting(Accountor target) {
+ if (rootBuffer) {
+ boolean outcome = acct.transferTo(target, this, length);
+ acct = target;
+ return outcome;
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public synchronized boolean release() {
+ return release(1);
+ }
+
+ /**
+ * Release the provided number of reference counts. If this is a root buffer, will decrease accounting if the local reference count returns to zero.
+ */
+ @Override
+ public synchronized boolean release(int decrement) {
+
+ if(rootBuffer){
+ final long newRefCnt = this.rootRefCnt.addAndGet(-decrement);
+ Preconditions.checkArgument(newRefCnt > -1, "Buffer has negative reference count.");
+ if (newRefCnt == 0) {
+ b.release(decrement);
+ acct.release(this, length);
+ return true;
+ }else{
+ return false;
+ }
+ }else{
+ return b.release(decrement);
+ }
+ }
+
+ @Override
+ public int capacity() {
+ return length;
+ }
+
+ @Override
+ public synchronized ByteBuf capacity(int newCapacity) {
+ if (rootBuffer) {
+ if (newCapacity == length) {
+ return this;
+ } else if (newCapacity < length) {
+ b.capacity(newCapacity);
+ int diff = length - b.capacity();
+ acct.releasePartial(this, diff);
+ this.length = length - diff;
+ return this;
+ } else {
+ throw new UnsupportedOperationException("Accounting byte buf doesn't support increasing allocations.");
+ }
+ } else {
+ throw new UnsupportedOperationException("Non root bufs doen't support changing allocations.");
+ }
+ }
+
+ @Override
+ public int maxCapacity() {
+ return length;
+ }
+
+ @Override
+ public ByteBufAllocator alloc() {
+ return b.alloc();
+ }
+
+ @Override
+ public ByteOrder order() {
+ return ByteOrder.LITTLE_ENDIAN;
+ }
+
+ @Override
+ public ByteBuf order(ByteOrder endianness) {
+ // if(endianness != ByteOrder.LITTLE_ENDIAN) throw new
+ // UnsupportedOperationException("Drill buffers only support little endian.");
+ return this;
+ }
+
+ @Override
+ public ByteBuf unwrap() {
+ return b;
+ }
+
+ @Override
+ public boolean isDirect() {
+ return true;
+ }
+
+ @Override
+ public ByteBuf readBytes(int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf readSlice(int length) {
+ ByteBuf slice = slice(readerIndex(), length);
+ readerIndex(readerIndex() + length);
+ return slice;
+ }
+
+ @Override
+ public ByteBuf copy() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf copy(int index, int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf slice() {
+ return slice(readerIndex(), readableBytes());
+ }
+
+ @Override
+ public DrillBuf slice(int index, int length) {
+ DrillBuf buf = new DrillBuf(this, index, length);
+ buf.writerIndex = length;
+ return buf;
+ }
+
+ @Override
+ public DrillBuf duplicate() {
+ return new DrillBuf(this, 0, length);
+ }
+
+ @Override
+ public int nioBufferCount() {
+ return 1;
+ }
+
+ @Override
+ public ByteBuffer nioBuffer() {
+ return nioBuffer(readerIndex(), readableBytes());
+ }
+
+ @Override
+ public ByteBuffer nioBuffer(int index, int length) {
+ return b.nioBuffer(offset + index, length);
+ }
+
+ @Override
+ public ByteBuffer internalNioBuffer(int index, int length) {
+ return b.internalNioBuffer(offset + index, length);
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers() {
+ return new ByteBuffer[]{nioBuffer()};
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers(int index, int length) {
+ return new ByteBuffer[]{nioBuffer(index, length)};
+ }
+
+ @Override
+ public boolean hasArray() {
+ return b.hasArray();
+ }
+
+ @Override
+ public byte[] array() {
+ return b.array();
+ }
+
+ @Override
+ public int arrayOffset() {
+ return b.arrayOffset();
+ }
+
+ @Override
+ public boolean hasMemoryAddress() {
+ return true;
+ }
+
+ @Override
+ public long memoryAddress() {
+ return this.addr;
+ }
+
+ @Override
+ public String toString(Charset charset) {
+ return toString(readerIndex, readableBytes(), charset);
+ }
+
+ @Override
+ public String toString(int index, int length, Charset charset) {
+ if (length == 0) {
+ return "";
+ }
+
+ ByteBuffer nioBuffer;
+ if (nioBufferCount() == 1) {
+ nioBuffer = nioBuffer(index, length);
+ } else {
+ nioBuffer = ByteBuffer.allocate(length);
+ getBytes(index, nioBuffer);
+ nioBuffer.flip();
+ }
+
+ return ByteBufUtil.decodeString(nioBuffer, charset);
+ }
+
+ @Override
+ public int hashCode() {
+ return System.identityHashCode(this);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // identity equals only.
+ return this == obj;
+ }
+
+ @Override
+ public synchronized ByteBuf retain(int increment) {
+ if(rootBuffer){
+ this.rootRefCnt.addAndGet(increment);
+ }else{
+ b.retain(increment);
+ }
+ return this;
+ }
+
+ @Override
+ public ByteBuf retain() {
+ return retain(1);
+ }
+
+ @Override
+ public long getLong(int index) {
+ chk(index, 8);
+ long v = PlatformDependent.getLong(addr(index));
+ return v;
+ }
+
+ @Override
+ public float getFloat(int index) {
+ return Float.intBitsToFloat(getInt(index));
+ }
+
+ @Override
+ public double getDouble(int index) {
+ return Double.longBitsToDouble(getLong(index));
+ }
+
+ @Override
+ public char getChar(int index) {
+ return (char) getShort(index);
+ }
+
+ @Override
+ public long getUnsignedInt(int index) {
+ return getInt(index) & 0xFFFFFFFFL;
+ }
+
+ @Override
+ public int getInt(int index) {
+ chk(index, 4);
+ int v = PlatformDependent.getInt(addr(index));
+ return v;
+ }
+
+ @Override
+ public int getUnsignedShort(int index) {
+ return getShort(index) & 0xFFFF;
+ }
+
+ @Override
+ public short getShort(int index) {
+ chk(index, 2);
+ short v = PlatformDependent.getShort(addr(index));
+ return v;
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ chk(index, 2);
+ PlatformDependent.putShort(addr(index), (short) value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ chk(index, 4);
+ PlatformDependent.putInt(addr(index), value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ chk(index, 8);
+ PlatformDependent.putLong(addr(index), value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setChar(int index, int value) {
+ chk(index, 2);
+ PlatformDependent.putShort(addr(index), (short) value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setFloat(int index, float value) {
+ chk(index, 4);
+ PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf setDouble(int index, double value) {
+ chk(index, 8);
+ PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeShort(int value) {
+ ensure(2);
+ PlatformDependent.putShort(addr(writerIndex), (short) value);
+ writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeInt(int value) {
+ ensure(4);
+ PlatformDependent.putInt(addr(writerIndex), value);
+ writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeLong(long value) {
+ ensure(8);
+ PlatformDependent.putLong(addr(writerIndex), value);
+ writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeChar(int value) {
+ ensure(2);
+ PlatformDependent.putShort(addr(writerIndex), (short) value);
+ writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeFloat(float value) {
+ ensure(4);
+ PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value));
+ writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeDouble(double value) {
+ ensure(8);
+ PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value));
+ writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+ b.getBytes(index + offset, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuffer dst) {
+ b.getBytes(index + offset, dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ chk(index, 1);
+ PlatformDependent.putByte(addr(index), (byte) value);
+ return this;
+ }
+
+ public void setByte(int index, byte b){
+ chk(index, 1);
+ PlatformDependent.putByte(addr(index), b);
+ }
+
+ public void writeByteUnsafe(byte b){
+ PlatformDependent.putByte(addr(readerIndex), b);
+ readerIndex++;
+ }
+
+ @Override
+ protected byte _getByte(int index) {
+ return getByte(index);
+ }
+
+ @Override
+ protected short _getShort(int index) {
+ return getShort(index);
+ }
+
+ @Override
+ protected int _getInt(int index) {
+ return getInt(index);
+ }
+
+ @Override
+ protected long _getLong(int index) {
+ return getLong(index);
+ }
+
+ @Override
+ protected void _setByte(int index, int value) {
+ setByte(index, value);
+ }
+
+ @Override
+ protected void _setShort(int index, int value) {
+ setShort(index, value);
+ }
+
+ @Override
+ protected void _setMedium(int index, int value) {
+ setMedium(index, value);
+ }
+
+ @Override
+ protected void _setInt(int index, int value) {
+ setInt(index, value);
+ }
+
+ @Override
+ protected void _setLong(int index, long value) {
+ setLong(index, value);
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+ b.getBytes(index + offset, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
+ b.getBytes(index + offset, out, length);
+ return this;
+ }
+
+ @Override
+ protected int _getUnsignedMedium(int index) {
+ long addr = addr(index);
+ return (PlatformDependent.getByte(addr) & 0xff) << 16 |
+ (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
+ PlatformDependent.getByte(addr + 2) & 0xff;
+ }
+
+ @Override
+ public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
+ return b.getBytes(index + offset, out, length);
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+ b.setBytes(index + offset, src, srcIndex, length);
+ return this;
+ }
+
+ public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
+ if (src.isDirect()) {
+ checkIndex(index, length);
+ PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this.memoryAddress() + index,
+ length);
+ } else {
+ if (srcIndex == 0 && src.capacity() == length) {
+ b.setBytes(index + offset, src);
+ } else {
+ ByteBuffer newBuf = src.duplicate();
+ newBuf.position(srcIndex);
+ newBuf.limit(srcIndex + length);
+ b.setBytes(index + offset, src);
+ }
+ }
+
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+ b.setBytes(index + offset, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuffer src) {
+ b.setBytes(index + offset, src);
+ return this;
+ }
+
+ @Override
+ public int setBytes(int index, InputStream in, int length) throws IOException {
+ return b.setBytes(index + offset, in, length);
+ }
+
+ @Override
+ public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
+ return b.setBytes(index + offset, in, length);
+ }
+
+ @Override
+ public byte getByte(int index) {
+ chk(index, 1);
+ return PlatformDependent.getByte(addr(index));
+ }
+
+ public static DrillBuf getEmpty(BufferAllocator allocator, Accountor a) {
+ return new DrillBuf(allocator, a);
+ }
+
+ public boolean isRootBuffer() {
+ return rootBuffer;
+ }
+
+ @Override
+ public void close() {
+ release();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/io/netty/buffer/FakeAllocator.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/FakeAllocator.java b/exec/memory/base/src/main/java/io/netty/buffer/FakeAllocator.java
new file mode 100644
index 0000000..53ca91c
--- /dev/null
+++ b/exec/memory/base/src/main/java/io/netty/buffer/FakeAllocator.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.memory.Accountor;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.LimitConsumer;
+import org.apache.drill.exec.util.Pointer;
+
+class FakeAllocator implements BufferAllocator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FakeAllocator.class);
+
+
+ public static final Accountor FAKE_ACCOUNTOR = new FakeAccountor();
+ public static final BufferAllocator FAKE_ALLOCATOR = new FakeAllocator();
+
+ @Override
+ public DrillBuf buffer(int size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DrillBuf buffer(int minSize, int maxSize) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBufAllocator getUnderlyingAllocator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BufferAllocator getChildAllocator(LimitConsumer consumer, long initialReservation, long maximumReservation,
+ boolean applyFragmentLimit)
+ throws OutOfMemoryException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DrillBuf getEmpty() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public PreAllocator getNewPreAllocator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void resetLimits() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setLimit(long l) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getLimit(){
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public long getAllocatedMemory() {
+ return 0;
+ }
+
+ @Override
+ public long getPeakMemoryAllocation() {
+ return 0;
+ }
+
+ static class FakeAccountor implements Accountor {
+
+ public FakeAccountor() {
+ }
+
+ @Override
+ public boolean transferTo(Accountor target, DrillBuf buf, long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getAvailable() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getCapacity() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getAllocation() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean reserve(long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean forceAdditionalReservation(long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void reserved(long expected, DrillBuf buf) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void releasePartial(DrillBuf buf, long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void release(DrillBuf buf, long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean transferIn(DrillBuf buf, long size) {
+ return false;
+ }
+
+ @Override
+ public long getPeakMemoryAllocation() {
+ return 0;
+ }
+
+ @Override
+ public long resetFragmentLimits() {
+ return 0;
+ }
+
+ @Override
+ public void setFragmentLimit(long add) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getFragmentLimit() {
+ return 0;
+ }
+
+
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut) {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/io/netty/buffer/LargeBuffer.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/LargeBuffer.java b/exec/memory/base/src/main/java/io/netty/buffer/LargeBuffer.java
new file mode 100644
index 0000000..f1d4842
--- /dev/null
+++ b/exec/memory/base/src/main/java/io/netty/buffer/LargeBuffer.java
@@ -0,0 +1,350 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This is basically a complete copy of DuplicatedByteBuf. We copy because we can't override the release methods to keep
+ * global track of created Large Buffers.
+ */
+public class LargeBuffer extends AbstractByteBuf {
+
+ private final AtomicLong hugeBufferSize;
+ private final AtomicLong hugeBufferCount;
+
+ @Override
+ public ByteBuffer nioBuffer(int index, int length) {
+ return unwrap().nioBuffer(index, length);
+ }
+
+ private final ByteBuf buffer;
+ private final int initCap;
+
+ public LargeBuffer(ByteBuf buffer, AtomicLong hugeBufferSize, AtomicLong hugeBufferCount) {
+ super(buffer.maxCapacity());
+ initCap = buffer.capacity();
+ this.hugeBufferCount = hugeBufferCount;
+ this.hugeBufferSize = hugeBufferSize;
+
+ if (buffer instanceof LargeBuffer) {
+ this.buffer = ((LargeBuffer) buffer).buffer;
+ } else {
+ this.buffer = buffer;
+ }
+
+ setIndex(buffer.readerIndex(), buffer.writerIndex());
+ }
+
+ @Override
+ public ByteBuf unwrap() {
+ return buffer;
+ }
+
+ @Override
+ public ByteBufAllocator alloc() {
+ return buffer.alloc();
+ }
+
+ @Override
+ public ByteOrder order() {
+ return buffer.order();
+ }
+
+ @Override
+ public boolean isDirect() {
+ return buffer.isDirect();
+ }
+
+ @Override
+ public int capacity() {
+ return buffer.capacity();
+ }
+
+ @Override
+ public ByteBuf capacity(int newCapacity) {
+ buffer.capacity(newCapacity);
+ return this;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return buffer.hasArray();
+ }
+
+ @Override
+ public byte[] array() {
+ return buffer.array();
+ }
+
+ @Override
+ public int arrayOffset() {
+ return buffer.arrayOffset();
+ }
+
+ @Override
+ public boolean hasMemoryAddress() {
+ return buffer.hasMemoryAddress();
+ }
+
+ @Override
+ public long memoryAddress() {
+ return buffer.memoryAddress();
+ }
+
+ @Override
+ public byte getByte(int index) {
+ return _getByte(index);
+ }
+
+ @Override
+ protected byte _getByte(int index) {
+ return buffer.getByte(index);
+ }
+
+ @Override
+ public short getShort(int index) {
+ return _getShort(index);
+ }
+
+ @Override
+ protected short _getShort(int index) {
+ return buffer.getShort(index);
+ }
+
+ @Override
+ public int getUnsignedMedium(int index) {
+ return _getUnsignedMedium(index);
+ }
+
+ @Override
+ protected int _getUnsignedMedium(int index) {
+ return buffer.getUnsignedMedium(index);
+ }
+
+ @Override
+ public int getInt(int index) {
+ return _getInt(index);
+ }
+
+ @Override
+ protected int _getInt(int index) {
+ return buffer.getInt(index);
+ }
+
+ @Override
+ public long getLong(int index) {
+ return _getLong(index);
+ }
+
+ @Override
+ protected long _getLong(int index) {
+ return buffer.getLong(index);
+ }
+
+ @Override
+ public ByteBuf copy(int index, int length) {
+ return new LargeBuffer(buffer.copy(index, length), hugeBufferSize, hugeBufferCount);
+ }
+
+ @Override
+ public ByteBuf slice(int index, int length) {
+ return new SlicedByteBuf(this, index, length);
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+ buffer.getBytes(index, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+ buffer.getBytes(index, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuffer dst) {
+ buffer.getBytes(index, dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ _setByte(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setByte(int index, int value) {
+ buffer.setByte(index, value);
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ _setShort(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setShort(int index, int value) {
+ buffer.setShort(index, value);
+ }
+
+ @Override
+ public ByteBuf setMedium(int index, int value) {
+ _setMedium(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setMedium(int index, int value) {
+ buffer.setMedium(index, value);
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ _setInt(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setInt(int index, int value) {
+ buffer.setInt(index, value);
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ _setLong(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setLong(int index, long value) {
+ buffer.setLong(index, value);
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+ buffer.setBytes(index, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+ buffer.setBytes(index, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuffer src) {
+ buffer.setBytes(index, src);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, OutputStream out, int length)
+ throws IOException {
+ buffer.getBytes(index, out, length);
+ return this;
+ }
+
+ @Override
+ public int getBytes(int index, GatheringByteChannel out, int length)
+ throws IOException {
+ return buffer.getBytes(index, out, length);
+ }
+
+ @Override
+ public int setBytes(int index, InputStream in, int length)
+ throws IOException {
+ return buffer.setBytes(index, in, length);
+ }
+
+ @Override
+ public int setBytes(int index, ScatteringByteChannel in, int length)
+ throws IOException {
+ return buffer.setBytes(index, in, length);
+ }
+
+ @Override
+ public int nioBufferCount() {
+ return buffer.nioBufferCount();
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers(int index, int length) {
+ return buffer.nioBuffers(index, length);
+ }
+
+ @Override
+ public ByteBuffer internalNioBuffer(int index, int length) {
+ return nioBuffer(index, length);
+ }
+
+ @Override
+ public int forEachByte(int index, int length, ByteBufProcessor processor) {
+ return buffer.forEachByte(index, length, processor);
+ }
+
+ @Override
+ public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
+ return buffer.forEachByteDesc(index, length, processor);
+ }
+
+ @Override
+ public final int refCnt() {
+ return unwrap().refCnt();
+ }
+
+ @Override
+ public final ByteBuf retain() {
+ unwrap().retain();
+ return this;
+ }
+
+ @Override
+ public final ByteBuf retain(int increment) {
+ unwrap().retain(increment);
+ return this;
+ }
+
+ @Override
+ public boolean release() {
+ return release(1);
+ }
+
+ @Override
+ public boolean release(int decrement) {
+ boolean released = unwrap().release(decrement);
+ if (released) {
+ hugeBufferSize.addAndGet(-initCap);
+ hugeBufferCount.decrementAndGet();
+ }
+ return released;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
new file mode 100644
index 0000000..5809df5
--- /dev/null
+++ b/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import io.netty.util.internal.StringUtil;
+
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.util.AssertionUtil;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+
+public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PooledByteBufAllocatorL.class);
+
+ private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
+ private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
+
+ private static final String METRIC_PREFIX = "drill.allocator.";
+ private final MetricRegistry registry;
+ private final AtomicLong hugeBufferSize = new AtomicLong(0);
+ private final AtomicLong hugeBufferCount = new AtomicLong(0);
+ private final AtomicLong normalBufferSize = new AtomicLong(0);
+ private final AtomicLong normalBufferCount = new AtomicLong(0);
+
+ private final PoolArena<ByteBuffer>[] directArenas;
+ private final MemoryStatusThread statusThread;
+ private final Histogram largeBuffersHist;
+ private final Histogram normalBuffersHist;
+
+ public PooledByteBufAllocatorL(MetricRegistry registry) {
+ super(true);
+ this.registry = registry;
+ try {
+ Field f = PooledByteBufAllocator.class.getDeclaredField("directArenas");
+ f.setAccessible(true);
+ this.directArenas = (PoolArena<ByteBuffer>[]) f.get(this);
+ } catch (Exception e) {
+ throw new RuntimeException("Failure while initializing allocator. Unable to retrieve direct arenas field.", e);
+ }
+
+ if (memoryLogger.isTraceEnabled()) {
+ statusThread = new MemoryStatusThread();
+ statusThread.start();
+ } else {
+ statusThread = null;
+ }
+ removeOldMetrics();
+
+ registry.register(METRIC_PREFIX + "normal.size", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return normalBufferSize.get();
+ }
+ });
+
+ registry.register(METRIC_PREFIX + "normal.count", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return normalBufferCount.get();
+ }
+ });
+
+ registry.register(METRIC_PREFIX + "huge.size", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return hugeBufferSize.get();
+ }
+ });
+
+ registry.register(METRIC_PREFIX + "huge.count", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return hugeBufferCount.get();
+ }
+ });
+
+ largeBuffersHist = registry.histogram(METRIC_PREFIX + "huge.hist");
+ normalBuffersHist = registry.histogram(METRIC_PREFIX + "normal.hist");
+
+ }
+
+ private synchronized void removeOldMetrics() {
+ registry.removeMatching(new MetricFilter() {
+ @Override
+ public boolean matches(String name, Metric metric) {
+ return name.startsWith("drill.allocator.");
+ }
+
+ });
+ }
+
+ @Override
+ protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
+ throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+ }
+
+ @Override
+ protected UnsafeDirectLittleEndian newDirectBuffer(int initialCapacity, int maxCapacity) {
+ PoolThreadCache cache = threadCache.get();
+ PoolArena<ByteBuffer> directArena = cache.directArena;
+
+ if (directArena != null) {
+
+ if (initialCapacity > directArena.chunkSize) {
+ // This is beyond chunk size so we'll allocate separately.
+ ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity);
+
+ hugeBufferCount.incrementAndGet();
+ hugeBufferSize.addAndGet(buf.capacity());
+ largeBuffersHist.update(buf.capacity());
+ // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
+ return new UnsafeDirectLittleEndian(new LargeBuffer(buf, hugeBufferSize, hugeBufferCount));
+
+ } else {
+ // within chunk, use arena.
+ ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity);
+ if (!(buf instanceof PooledUnsafeDirectByteBuf)) {
+ fail();
+ }
+
+ normalBuffersHist.update(buf.capacity());
+ if (AssertionUtil.ASSERT_ENABLED) {
+ normalBufferSize.addAndGet(buf.capacity());
+ normalBufferCount.incrementAndGet();
+ }
+
+ return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount, normalBufferSize);
+ }
+
+ } else {
+ throw fail();
+ }
+ }
+
+ private UnsupportedOperationException fail() {
+ return new UnsupportedOperationException(
+ "Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
+ }
+
+
+ @Override
+ public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
+ if (initialCapacity == 0 && maxCapacity == 0) {
+ newDirectBuffer(initialCapacity, maxCapacity);
+ }
+ validate(initialCapacity, maxCapacity);
+ return newDirectBuffer(initialCapacity, maxCapacity);
+ }
+
+ @Override
+ public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
+ throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+ }
+
+
+ private static void validate(int initialCapacity, int maxCapacity) {
+ if (initialCapacity < 0) {
+ throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)");
+ }
+ if (initialCapacity > maxCapacity) {
+ throw new IllegalArgumentException(String.format(
+ "initialCapacity: %d (expected: not greater than maxCapacity(%d)",
+ initialCapacity, maxCapacity));
+ }
+ }
+
+ private class MemoryStatusThread extends Thread {
+
+ public MemoryStatusThread() {
+ super("memory-status-logger");
+ this.setDaemon(true);
+ this.setName("allocation.logger");
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString());
+ try {
+ Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000);
+ } catch (InterruptedException e) {
+ return;
+ }
+
+ }
+ }
+
+ }
+
+ public void checkAndReset() {
+ if (hugeBufferCount.get() != 0 || normalBufferCount.get() != 0) {
+ StringBuilder buf = new StringBuilder();
+ buf.append("Large buffers outstanding: ");
+ buf.append(hugeBufferCount.get());
+ buf.append(" totaling ");
+ buf.append(hugeBufferSize.get());
+ buf.append(" bytes.");
+ buf.append('\n');
+ buf.append("Normal buffers outstanding: ");
+ buf.append(normalBufferCount.get());
+ buf.append(" totaling ");
+ buf.append(normalBufferSize.get());
+ buf.append(" bytes.");
+ hugeBufferCount.set(0);
+ normalBufferCount.set(0);
+ hugeBufferSize.set(0);
+ normalBufferSize.set(0);
+ throw new DrillRuntimeException(buf.toString());
+ }
+ }
+
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append(directArenas.length);
+ buf.append(" direct arena(s):");
+ buf.append(StringUtil.NEWLINE);
+ for (PoolArena<ByteBuffer> a : directArenas) {
+ buf.append(a);
+ }
+
+ buf.append("Large buffers outstanding: ");
+ buf.append(this.hugeBufferCount.get());
+ buf.append(" totaling ");
+ buf.append(this.hugeBufferSize.get());
+ buf.append(" bytes.");
+ buf.append('\n');
+ buf.append("Normal buffers outstanding: ");
+ buf.append(this.normalBufferCount.get());
+ buf.append(" totaling ");
+ buf.append(this.normalBufferSize.get());
+ buf.append(" bytes.");
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
new file mode 100644
index 0000000..50a9a01
--- /dev/null
+++ b/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.netty.buffer;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.nio.ByteOrder;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.drill.exec.util.AssertionUtil;
+
+public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
+ private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
+ private final AbstractByteBuf wrapped;
+ private final long memoryAddress;
+ private AtomicLong bufferCount;
+ private AtomicLong bufferSize;
+ private long initCap = -1;
+
+ UnsafeDirectLittleEndian(LargeBuffer buf) {
+ this(buf, true);
+ }
+
+ UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong bufferCount, AtomicLong bufferSize) {
+ this(buf, true);
+ this.bufferCount = bufferCount;
+ this.bufferSize = bufferSize;
+
+ // initCap is used if we're tracking memory release. If we're in non-debug mode, we'll skip this.
+ this.initCap = AssertionUtil.ASSERT_ENABLED ? capacity() : -1;
+ }
+
+ private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake) {
+ super(buf);
+ if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
+ throw new IllegalStateException("Drill only runs on LittleEndian systems.");
+ }
+ wrapped = buf;
+ this.memoryAddress = buf.memoryAddress();
+ }
+ private long addr(int index) {
+ return memoryAddress + index;
+ }
+
+ @Override
+ public long getLong(int index) {
+// wrapped.checkIndex(index, 8);
+ long v = PlatformDependent.getLong(addr(index));
+ return v;
+ }
+
+ @Override
+ public float getFloat(int index) {
+ return Float.intBitsToFloat(getInt(index));
+ }
+
+ @Override
+ public ByteBuf slice() {
+ return slice(this.readerIndex(), readableBytes());
+ }
+
+ @Override
+ public ByteBuf slice(int index, int length) {
+ return new SlicedByteBuf(this, index, length);
+ }
+
+ @Override
+ public ByteOrder order() {
+ return ByteOrder.LITTLE_ENDIAN;
+ }
+
+ @Override
+ public ByteBuf order(ByteOrder endianness) {
+ return this;
+ }
+
+ @Override
+ public double getDouble(int index) {
+ return Double.longBitsToDouble(getLong(index));
+ }
+
+ @Override
+ public char getChar(int index) {
+ return (char) getShort(index);
+ }
+
+ @Override
+ public long getUnsignedInt(int index) {
+ return getInt(index) & 0xFFFFFFFFL;
+ }
+
+ @Override
+ public int getInt(int index) {
+// wrapped.checkIndex(index, 4);
+ int v = PlatformDependent.getInt(addr(index));
+ return v;
+ }
+
+ @Override
+ public int getUnsignedShort(int index) {
+ return getShort(index) & 0xFFFF;
+ }
+
+ @Override
+ public short getShort(int index) {
+// wrapped.checkIndex(index, 2);
+ short v = PlatformDependent.getShort(addr(index));
+ return v;
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ wrapped.checkIndex(index, 2);
+ _setShort(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ wrapped.checkIndex(index, 4);
+ _setInt(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ wrapped.checkIndex(index, 8);
+ _setLong(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setChar(int index, int value) {
+ setShort(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setFloat(int index, float value) {
+ setInt(index, Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf setDouble(int index, double value) {
+ setLong(index, Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeShort(int value) {
+ wrapped.ensureWritable(2);
+ _setShort(wrapped.writerIndex, value);
+ wrapped.writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeInt(int value) {
+ wrapped.ensureWritable(4);
+ _setInt(wrapped.writerIndex, value);
+ wrapped.writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeLong(long value) {
+ wrapped.ensureWritable(8);
+ _setLong(wrapped.writerIndex, value);
+ wrapped.writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeChar(int value) {
+ writeShort(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeFloat(float value) {
+ writeInt(Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeDouble(double value) {
+ writeLong(Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ private void _setShort(int index, int value) {
+ PlatformDependent.putShort(addr(index), (short) value);
+ }
+
+ private void _setInt(int index, int value) {
+ PlatformDependent.putInt(addr(index), value);
+ }
+
+ private void _setLong(int index, long value) {
+ PlatformDependent.putLong(addr(index), value);
+ }
+
+ @Override
+ public byte getByte(int index) {
+ return PlatformDependent.getByte(addr(index));
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ PlatformDependent.putByte(addr(index), (byte) value);
+ return this;
+ }
+
+ @Override
+ public boolean release() {
+ return release(1);
+ }
+
+ @Override
+ public boolean release(int decrement) {
+ boolean released = super.release(decrement);
+ if (released && initCap != -1) {
+ bufferCount.decrementAndGet();
+ bufferSize.addAndGet(-initCap);
+ }
+ return released;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java b/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
new file mode 100644
index 0000000..17d099d
--- /dev/null
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/exception/OutOfMemoryException.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.exception;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+public class OutOfMemoryException extends DrillRuntimeException {
+ private static final long serialVersionUID = -6858052345185793382L;
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutOfMemoryException.class);
+
+ public OutOfMemoryException() {
+ super();
+ }
+
+ public OutOfMemoryException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public OutOfMemoryException(String message, Throwable cause) {
+ super(message, cause);
+
+ }
+
+ public OutOfMemoryException(String message) {
+ super(message);
+
+ }
+
+ public OutOfMemoryException(Throwable cause) {
+ super(cause);
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/org/apache/drill/exec/memory/Accountor.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/Accountor.java b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/Accountor.java
new file mode 100644
index 0000000..7014a0b
--- /dev/null
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/Accountor.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.DrillBuf;
+
+public interface Accountor extends AutoCloseable{
+
+ public boolean transferTo(Accountor target, DrillBuf buf, long size);
+ public boolean transferIn(DrillBuf buf, long size);
+ public long getAvailable();
+ public long getCapacity();
+ public long getAllocation();
+ public long getPeakMemoryAllocation();
+
+ public boolean reserve(long size);
+ public boolean forceAdditionalReservation(long size);
+
+ public void reserved(long expected, DrillBuf buf);
+
+ public void release(DrillBuf buf, long size);
+ public void releasePartial(DrillBuf buf, long size);
+ public long resetFragmentLimits();
+ public void close();
+
+ public void setFragmentLimit(long add);
+ public long getFragmentLimit();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
new file mode 100644
index 0000000..a708e92
--- /dev/null
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.DrillBuf;
+
+import java.io.Closeable;
+
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.util.Pointer;
+
+/**
+ * Wrapper class to deal with byte buffer allocation. Ensures users only use designated methods. Also allows inser
+ */
+public interface BufferAllocator extends Closeable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BufferAllocator.class);
+
+ /**
+ * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
+ * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
+ *
+ * @param size
+ * The size in bytes.
+ * @return A new ByteBuf.
+ * @throws OutOfMemoryException if buffer cannot be allocated
+ */
+ public abstract DrillBuf buffer(int size);
+
+ /**
+ * Allocate a new or reused buffer within provided range. Note that the buffer may technically be larger than the
+ * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
+ *
+ * @param minSize The minimum size in bytes.
+ * @param maxSize The maximum size in bytes.
+ * @return A new ByteBuf.
+ * @throws OutOfMemoryException if buffer cannot be allocated
+ */
+ public abstract DrillBuf buffer(int minSize, int maxSize);
+
+ public abstract ByteBufAllocator getUnderlyingAllocator();
+
+ /**
+ * Create a child allocator nested below this one.
+ *
+ * @param context
+ * - BufferManager associated with the new child allocator
+ * @param initialReservation
+ * - specified in bytes
+ * @param maximumReservation
+ * - specified in bytes
+ * @param applyFragmentLimit
+ * - flag to conditionally enable fragment memory limits
+ * @return - a new buffer allocator owned by the parent it was spawned from
+ * @throws OutOfMemoryException
+ * - when off-heap memory has been exhausted
+ */
+ public abstract BufferAllocator getChildAllocator(LimitConsumer limitListener, long initialReservation,
+ long maximumReservation, boolean applyFragmentLimit) throws OutOfMemoryException;
+
+ /**
+ * Take over ownership of fragment accounting. Always takes over ownership.
+ * @param buf
+ * @return false if over allocation.
+ */
+ public boolean takeOwnership(DrillBuf buf) ;
+
+ /**
+ * Take over ownership of fragment accounting. Always takes over ownership.
+ * @param buf
+ * @return false if over allocation.
+ */
+ public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut);
+
+ public PreAllocator getNewPreAllocator();
+
+ //public void addFragmentContext(FragmentContext c);
+
+ /**
+ * For Top Level Allocators. Reset the fragment limits for all allocators
+ */
+ public void resetLimits();
+
+ /**
+ * For Child allocators to set the Fragment limit for the corresponding fragment allocator.
+ * @param l the new fragment limit
+ */
+ public void setLimit(long l);
+
+ public long getLimit();
+
+
+ /**
+ * Not thread safe.
+ *
+ * WARNING: unclaimed pre-allocations leak memory. If you call preAllocate(), you must
+ * make sure to ultimately try to get the buffer and release it.
+ */
+ public interface PreAllocator {
+ public boolean preAllocate(int bytes);
+
+ public DrillBuf getAllocation();
+ }
+
+ /**
+ * @param bytes
+ * @return
+ */
+
+ /**
+ *
+ */
+
+ /**
+ * Close and release all buffers generated from this buffer pool.
+ */
+ @Override
+ public abstract void close();
+
+ public abstract long getAllocatedMemory();
+
+ public abstract long getPeakMemoryAllocation();
+
+ public DrillBuf getEmpty();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
new file mode 100644
index 0000000..777d9d2
--- /dev/null
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/LimitConsumer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+public interface LimitConsumer {
+
+ public String getIdentifier();
+ public long getAllocated();
+
+ public long getLimit();
+
+ public void setLimit(long limit);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/org/apache/drill/exec/ops/BufferManager.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/ops/BufferManager.java b/exec/memory/base/src/main/java/org/apache/drill/exec/ops/BufferManager.java
new file mode 100644
index 0000000..4df8f0e
--- /dev/null
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/ops/BufferManager.java
@@ -0,0 +1,66 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.ops;
+
+import io.netty.buffer.DrillBuf;
+
+/**
+ * Manages a list of {@link DrillBuf}s that can be reallocated as needed. Upon
+ * re-allocation the old buffer will be freed. Managing a list of these buffers
+ * prevents some parts of the system from needing to define a correct location
+ * to place the final call to free them.
+ *
+ * The current uses of these types of buffers are within the pluggable components of Drill.
+ * In UDFs, memory management should not be a concern. We provide access to re-allocatable
+ * DrillBufs to give UDF writers general purpose buffers we can account for. To prevent the need
+ * for UDFs to contain boilerplate to close all of the buffers they request, this list
+ * is tracked at a higher level and all of the buffers are freed once we are sure that
+ * the code depending on them is done executing (currently {@link FragmentContext}
+ * and {@link QueryContext}.
+ */
+public interface BufferManager extends AutoCloseable {
+
+ /**
+ * Replace an old buffer with a new version at least of the provided size. Does not copy data.
+ *
+ * @param old
+ * Old Buffer that the user is no longer going to use.
+ * @param newSize
+ * Size of new replacement buffer.
+ * @return
+ */
+ public DrillBuf replace(DrillBuf old, int newSize);
+
+ /**
+ * Get a managed buffer of indeterminate size.
+ *
+ * @return A buffer.
+ */
+ public DrillBuf getManagedBuffer();
+
+ /**
+ * Get a managed buffer of at least a certain size.
+ *
+ * @param size
+ * The desired size
+ * @return A buffer
+ */
+ public DrillBuf getManagedBuffer(int size);
+
+ public void close();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/base/src/main/java/org/apache/drill/exec/util/Pointer.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/util/Pointer.java b/exec/memory/base/src/main/java/org/apache/drill/exec/util/Pointer.java
new file mode 100644
index 0000000..074ed76
--- /dev/null
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/util/Pointer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+public class Pointer<T> {
+ public T value;
+
+ public Pointer(){}
+
+ public Pointer(T value){
+ this.value = value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/impl/pom.xml
----------------------------------------------------------------------
diff --git a/exec/memory/impl/pom.xml b/exec/memory/impl/pom.xml
new file mode 100644
index 0000000..aa5f0e7
--- /dev/null
+++ b/exec/memory/impl/pom.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>memory-parent</artifactId>
+ <groupId>org.apache.drill.memory</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>drill-memory-impl</artifactId>
+ <name>exec/memory/impl</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-protocol</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.memory</groupId>
+ <artifactId>drill-memory-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
+
+ </dependencies>
+
+
+ <build>
+ </build>
+
+
+
+</project>
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
----------------------------------------------------------------------
diff --git a/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
new file mode 100644
index 0000000..0ac93e4
--- /dev/null
+++ b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
@@ -0,0 +1,437 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.util.AssertionUtil;
+
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+public class AccountorImpl implements Accountor {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AccountorImpl.class);
+
+ private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
+
+ public static final String ENABLE_FRAGMENT_MEMORY_LIMIT = "drill.exec.memory.enable_frag_limit";
+ public static final String FRAGMENT_MEM_OVERCOMMIT_FACTOR = "drill.exec.memory.frag_mem_overcommit_factor";
+
+ private final AtomicRemainder remainder;
+ private final long total;
+ private ConcurrentMap<ByteBuf, DebugStackTrace> buffers = Maps.newConcurrentMap();
+ private AccountorImpl parent;
+
+ private final boolean errorOnLeak;
+ // some operators are no subject to the fragment limit. They set the applyFragmentLimit to false
+
+ private final boolean enableFragmentLimit;
+ private final double fragmentMemOvercommitFactor;
+
+ private final boolean DEFAULT_ENABLE_FRAGMENT_LIMIT=false;
+ private final double DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR=1.5;
+
+ private final boolean applyFragmentLimit;
+
+ private final LimitConsumer limitConsumer;
+ long fragmentLimit;
+
+ private long peakMemoryAllocation = 0;
+
+ // The top level Allocator has an accountor that keeps track of all the LimitConsumers currently executing.
+ // This enables the top level accountor to calculate a new fragment limit whenever necessary.
+ private final List<LimitConsumer> limitConsumers;
+
+ public AccountorImpl(DrillConfig config, boolean errorOnLeak, LimitConsumer context, AccountorImpl parent, long max,
+ long preAllocated, boolean applyFragLimit) {
+ // TODO: fix preallocation stuff
+ this.errorOnLeak = errorOnLeak;
+ AtomicRemainder parentRemainder = parent != null ? parent.remainder : null;
+ this.parent = parent;
+
+ boolean enableFragmentLimit;
+ double fragmentMemOvercommitFactor;
+
+ try {
+ enableFragmentLimit = config.getBoolean(ENABLE_FRAGMENT_MEMORY_LIMIT);
+ fragmentMemOvercommitFactor = config.getDouble(FRAGMENT_MEM_OVERCOMMIT_FACTOR);
+ }catch(Exception e){
+ enableFragmentLimit = DEFAULT_ENABLE_FRAGMENT_LIMIT;
+ fragmentMemOvercommitFactor = DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR;
+ }
+ this.enableFragmentLimit = enableFragmentLimit;
+ this.fragmentMemOvercommitFactor = fragmentMemOvercommitFactor;
+
+
+ this.applyFragmentLimit=applyFragLimit;
+
+ this.remainder = new AtomicRemainder(errorOnLeak, parentRemainder, max, preAllocated, applyFragmentLimit);
+ this.total = max;
+ this.limitConsumer = context;
+ this.fragmentLimit=this.total; // Allow as much as possible to start with;
+ if (ENABLE_ACCOUNTING) {
+ buffers = Maps.newConcurrentMap();
+ } else {
+ buffers = null;
+ }
+ this.limitConsumers = new ArrayList<LimitConsumer>();
+ if(parent!=null && parent.parent==null){ // Only add the fragment context to the fragment level accountor
+ synchronized(this) {
+ addLimitConsumer(this.limitConsumer);
+ }
+ }
+ }
+
+ public boolean transferTo(Accountor target, DrillBuf buf, long size) {
+ return transfer(target, buf, size, true);
+ }
+
+ public boolean transferIn(DrillBuf buf, long size) {
+ return transfer(this, buf, size, false);
+ }
+
+ private boolean transfer(Accountor target, DrillBuf buf, long size, boolean release) {
+ boolean withinLimit = target.forceAdditionalReservation(size);
+ if(release){
+ release(buf, size);
+ }
+
+ if (ENABLE_ACCOUNTING) {
+ if (target instanceof AccountorImpl) {
+ ((AccountorImpl) target).buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread()
+ .getStackTrace()));
+ }
+ }
+ return withinLimit;
+ }
+
+ public long getAvailable() {
+ if (parent != null) {
+ return Math.min(parent.getAvailable(), getCapacity() - getAllocation());
+ }
+ return getCapacity() - getAllocation();
+ }
+
+ public long getCapacity() {
+ return fragmentLimit;
+ }
+
+ public long getAllocation() {
+ return remainder.getUsed();
+ }
+
+ public long getPeakMemoryAllocation() {
+ return peakMemoryAllocation;
+ }
+
+ public boolean reserve(long size) {
+ boolean status = remainder.get(size, this.applyFragmentLimit);
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+ return status;
+ }
+
+ public boolean forceAdditionalReservation(long size) {
+ if (size > 0) {
+ boolean status = remainder.forceGet(size);
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+ return status;
+ } else {
+ return true;
+ }
+ }
+
+ public void reserved(long expected, DrillBuf buf) {
+ // make sure to take away the additional memory that happened due to rounding.
+
+ long additional = buf.capacity() - expected;
+ if (additional > 0) {
+ remainder.forceGet(additional);
+ }
+
+ if (ENABLE_ACCOUNTING) {
+ buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread().getStackTrace()));
+ }
+
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+ }
+
+
+ public void releasePartial(DrillBuf buf, long size) {
+ remainder.returnAllocation(size);
+ if (ENABLE_ACCOUNTING) {
+ if (buf != null) {
+ DebugStackTrace dst = buffers.get(buf);
+ if (dst == null) {
+ throw new IllegalStateException("Partially releasing a buffer that has already been released. Buffer: " + buf);
+ }
+ dst.size -= size;
+ if (dst.size < 0) {
+ throw new IllegalStateException("Partially releasing a buffer that has already been released. Buffer: " + buf);
+ }
+ }
+ }
+ }
+
+ void release(long size) {
+ remainder.returnAllocation(size);
+ }
+
+ public void release(DrillBuf buf, long size) {
+ remainder.returnAllocation(size);
+ if (ENABLE_ACCOUNTING) {
+ if (buf != null && buffers.remove(buf) == null) {
+ throw new IllegalStateException("Releasing a buffer that has already been released. Buffer: " + buf);
+ }
+ }
+ }
+
+ private void addLimitConsumer(LimitConsumer c) {
+ if (c == null) {
+ return;
+ }
+
+ if (parent != null){
+ parent.addLimitConsumer(c);
+ }else {
+ if(logger.isTraceEnabled()) {
+ String fragStr = c == null ? "[Null Context]" : c.getIdentifier();
+ fragStr+=" (Object Id: "+System.identityHashCode(c)+")";
+ StackTraceElement[] ste = (new Throwable()).getStackTrace();
+ StringBuffer sb = new StringBuffer();
+ for (StackTraceElement s : ste) {
+ sb.append(s.toString());
+ sb.append("\n");
+ }
+
+ logger.trace("Fragment " + fragStr + " added to root accountor.\n"+sb.toString());
+ }
+ synchronized(this) {
+ limitConsumers.add(c);
+ }
+ }
+ }
+
+ private void removeLimitConsumer(LimitConsumer c) {
+ if (c == null) {
+ return;
+ }
+
+ if (parent != null){
+ if (parent.parent==null){
+ // only fragment level allocators will have the fragment context saved
+ parent.removeLimitConsumer(c);
+ }
+ }else{
+ if(logger.isDebugEnabled()) {
+ String fragStr = c == null ? "[Null Context]" : c.getIdentifier();
+ fragStr += " (Object Id: " + System.identityHashCode(c) + ")";
+ logger.trace("Fragment " + fragStr + " removed from root accountor");
+ }
+ synchronized(this) {
+ limitConsumers.remove(c);
+ }
+ }
+ }
+
+ public long resetFragmentLimits(){
+ // returns the new capacity
+ if(!this.enableFragmentLimit){
+ return getCapacity();
+ }
+
+ if(parent!=null){
+ parent.resetFragmentLimits();
+ }else {
+ //Get remaining memory available per fragment and distribute it EQUALLY among all the fragments.
+ //Fragments get the memory limit added to the amount already allocated.
+ //This favours fragments that are already running which will get a limit greater than newly started fragments.
+ //If the already running fragments end quickly, their limits will be assigned back to the remaining fragments
+ //quickly. If they are long running, then we want to favour them with larger limits anyway.
+ synchronized (this) {
+ int nFragments = limitConsumers.size();
+ long allocatedMemory=0;
+ for (LimitConsumer fragment : limitConsumers) {
+ allocatedMemory += fragment.getAllocated();
+ }
+ if(logger.isTraceEnabled()) {
+ logger.trace("Resetting Fragment Memory Limit: total Available memory== "+total
+ +" Total Allocated Memory :"+allocatedMemory
+ +" Number of fragments: "+nFragments
+ + " fragmentMemOvercommitFactor: "+fragmentMemOvercommitFactor
+ + " Root fragment limit: "+this.fragmentLimit + "(Root obj: "+System.identityHashCode(this)+")"
+ );
+ }
+ if(nFragments>0) {
+ long rem = (total - allocatedMemory) / nFragments;
+ for (LimitConsumer fragment : limitConsumers) {
+ fragment.setLimit((long) (rem * fragmentMemOvercommitFactor));
+ }
+ }
+ if(logger.isTraceEnabled() && false){
+ StringBuffer sb= new StringBuffer();
+ sb.append("[root](0:0)");
+ sb.append("Allocated memory: ");
+ sb.append(this.getAllocation());
+ sb.append(" Fragment Limit: ");
+ sb.append(this.getFragmentLimit());
+ logger.trace(sb.toString());
+ for (LimitConsumer fragment : limitConsumers) {
+ sb= new StringBuffer();
+ sb.append('[');
+ sb.append(fragment.getIdentifier());
+ sb.append(']');
+ sb.append("Allocated memory: ");
+ sb.append(fragment.getAllocated());
+ sb.append(" Fragment Limit: ");
+ sb.append(fragment.getLimit());
+ logger.trace(sb.toString());
+ }
+ logger.trace("Resetting Complete");
+ }
+ }
+ }
+ return getCapacity();
+ }
+
+ public void close() {
+ // remove the fragment context and reset fragment limits whenever an allocator closes
+ if (parent != null && parent.parent == null && limitConsumer != null) {
+ logger.debug("Fragment " + limitConsumer.getIdentifier() + " accountor being closed");
+ removeLimitConsumer(limitConsumer);
+ }
+ resetFragmentLimits();
+
+ if (ENABLE_ACCOUNTING && !buffers.isEmpty()) {
+ StringBuffer sb = new StringBuffer();
+ sb.append("Attempted to close accountor with ");
+ sb.append(buffers.size());
+ sb.append(" buffer(s) still allocated for ");
+ sb.append(limitConsumer.getIdentifier());
+ sb.append(".\n");
+
+ Multimap<DebugStackTrace, DebugStackTrace> multi = LinkedListMultimap.create();
+ for (DebugStackTrace t : buffers.values()) {
+ multi.put(t, t);
+ }
+
+ for (DebugStackTrace entry : multi.keySet()) {
+ Collection<DebugStackTrace> allocs = multi.get(entry);
+
+ sb.append("\n\n\tTotal ");
+ sb.append(allocs.size());
+ sb.append(" allocation(s) of byte size(s): ");
+ for (DebugStackTrace alloc : allocs) {
+ sb.append(alloc.size);
+ sb.append(", ");
+ }
+
+ sb.append("at stack location:\n");
+ entry.addToString(sb);
+ }
+ if (!buffers.isEmpty()) {
+ IllegalStateException e = new IllegalStateException(sb.toString());
+ if (errorOnLeak) {
+ throw e;
+ } else {
+ logger.warn("Memory leaked.", e);
+ }
+ }
+ }
+
+ remainder.close();
+
+ }
+
+ public void setFragmentLimit(long add) {
+ // We ADD the limit to the current allocation. If none has been allocated, this
+ // sets a new limit. If memory has already been allocated, the fragment gets its
+ // limit based on the allocation, though this might still result in reducing the
+ // limit.
+
+ if (parent != null && parent.parent==null) { // This is a fragment level accountor
+ this.fragmentLimit=getAllocation()+add;
+ this.remainder.setLimit(this.fragmentLimit);
+ logger.trace("Fragment " + limitConsumer.getIdentifier() + " memory limit set to " + this.fragmentLimit);
+ }
+ }
+
+ public long getFragmentLimit(){
+ return this.fragmentLimit;
+ }
+
+ public class DebugStackTrace {
+
+ private StackTraceElement[] elements;
+ private long size;
+
+ public DebugStackTrace(long size, StackTraceElement[] elements) {
+ super();
+ this.elements = elements;
+ this.size = size;
+ }
+
+ public void addToString(StringBuffer sb) {
+ for (int i = 3; i < elements.length; i++) {
+ sb.append("\t\t");
+ sb.append(elements[i]);
+ sb.append("\n");
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(elements);
+// result = prime * result + (int) (size ^ (size >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ DebugStackTrace other = (DebugStackTrace) obj;
+ if (!Arrays.equals(elements, other.elements)) {
+ return false;
+ }
+ // weird equal where size doesn't matter for multimap purposes.
+// if (size != other.size)
+// return false;
+ return true;
+ }
+
+ }
+
+}
[44/45] drill git commit: DRILL-4085: Disable RPC thread offload
until concurrency bug is found.
Posted by ja...@apache.org.
DRILL-4085: Disable RPC thread offload until concurrency bug is found.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2359294c
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2359294c
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2359294c
Branch: refs/heads/master
Commit: 2359294cfc936773ebcc3f187b79222ec88cc1e5
Parents: bdd2386
Author: Jacques Nadeau <ja...@apache.org>
Authored: Fri Nov 6 14:25:05 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 18:36:26 2015 -0800
----------------------------------------------------------------------
.../main/java/org/apache/drill/exec/rpc/RpcBus.java | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/2359294c/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
index 61922a1..b6f3032 100644
--- a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
@@ -59,6 +59,7 @@ public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> imp
final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(this.getClass());
private static final OutboundRpcMessage PONG = new OutboundRpcMessage(RpcMode.PONG, 0, 0, Acks.OK);
+ private static final boolean ENABLE_SEPARATE_THREADS = "true".equals(System.getProperty("drill.enable_rpc_offload"));
protected final CoordinationQueue queue = new CoordinationQueue(16, 16);
@@ -256,16 +257,25 @@ public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> imp
}
+ private class SameExecutor implements Executor {
+
+ @Override
+ public void execute(Runnable command) {
+ command.run();
+ }
+
+ }
protected class InboundHandler extends MessageToMessageDecoder<InboundRpcMessage> {
- private final RpcEventHandler exec;
+ private final Executor exec;
private final C connection;
public InboundHandler(C connection) {
super();
this.connection = connection;
- this.exec = new RpcEventHandler(rpcConfig.getExecutor());
+ final Executor underlyingExecutor = ENABLE_SEPARATE_THREADS ? rpcConfig.getExecutor() : new SameExecutor();
+ this.exec = new RpcEventHandler(underlyingExecutor);
}
@Override
[24/45] drill git commit: DRILL-3987: (MOVE) Extract RPC,
memory-base and memory-impl as separate modules.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
----------------------------------------------------------------------
diff --git a/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
new file mode 100644
index 0000000..0f6b8b0
--- /dev/null
+++ b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ *
+ *
+ * TODO: Fix this so that preallocation can never be released back to general pool until allocator is closed.
+ */
+public class AtomicRemainder {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AtomicRemainder.class);
+
+ private final AtomicRemainder parent;
+ private final AtomicLong availableShared;
+ private final AtomicLong availablePrivate;
+ private final long initTotal;
+ private final long initShared;
+ private final long initPrivate;
+ private long limit; // An Allocator can set a variable limit less than or equal to the initTotal
+ private boolean hasLimit; // True for Atomic Remainders associated with a Fragment. May be true for Operator Level allocators some day.
+ private boolean closed = false;
+ private final boolean errorOnLeak;
+ private final boolean applyFragmentLimit;
+
+ public AtomicRemainder(boolean errorOnLeak, AtomicRemainder parent, long max, long pre, boolean applyFragLimit) {
+ this.errorOnLeak = errorOnLeak;
+ this.parent = parent;
+ this.availableShared = new AtomicLong(max - pre);
+ this.availablePrivate = new AtomicLong(pre);
+ this.initTotal = max;
+ this.initShared = max - pre;
+ this.initPrivate = pre;
+ this.limit = max;
+ this.hasLimit=false;
+ this.applyFragmentLimit=applyFragLimit; // If this is an operator that is exempt from the fragment limit, set this to false.
+// logger.info("new AtomicRemainder. a.s. {} a.p. {} hashcode {}", availableShared, availablePrivate, hashCode(), new Exception());
+ }
+
+ public long getRemainder() {
+ return availableShared.get() + availablePrivate.get();
+ }
+
+ public long getUsed() {
+ return initTotal - getRemainder();
+ }
+
+ /**
+ * Allow an allocator to constrain the remainder to a particular limit that is lower than the initTotal.
+ * If limit is larger than initTotal, then the function will do nothing and the hasLimit flag will not be set.
+ * @param limit new remainder limit
+ */
+ public void setLimit(long limit) {
+ if(limit<initTotal){
+ this.hasLimit=true;
+ this.limit=limit;
+ }
+
+ }
+ /**
+ * Automatically allocate memory. This is used when an actual allocation happened to be larger than requested, or when
+ * a buffer has it's ownership passed to another allocator.<br>
+ * This memory has already been used up so it must be accurately accounted for in future allocations.
+ *
+ * @param size extra allocated memory that needs to be accounted for
+ */
+ public boolean forceGet(long size) {
+ if (get(size, this.applyFragmentLimit)) {
+ return true;
+ } else {
+ availableShared.addAndGet(-size);
+ if (parent != null) {
+ parent.forceGet(size);
+ }
+ return false;
+ }
+ }
+
+ public boolean get(long size, boolean applyFragmentLimitForChild) {
+ if (availablePrivate.get() < 1) {
+ // if there is no preallocated memory, we can operate normally.
+
+ // if there is a parent allocator, check it before allocating.
+ if (parent != null && !parent.get(size, this.applyFragmentLimit)) {
+ return false;
+ }
+
+ // If we need to allocate memory beyond the allowed Fragment Limit
+ if(applyFragmentLimitForChild && this.applyFragmentLimit && this.hasLimit && (getUsed()+size > this.limit)){
+ if (parent != null) {
+ parent.returnAllocation(size);
+ }
+ StackTraceElement[] ste = (new Throwable()).getStackTrace();
+ StringBuilder sb = new StringBuilder();
+ for (StackTraceElement s : ste) {
+ sb.append(s.toString());
+ sb.append("\n");
+ }
+ logger.warn("No more memory. Fragment limit ({} bytes) reached. Trying to allocate {} bytes. {} bytes " +
+ "already allocated.\n{}", limit, size, getUsed(), sb.toString());
+ return false;
+ }
+
+ // attempt to get shared memory, if fails, return false.
+ long outcome = availableShared.addAndGet(-size);
+// assert outcome <= initShared;
+ if (outcome < 0) {
+ availableShared.addAndGet(size);
+ if (parent != null) {
+ parent.returnAllocation(size);
+ }
+ return false;
+ } else {
+// if (DEBUG)
+// logger.info("Get {}. a.s. {} a.p. {} hashcode {}", size, availableShared, availablePrivate, hashCode(), new Exception());
+ return true;
+ }
+
+ } else {
+ // if there is preallocated memory, use that first.
+ long unaccount = availablePrivate.addAndGet(-size);
+ if (unaccount >= 0) {
+// if (DEBUG)
+// logger.info("Get {}. a.s. {} a.p. {} hashcode {}", size, availableShared, availablePrivate, hashCode(), new Exception());
+ return true;
+ } else {
+
+ long additionalSpaceNeeded = -unaccount;
+ // if there is a parent allocator, check it before allocating.
+ if (parent != null && !parent.get(additionalSpaceNeeded, this.applyFragmentLimit)) {
+ // parent allocation failed, return space to private pool.
+ availablePrivate.getAndAdd(size);
+ return false;
+ }
+
+ // we got space from parent pool. lets make sure we have space locally available.
+ long account = availableShared.addAndGet(-additionalSpaceNeeded);
+ if (account >= 0) {
+ // we were succesful, move private back to zero (since we allocated using shared).
+ availablePrivate.addAndGet(additionalSpaceNeeded);
+// if (DEBUG)
+// logger.info("Get {}. a.s. {} a.p. {} hashcode {}", size, availableShared, availablePrivate, hashCode(), new Exception());
+ return true;
+ } else {
+ // we failed to get space from available shared. Return allocations to initial state.
+ availablePrivate.addAndGet(size);
+ availableShared.addAndGet(additionalSpaceNeeded);
+ if (parent != null) {
+ parent.returnAllocation(additionalSpaceNeeded);
+ }
+ return false;
+ }
+ }
+
+ }
+
+ }
+
+ /**
+ * Return the memory accounting to the allocation pool. Make sure to first maintain hold of the preallocated memory.
+ *
+ * @param size amount of memory returned
+ */
+ public void returnAllocation(long size) {
+ long privateSize = availablePrivate.get();
+ long privateChange = Math.min(size, initPrivate - privateSize);
+ long sharedChange = size - privateChange;
+ availablePrivate.addAndGet(privateChange);
+ availableShared.addAndGet(sharedChange);
+// if (DEBUG)
+// logger.info("Return allocation {}, a.s. {} a.p. {} hashcode {}", size, availableShared, availablePrivate, hashCode(), new Exception());
+ if (parent != null) {
+ parent.returnAllocation(sharedChange);
+ }
+ }
+
+ public void close() {
+ if (closed) {
+ logger.warn("Tried to close remainder, but it has already been closed", new Exception());
+ return;
+ }
+ if (availablePrivate.get() != initPrivate || availableShared.get() != initShared) {
+ IllegalStateException e = new IllegalStateException(
+ String
+ .format(ERROR, initPrivate, availablePrivate.get(), initPrivate - availablePrivate.get(), initShared, availableShared.get(), initShared - availableShared.get()));
+ if (errorOnLeak) {
+ throw e;
+ } else {
+ logger.warn("Memory leaked during query.", e);
+ }
+ }
+ if (parent != null) {
+ parent.returnAllocation(initPrivate);
+ }
+ closed = true;
+ }
+
+ static final String ERROR = "Failure while closing accountor. Expected private and shared pools to be set to initial values. However, one or more were not. Stats are\n\tzone\tinit\tallocated\tdelta \n\tprivate\t%d\t%d\t%d \n\tshared\t%d\t%d\t%d.";
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/RootAllocatorFactory.java
----------------------------------------------------------------------
diff --git a/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/RootAllocatorFactory.java b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/RootAllocatorFactory.java
new file mode 100644
index 0000000..a413e4a
--- /dev/null
+++ b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/RootAllocatorFactory.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import org.apache.drill.common.config.DrillConfig;
+
+import com.google.common.annotations.VisibleForTesting;
+
+public class RootAllocatorFactory {
+ /**
+ * Constructor to prevent instantiation of this static utility class.
+ */
+ private RootAllocatorFactory() {}
+
+ /**
+ * Factory method.
+ *
+ * @param drillConfig the DrillConfig
+ * @return a new root allocator
+ */
+ public static BufferAllocator newRoot(final DrillConfig drillConfig) {
+/* TODO(DRILL-1942)
+ if (BaseAllocator.DEBUG) {
+ return new TopLevelAllocator(drillConfig);
+ }
+*/
+
+ return new TopLevelAllocator(drillConfig);
+ }
+
+ /**
+ * Factory method for testing, where a DrillConfig may not be available.
+ *
+ * @param allocationPolicy the allocation policy
+ * @param initAllocation initial allocation from parent
+ * @param maxReservation maximum amount of memory this can hand out
+ * @param flags one of BufferAllocator's F_* flags
+ * @return a new root allocator
+ */
+/* TODO(DRILL-1942)
+ @VisibleForTesting
+ public static BufferAllocator newRoot(
+ final AllocationPolicy allocationPolicy,
+ final long initAllocation, final long maxReservation, final int flags) {
+ if (BaseAllocator.DEBUG) {
+ return new RootAllocatorDebug(allocationPolicy, initAllocation, maxReservation, flags);
+ }
+
+ return new RootAllocator(allocationPolicy, initAllocation, maxReservation, flags);
+ }
+*/
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
new file mode 100644
index 0000000..1684970
--- /dev/null
+++ b/exec/memory/impl/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.DrillBuf;
+import io.netty.buffer.PooledByteBufAllocatorL;
+import io.netty.buffer.UnsafeDirectLittleEndian;
+
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.metrics.DrillMetrics;
+import org.apache.drill.exec.util.AssertionUtil;
+import org.apache.drill.exec.util.Pointer;
+
+public class TopLevelAllocator implements BufferAllocator {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopLevelAllocator.class);
+ public static final String CHILD_BUFFER_INJECTION_SITE = "child.buffer";
+
+ public static final String TOP_LEVEL_MAX_ALLOC = "drill.exec.memory.top.max";
+ public static final String ERROR_ON_MEMORY_LEAK = "drill.exec.debug.error_on_leak";
+
+ public static long MAXIMUM_DIRECT_MEMORY;
+
+ private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
+ private final Map<ChildAllocator, StackTraceElement[]> childrenMap;
+ private final PooledByteBufAllocatorL innerAllocator;
+ private final AccountorImpl acct;
+ private final boolean errorOnLeak;
+ private final DrillBuf empty;
+ private final DrillConfig config;
+
+ private TopLevelAllocator(DrillConfig config, long maximumAllocation, boolean errorOnLeak){
+ MAXIMUM_DIRECT_MEMORY = maximumAllocation;
+ innerAllocator = new PooledByteBufAllocatorL(DrillMetrics.getInstance());
+ this.config=(config!=null) ? config : DrillConfig.create();
+ this.errorOnLeak = errorOnLeak;
+ this.acct = new AccountorImpl(config, errorOnLeak, null, null, maximumAllocation, 0, true);
+ this.empty = DrillBuf.getEmpty(this, acct);
+ this.childrenMap = ENABLE_ACCOUNTING ? new IdentityHashMap<ChildAllocator, StackTraceElement[]>() : null;
+ }
+
+ TopLevelAllocator(DrillConfig config) {
+ this(config, Math.min(DrillConfig.getMaxDirectMemory(), config.getLong(TOP_LEVEL_MAX_ALLOC)),
+ config.getBoolean(ERROR_ON_MEMORY_LEAK)
+ );
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf) {
+ return buf.transferAccounting(acct);
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> out) {
+ DrillBuf b = new DrillBuf(this, acct, buf);
+ out.value = b;
+ return acct.transferIn(b, b.capacity());
+ }
+
+ public DrillBuf buffer(int min, int max) {
+ if (min == 0) {
+ return empty;
+ }
+ if(!acct.reserve(min)) {
+ throw new OutOfMemoryException(createErrorMsg(this, min));
+ }
+
+ try {
+ UnsafeDirectLittleEndian buffer = innerAllocator.directBuffer(min, max);
+ DrillBuf wrapped = new DrillBuf(this, acct, buffer);
+ acct.reserved(min, wrapped);
+ return wrapped;
+ } catch (OutOfMemoryError e) {
+ if ("Direct buffer memory".equals(e.getMessage())) {
+ acct.release(min);
+ throw new OutOfMemoryException(createErrorMsg(this, min), e);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ @Override
+ public DrillBuf buffer(int size) {
+ return buffer(size, size);
+ }
+
+ @Override
+ public long getAllocatedMemory() {
+ return acct.getAllocation();
+ }
+
+ @Override
+ public long getPeakMemoryAllocation() {
+ return acct.getPeakMemoryAllocation();
+ }
+
+ @Override
+ public ByteBufAllocator getUnderlyingAllocator() {
+ return innerAllocator;
+ }
+
+ @Override
+ public BufferAllocator getChildAllocator(LimitConsumer limitConsumer, long initialReservation,
+ long maximumReservation,
+ boolean applyFragmentLimit) {
+ if(!acct.reserve(initialReservation)){
+ logger.debug(String.format("You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.", initialReservation, acct.getCapacity() - acct.getAllocation()));
+ throw new OutOfMemoryException(
+ String
+ .format(
+ "You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.",
+ initialReservation, acct.getCapacity() - acct.getAllocation()));
+ }
+ logger.debug("New child allocator with initial reservation {}", initialReservation);
+ ChildAllocator allocator = new ChildAllocator(limitConsumer, acct, maximumReservation, initialReservation,
+ childrenMap, applyFragmentLimit);
+ if(ENABLE_ACCOUNTING){
+ childrenMap.put(allocator, Thread.currentThread().getStackTrace());
+ }
+
+ return allocator;
+ }
+
+ @Override
+ public void resetLimits() {
+ acct.resetFragmentLimits();
+ }
+
+ @Override
+ public void setLimit(long limit){
+ acct.setFragmentLimit(limit);
+ }
+
+ @Override
+ public long getLimit(){
+ return acct.getFragmentLimit();
+ }
+
+ @Override
+ public void close() {
+ if (ENABLE_ACCOUNTING) {
+ for (Entry<ChildAllocator, StackTraceElement[]> child : childrenMap.entrySet()) {
+ if (!child.getKey().isClosed()) {
+ StringBuilder sb = new StringBuilder();
+ StackTraceElement[] elements = child.getValue();
+ for (int i = 0; i < elements.length; i++) {
+ sb.append("\t\t");
+ sb.append(elements[i]);
+ sb.append("\n");
+ }
+ throw new IllegalStateException("Failure while trying to close allocator: Child level allocators not closed. Stack trace: \n" + sb);
+ }
+ }
+ }
+ acct.close();
+ }
+
+
+
+ @Override
+ public DrillBuf getEmpty() {
+ return empty;
+ }
+
+
+
+ private class ChildAllocator implements BufferAllocator {
+ private final DrillBuf empty;
+ private AccountorImpl childAcct;
+ private Map<ChildAllocator, StackTraceElement[]> children = new HashMap<>();
+ private boolean closed = false;
+ private LimitConsumer limitConsumer;
+ private Map<ChildAllocator, StackTraceElement[]> thisMap;
+ private boolean applyFragmentLimit;
+
+ public ChildAllocator(LimitConsumer limitConsumer,
+ AccountorImpl parentAccountor,
+ long max,
+ long pre,
+ Map<ChildAllocator,
+ StackTraceElement[]> map,
+ boolean applyFragmentLimit) {
+ assert max >= pre;
+ this.applyFragmentLimit = applyFragmentLimit;
+ this.limitConsumer = limitConsumer;
+ childAcct = new AccountorImpl(config, errorOnLeak, limitConsumer, parentAccountor, max, pre, applyFragmentLimit);
+ thisMap = map;
+ empty = DrillBuf.getEmpty(this, childAcct);
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf) {
+ return buf.transferAccounting(childAcct);
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> out) {
+ DrillBuf b = new DrillBuf(this, acct, buf);
+ out.value = b;
+ return acct.transferIn(b, b.capacity());
+ }
+
+
+ @Override
+ public DrillBuf buffer(int size, int max) {
+ if (size == 0) {
+ return empty;
+ }
+ if(!childAcct.reserve(size)) {
+ throw new OutOfMemoryException(createErrorMsg(this, size));
+ }
+
+ try {
+ UnsafeDirectLittleEndian buffer = innerAllocator.directBuffer(size, max);
+ DrillBuf wrapped = new DrillBuf(this, childAcct, buffer);
+ childAcct.reserved(buffer.capacity(), wrapped);
+ return wrapped;
+ } catch (OutOfMemoryError e) {
+ if ("Direct buffer memory".equals(e.getMessage())) {
+ childAcct.release(size);
+ throw new OutOfMemoryException(createErrorMsg(this, size), e);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ public DrillBuf buffer(int size) {
+ return buffer(size, size);
+ }
+
+ @Override
+ public ByteBufAllocator getUnderlyingAllocator() {
+ return innerAllocator;
+ }
+
+ @Override
+ public BufferAllocator getChildAllocator(LimitConsumer limitConsumer, long initialReservation,
+ long maximumReservation,
+ boolean applyFragmentLimit) {
+ if (!childAcct.reserve(initialReservation)) {
+ throw new OutOfMemoryException(
+ String
+ .format(
+ "You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.",
+ initialReservation, childAcct.getAvailable()));
+ }
+ logger.debug("New child allocator with initial reservation {}", initialReservation);
+ ChildAllocator newChildAllocator = new ChildAllocator(limitConsumer, childAcct, maximumReservation,
+ initialReservation, null, applyFragmentLimit);
+ this.children.put(newChildAllocator, Thread.currentThread().getStackTrace());
+ return newChildAllocator;
+ }
+
+ public PreAllocator getNewPreAllocator() {
+ return new PreAlloc(this, this.childAcct);
+ }
+
+ @Override
+ public void resetLimits(){
+ childAcct.resetFragmentLimits();
+ }
+
+ @Override
+ public void setLimit(long limit){
+ childAcct.setFragmentLimit(limit);
+ }
+
+ @Override
+ public long getLimit(){
+ return childAcct.getFragmentLimit();
+ }
+
+ @Override
+ public void close() {
+ if (ENABLE_ACCOUNTING) {
+ if (thisMap != null) {
+ thisMap.remove(this);
+ }
+ for (ChildAllocator child : children.keySet()) {
+ if (!child.isClosed()) {
+ StringBuilder sb = new StringBuilder();
+ StackTraceElement[] elements = children.get(child);
+ for (int i = 1; i < elements.length; i++) {
+ sb.append("\t\t");
+ sb.append(elements[i]);
+ sb.append("\n");
+ }
+
+
+ IllegalStateException e = new IllegalStateException(String.format(
+ "Failure while trying to close child allocator: Child level allocators not closed. Identifier: %s. Stack trace: \n %s",
+ limitConsumer.getIdentifier(), sb.toString()));
+ if (errorOnLeak) {
+ throw e;
+ } else {
+ logger.warn("Memory leak.", e);
+ }
+ }
+ }
+ }
+ childAcct.close();
+ closed = true;
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public long getAllocatedMemory() {
+ return childAcct.getAllocation();
+ }
+
+ @Override
+ public long getPeakMemoryAllocation() {
+ return childAcct.getPeakMemoryAllocation();
+ }
+
+ @Override
+ public DrillBuf getEmpty() {
+ return empty;
+ }
+
+
+ }
+
+ public PreAllocator getNewPreAllocator() {
+ return new PreAlloc(this, this.acct);
+ }
+
+ public class PreAlloc implements PreAllocator{
+ int bytes = 0;
+ final Accountor acct;
+ final BufferAllocator allocator;
+ private PreAlloc(BufferAllocator allocator, Accountor acct) {
+ this.acct = acct;
+ this.allocator = allocator;
+ }
+
+ /**
+ *
+ */
+ public boolean preAllocate(int bytes) {
+
+ if (!acct.reserve(bytes)) {
+ return false;
+ }
+ this.bytes += bytes;
+ return true;
+
+ }
+
+
+ public DrillBuf getAllocation() {
+ DrillBuf b = new DrillBuf(allocator, acct, innerAllocator.directBuffer(bytes, bytes));
+ acct.reserved(bytes, b);
+ return b;
+ }
+ }
+
+ private static String createErrorMsg(final BufferAllocator allocator, final int size) {
+ return String.format("Unable to allocate buffer of size %d due to memory limit. Current allocation: %d",
+ size, allocator.getAllocatedMemory());
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/impl/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
----------------------------------------------------------------------
diff --git a/exec/memory/impl/src/test/java/org/apache/drill/exec/memory/TestEndianess.java b/exec/memory/impl/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
new file mode 100644
index 0000000..2028a23
--- /dev/null
+++ b/exec/memory/impl/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import static org.junit.Assert.assertEquals;
+import io.netty.buffer.ByteBuf;
+
+import org.apache.drill.common.DrillAutoCloseables;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.test.DrillTest;
+import org.junit.Test;
+
+
+public class TestEndianess extends DrillTest {
+ @Test
+ public void testLittleEndian() {
+ final DrillConfig drillConfig = DrillConfig.create();
+ final BufferAllocator a = RootAllocatorFactory.newRoot(drillConfig);
+ final ByteBuf b = a.buffer(4);
+ b.setInt(0, 35);
+ assertEquals(b.getByte(0), 35);
+ assertEquals(b.getByte(1), 0);
+ assertEquals(b.getByte(2), 0);
+ assertEquals(b.getByte(3), 0);
+ b.release();
+ DrillAutoCloseables.closeNoChecked(a);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/memory/pom.xml
----------------------------------------------------------------------
diff --git a/exec/memory/pom.xml b/exec/memory/pom.xml
new file mode 100644
index 0000000..8da11aa
--- /dev/null
+++ b/exec/memory/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>exec-parent</artifactId>
+ <groupId>org.apache.drill.exec</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+
+ <groupId>org.apache.drill.memory</groupId>
+ <artifactId>memory-parent</artifactId>
+ <packaging>pom</packaging>
+ <name>exec/memory/Parent Pom</name>
+
+ <modules>
+ <module>base</module>
+ <module>impl</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/pom.xml b/exec/pom.xml
index ebad5fa..0ad4d9c 100644
--- a/exec/pom.xml
+++ b/exec/pom.xml
@@ -45,6 +45,8 @@
<modules>
+ <module>memory</module>
+ <module>rpc</module>
<module>vector</module>
<module>java-exec</module>
<module>jdbc</module>
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/pom.xml
----------------------------------------------------------------------
diff --git a/exec/rpc/pom.xml b/exec/rpc/pom.xml
new file mode 100644
index 0000000..751a18a
--- /dev/null
+++ b/exec/rpc/pom.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>exec-parent</artifactId>
+ <groupId>org.apache.drill.exec</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>drill-rpc</artifactId>
+ <name>exec/rpc</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-protocol</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.memory</groupId>
+ <artifactId>drill-memory-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ <classifier>linux-x86_64</classifier>
+ <version>4.0.27.Final</version>
+ </dependency>
+
+
+
+ </dependencies>
+
+
+ <build>
+ </build>
+
+
+
+</project>
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/AbstractHandshakeHandler.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/AbstractHandshakeHandler.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/AbstractHandshakeHandler.java
new file mode 100644
index 0000000..9f426e4
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/AbstractHandshakeHandler.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.MessageToMessageDecoder;
+
+import java.util.List;
+
+import com.google.protobuf.Internal.EnumLite;
+import com.google.protobuf.MessageLite;
+import com.google.protobuf.Parser;
+
+public abstract class AbstractHandshakeHandler<T extends MessageLite> extends MessageToMessageDecoder<InboundRpcMessage> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractHandshakeHandler.class);
+
+ protected final EnumLite handshakeType;
+ protected final Parser<T> parser;
+ protected volatile int coordinationId;
+
+ public AbstractHandshakeHandler(EnumLite handshakeType, Parser<T> parser) {
+ super();
+ this.handshakeType = handshakeType;
+ this.parser = parser;
+ }
+
+
+ @Override
+ protected void decode(ChannelHandlerContext ctx, InboundRpcMessage inbound, List<Object> outputs) throws Exception {
+ try {
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Received handshake {}", inbound);
+ }
+ this.coordinationId = inbound.coordinationId;
+ ctx.channel().pipeline().remove(this);
+ if (inbound.rpcType != handshakeType.getNumber()) {
+ throw new RpcException(String.format("Handshake failure. Expected %s[%d] but received number [%d]",
+ handshakeType, handshakeType.getNumber(), inbound.rpcType));
+ }
+
+ T msg = parser.parseFrom(inbound.getProtobufBodyAsIS());
+ consumeHandshake(ctx, msg);
+ } finally {
+ // Consuming a handshake may result in exceptions, so make sure to release the message buffers.
+ inbound.pBody.release();
+ if (inbound.dBody != null) {
+ inbound.dBody.release();
+ }
+ }
+ }
+
+ protected abstract void consumeHandshake(ChannelHandlerContext ctx, T msg) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/Acks.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/Acks.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/Acks.java
new file mode 100644
index 0000000..3656717
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/Acks.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
+
+public class Acks {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Acks.class);
+
+ public static final Ack OK = Ack.newBuilder().setOk(true).build();
+ public static final Ack FAIL = Ack.newBuilder().setOk(false).build();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BaseRpcOutcomeListener.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BaseRpcOutcomeListener.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BaseRpcOutcomeListener.java
new file mode 100644
index 0000000..dfc62e0
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BaseRpcOutcomeListener.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
+
+public class BaseRpcOutcomeListener<T> implements RpcOutcomeListener<T> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseRpcOutcomeListener.class);
+
+ @Override
+ public void failed(RpcException ex) {
+ }
+
+ @Override
+ public void success(T value, ByteBuf buffer) {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void interrupted(final InterruptedException ex) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicClient.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
new file mode 100644
index 0000000..cf09be3
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
@@ -0,0 +1,316 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.timeout.IdleState;
+import io.netty.handler.timeout.IdleStateEvent;
+import io.netty.handler.timeout.IdleStateHandler;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+
+import java.net.SocketAddress;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
+import org.apache.drill.exec.rpc.RpcConnectionHandler.FailureType;
+
+import com.google.protobuf.Internal.EnumLite;
+import com.google.protobuf.MessageLite;
+import com.google.protobuf.Parser;
+
+public abstract class BasicClient<T extends EnumLite, R extends RemoteConnection, HANDSHAKE_SEND extends MessageLite, HANDSHAKE_RESPONSE extends MessageLite>
+ extends RpcBus<T, R> {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicClient.class);
+
+ // The percentage of time that should pass before sending a ping message to ensure server doesn't time us out. For
+ // example, if timeout is set to 30 seconds and we set percentage to 0.5, then if no write has happened within 15
+ // seconds, the idle state handler will send a ping message.
+ private static final double PERCENT_TIMEOUT_BEFORE_SENDING_PING = 0.5;
+
+ private final Bootstrap b;
+ protected R connection;
+ private final T handshakeType;
+ private final Class<HANDSHAKE_RESPONSE> responseClass;
+ private final Parser<HANDSHAKE_RESPONSE> handshakeParser;
+
+ private final IdlePingHandler pingHandler;
+
+ public BasicClient(RpcConfig rpcMapping, ByteBufAllocator alloc, EventLoopGroup eventLoopGroup, T handshakeType,
+ Class<HANDSHAKE_RESPONSE> responseClass, Parser<HANDSHAKE_RESPONSE> handshakeParser) {
+ super(rpcMapping);
+ this.responseClass = responseClass;
+ this.handshakeType = handshakeType;
+ this.handshakeParser = handshakeParser;
+ final long timeoutInMillis = rpcMapping.hasTimeout() ? (long) (rpcMapping.getTimeout() * 1000.0 * PERCENT_TIMEOUT_BEFORE_SENDING_PING)
+ : -1;
+ this.pingHandler = rpcMapping.hasTimeout() ? new IdlePingHandler(timeoutInMillis) : null;
+
+ b = new Bootstrap() //
+ .group(eventLoopGroup) //
+ .channel(TransportCheck.getClientSocketChannel()) //
+ .option(ChannelOption.ALLOCATOR, alloc) //
+ .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30 * 1000)
+ .option(ChannelOption.SO_REUSEADDR, true)
+ .option(ChannelOption.SO_RCVBUF, 1 << 17) //
+ .option(ChannelOption.SO_SNDBUF, 1 << 17) //
+ .option(ChannelOption.TCP_NODELAY, true)
+ .handler(new ChannelInitializer<SocketChannel>() {
+
+ @Override
+ protected void initChannel(SocketChannel ch) throws Exception {
+ // logger.debug("initializing client connection.");
+ connection = initRemoteConnection(ch);
+
+ ch.closeFuture().addListener(getCloseHandler(ch, connection));
+
+ final ChannelPipeline pipe = ch.pipeline();
+
+ pipe.addLast("protocol-decoder", getDecoder(connection.getAllocator()));
+ pipe.addLast("message-decoder", new RpcDecoder("c-" + rpcConfig.getName()));
+ pipe.addLast("protocol-encoder", new RpcEncoder("c-" + rpcConfig.getName()));
+ pipe.addLast("handshake-handler", new ClientHandshakeHandler());
+
+ if(pingHandler != null){
+ pipe.addLast("idle-state-handler", pingHandler);
+ }
+
+ pipe.addLast("message-handler", new InboundHandler(connection));
+ pipe.addLast("exception-handler", new RpcExceptionHandler(connection));
+ }
+ }); //
+
+ // if(TransportCheck.SUPPORTS_EPOLL){
+ // b.option(EpollChannelOption.SO_REUSEPORT, true); //
+ // }
+ }
+
+ public R initRemoteConnection(SocketChannel channel){
+ local=channel.localAddress();
+ remote=channel.remoteAddress();
+ return null;
+ };
+
+ private static final OutboundRpcMessage PING_MESSAGE = new OutboundRpcMessage(RpcMode.PING, 0, 0, Acks.OK);
+
+ /**
+ * Handler that watches for situations where we haven't read from the socket in a certain timeout. If we exceed this
+ * timeout, we send a PING message to the server to state that we are still alive.
+ */
+ private class IdlePingHandler extends IdleStateHandler {
+
+ private GenericFutureListener<Future<? super Void>> pingFailedHandler = new GenericFutureListener<Future<? super Void>>() {
+ public void operationComplete(Future<? super Void> future) throws Exception {
+ if (!future.isSuccess()) {
+ logger.error("Unable to maintain connection {}. Closing connection.", connection.getName());
+ connection.close();
+ }
+ }
+ };
+
+ public IdlePingHandler(long idleWaitInMillis) {
+ super(0, idleWaitInMillis, 0, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ protected void channelIdle(ChannelHandlerContext ctx, IdleStateEvent evt) throws Exception {
+ if (evt.state() == IdleState.WRITER_IDLE) {
+ ctx.writeAndFlush(PING_MESSAGE).addListener(pingFailedHandler);
+ }
+ }
+ }
+
+ public abstract ProtobufLengthDecoder getDecoder(BufferAllocator allocator);
+
+ public boolean isActive() {
+ return connection != null
+ && connection.getChannel() != null
+ && connection.getChannel().isActive();
+ }
+
+ protected abstract void validateHandshake(HANDSHAKE_RESPONSE validateHandshake) throws RpcException;
+
+ protected abstract void finalizeConnection(HANDSHAKE_RESPONSE handshake, R connection);
+
+ public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener,
+ T rpcType, SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
+ super.send(listener, connection, rpcType, protobufBody, clazz, dataBodies);
+ }
+
+ public <SEND extends MessageLite, RECEIVE extends MessageLite> DrillRpcFuture<RECEIVE> send(T rpcType,
+ SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
+ return super.send(connection, rpcType, protobufBody, clazz, dataBodies);
+ }
+
+ @Override
+ public boolean isClient() {
+ return true;
+ }
+
+ protected void connectAsClient(RpcConnectionHandler<R> connectionListener, HANDSHAKE_SEND handshakeValue,
+ String host, int port) {
+ ConnectionMultiListener cml = new ConnectionMultiListener(connectionListener, handshakeValue);
+ b.connect(host, port).addListener(cml.connectionHandler);
+ }
+
+ private class ConnectionMultiListener {
+ private final RpcConnectionHandler<R> l;
+ private final HANDSHAKE_SEND handshakeValue;
+
+ public ConnectionMultiListener(RpcConnectionHandler<R> l, HANDSHAKE_SEND handshakeValue) {
+ assert l != null;
+ assert handshakeValue != null;
+
+ this.l = l;
+ this.handshakeValue = handshakeValue;
+ }
+
+ public final ConnectionHandler connectionHandler = new ConnectionHandler();
+ public final HandshakeSendHandler handshakeSendHandler = new HandshakeSendHandler();
+
+ /**
+ * Manages connection establishment outcomes.
+ */
+ private class ConnectionHandler implements GenericFutureListener<ChannelFuture> {
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ boolean isInterrupted = false;
+
+ // We want to wait for at least 120 secs when interrupts occur. Establishing a connection fails/succeeds quickly,
+ // So there is no point propagating the interruption as failure immediately.
+ long remainingWaitTimeMills = 120000;
+ long startTime = System.currentTimeMillis();
+ // logger.debug("Connection operation finished. Success: {}", future.isSuccess());
+ while(true) {
+ try {
+ future.get(remainingWaitTimeMills, TimeUnit.MILLISECONDS);
+ if (future.isSuccess()) {
+ SocketAddress remote = future.channel().remoteAddress();
+ SocketAddress local = future.channel().localAddress();
+ setAddresses(remote, local);
+ // send a handshake on the current thread. This is the only time we will send from within the event thread.
+ // We can do this because the connection will not be backed up.
+ send(handshakeSendHandler, connection, handshakeType, handshakeValue, responseClass, true);
+ } else {
+ l.connectionFailed(FailureType.CONNECTION, new RpcException("General connection failure."));
+ }
+ // logger.debug("Handshake queued for send.");
+ break;
+ } catch (final InterruptedException interruptEx) {
+ remainingWaitTimeMills -= (System.currentTimeMillis() - startTime);
+ startTime = System.currentTimeMillis();
+ isInterrupted = true;
+ if (remainingWaitTimeMills < 1) {
+ l.connectionFailed(FailureType.CONNECTION, interruptEx);
+ break;
+ }
+ // Ignore the interrupt and continue to wait until we elapse remainingWaitTimeMills.
+ } catch (final Exception ex) {
+ logger.error("Failed to establish connection", ex);
+ l.connectionFailed(FailureType.CONNECTION, ex);
+ break;
+ }
+ }
+
+ if (isInterrupted) {
+ // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
+ // interruption and respond to it if it wants to.
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ /**
+ * manages handshake outcomes.
+ */
+ private class HandshakeSendHandler implements RpcOutcomeListener<HANDSHAKE_RESPONSE> {
+
+ @Override
+ public void failed(RpcException ex) {
+ logger.debug("Failure while initiating handshake", ex);
+ l.connectionFailed(FailureType.HANDSHAKE_COMMUNICATION, ex);
+ }
+
+ @Override
+ public void success(HANDSHAKE_RESPONSE value, ByteBuf buffer) {
+ // logger.debug("Handshake received. {}", value);
+ try {
+ BasicClient.this.validateHandshake(value);
+ BasicClient.this.finalizeConnection(value, connection);
+ l.connectionSucceeded(connection);
+ // logger.debug("Handshake completed succesfully.");
+ } catch (RpcException ex) {
+ l.connectionFailed(FailureType.HANDSHAKE_VALIDATION, ex);
+ }
+ }
+
+ @Override
+ public void interrupted(final InterruptedException ex) {
+ logger.warn("Interrupted while waiting for handshake response", ex);
+ l.connectionFailed(FailureType.HANDSHAKE_COMMUNICATION, ex);
+ }
+ }
+
+ }
+
+ private class ClientHandshakeHandler extends AbstractHandshakeHandler<HANDSHAKE_RESPONSE> {
+
+ public ClientHandshakeHandler() {
+ super(BasicClient.this.handshakeType, BasicClient.this.handshakeParser);
+ }
+
+ @Override
+ protected final void consumeHandshake(ChannelHandlerContext ctx, HANDSHAKE_RESPONSE msg) throws Exception {
+ // remove the handshake information from the queue so it doesn't sit there forever.
+ RpcOutcome<HANDSHAKE_RESPONSE> response = queue.getFuture(handshakeType.getNumber(), coordinationId,
+ responseClass);
+ response.set(msg, null);
+ }
+
+ }
+
+ public void setAutoRead(boolean enableAutoRead) {
+ connection.setAutoRead(enableAutoRead);
+ }
+
+ public void close() {
+ logger.debug("Closing client");
+ try {
+ connection.getChannel().close().get();
+ } catch (final InterruptedException | ExecutionException e) {
+ logger.warn("Failure while shutting {}", this.getClass().getName(), e);
+
+ // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
+ // interruption and respond to it if it wants to.
+ Thread.currentThread().interrupt();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
new file mode 100644
index 0000000..1cacf9c
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.rpc.BasicClientWithConnection.ServerConnection;
+
+import com.google.protobuf.Internal.EnumLite;
+import com.google.protobuf.MessageLite;
+import com.google.protobuf.Parser;
+
+public abstract class BasicClientWithConnection<T extends EnumLite, HANDSHAKE_SEND extends MessageLite, HANDSHAKE_RESPONSE extends MessageLite> extends BasicClient<T, ServerConnection, HANDSHAKE_SEND, HANDSHAKE_RESPONSE>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicClientWithConnection.class);
+
+ private BufferAllocator alloc;
+ private final String connectionName;
+
+ public BasicClientWithConnection(RpcConfig rpcMapping, BufferAllocator alloc, EventLoopGroup eventLoopGroup, T handshakeType,
+ Class<HANDSHAKE_RESPONSE> responseClass, Parser<HANDSHAKE_RESPONSE> handshakeParser, String connectionName) {
+ super(rpcMapping, alloc.getUnderlyingAllocator(), eventLoopGroup, handshakeType, responseClass, handshakeParser);
+ this.alloc = alloc;
+ this.connectionName = connectionName;
+ }
+
+ @Override
+ protected Response handle(ServerConnection connection, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException {
+ return handleReponse( (ConnectionThrottle) connection, rpcType, pBody, dBody);
+ }
+
+ protected abstract Response handleReponse(ConnectionThrottle throttle, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException ;
+
+ @Override
+ public ServerConnection initRemoteConnection(SocketChannel channel) {
+ super.initRemoteConnection(channel);
+ return new ServerConnection(connectionName, channel, alloc);
+ }
+
+ public static class ServerConnection extends RemoteConnection{
+
+ private final BufferAllocator alloc;
+
+ public ServerConnection(String name, SocketChannel channel, BufferAllocator alloc) {
+ super(channel, name);
+ this.alloc = alloc;
+ }
+
+ @Override
+ public BufferAllocator getAllocator() {
+ return alloc;
+ }
+
+
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicServer.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
new file mode 100644
index 0000000..2eebe59
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
@@ -0,0 +1,237 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.timeout.ReadTimeoutHandler;
+
+import java.io.IOException;
+import java.net.BindException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
+
+import com.google.common.base.Stopwatch;
+import com.google.protobuf.Internal.EnumLite;
+import com.google.protobuf.MessageLite;
+import com.google.protobuf.Parser;
+
+/**
+ * A server is bound to a port and is responsible for responding to various type of requests. In some cases, the inbound
+ * requests will generate more than one outbound request.
+ */
+public abstract class BasicServer<T extends EnumLite, C extends RemoteConnection> extends RpcBus<T, C> {
+ final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(this.getClass());
+
+ protected static final String TIMEOUT_HANDLER = "timeout-handler";
+
+ private ServerBootstrap b;
+ private volatile boolean connect = false;
+ private final EventLoopGroup eventLoopGroup;
+
+ public BasicServer(final RpcConfig rpcMapping, ByteBufAllocator alloc, EventLoopGroup eventLoopGroup) {
+ super(rpcMapping);
+ this.eventLoopGroup = eventLoopGroup;
+
+ b = new ServerBootstrap()
+ .channel(TransportCheck.getServerSocketChannel())
+ .option(ChannelOption.SO_BACKLOG, 1000)
+ .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30*1000)
+ .option(ChannelOption.TCP_NODELAY, true)
+ .option(ChannelOption.SO_REUSEADDR, true)
+ .option(ChannelOption.SO_RCVBUF, 1 << 17)
+ .option(ChannelOption.SO_SNDBUF, 1 << 17)
+ .group(eventLoopGroup) //
+ .childOption(ChannelOption.ALLOCATOR, alloc)
+
+ // .handler(new LoggingHandler(LogLevel.INFO))
+
+ .childHandler(new ChannelInitializer<SocketChannel>() {
+ @Override
+ protected void initChannel(SocketChannel ch) throws Exception {
+// logger.debug("Starting initialization of server connection.");
+ C connection = initRemoteConnection(ch);
+ ch.closeFuture().addListener(getCloseHandler(ch, connection));
+
+ final ChannelPipeline pipe = ch.pipeline();
+ pipe.addLast("protocol-decoder", getDecoder(connection.getAllocator(), getOutOfMemoryHandler()));
+ pipe.addLast("message-decoder", new RpcDecoder("s-" + rpcConfig.getName()));
+ pipe.addLast("protocol-encoder", new RpcEncoder("s-" + rpcConfig.getName()));
+ pipe.addLast("handshake-handler", getHandshakeHandler(connection));
+
+ if (rpcMapping.hasTimeout()) {
+ pipe.addLast(TIMEOUT_HANDLER,
+ new LogggingReadTimeoutHandler(connection, rpcMapping.getTimeout()));
+ }
+
+ pipe.addLast("message-handler", new InboundHandler(connection));
+ pipe.addLast("exception-handler", new RpcExceptionHandler(connection));
+
+ connect = true;
+// logger.debug("Server connection initialization completed.");
+ }
+ });
+
+// if(TransportCheck.SUPPORTS_EPOLL){
+// b.option(EpollChannelOption.SO_REUSEPORT, true); //
+// }
+ }
+
+ private class LogggingReadTimeoutHandler<C extends RemoteConnection> extends ReadTimeoutHandler {
+
+ private final C connection;
+ private final int timeoutSeconds;
+ public LogggingReadTimeoutHandler(C connection, int timeoutSeconds) {
+ super(timeoutSeconds);
+ this.connection = connection;
+ this.timeoutSeconds = timeoutSeconds;
+ }
+
+ @Override
+ protected void readTimedOut(ChannelHandlerContext ctx) throws Exception {
+ logger.info("RPC connection {} timed out. Timeout was set to {} seconds. Closing connection.", connection.getName(),
+ timeoutSeconds);
+ super.readTimedOut(ctx);
+ }
+
+ }
+
+ public OutOfMemoryHandler getOutOfMemoryHandler() {
+ return OutOfMemoryHandler.DEFAULT_INSTANCE;
+ }
+
+ protected void removeTimeoutHandler() {
+
+ }
+
+ public abstract ProtobufLengthDecoder getDecoder(BufferAllocator allocator, OutOfMemoryHandler outOfMemoryHandler);
+
+ @Override
+ public boolean isClient() {
+ return false;
+ }
+
+ protected abstract ServerHandshakeHandler<?> getHandshakeHandler(C connection);
+
+ protected static abstract class ServerHandshakeHandler<T extends MessageLite> extends AbstractHandshakeHandler<T> {
+
+ public ServerHandshakeHandler(EnumLite handshakeType, Parser<T> parser) {
+ super(handshakeType, parser);
+ }
+
+ @Override
+ protected void consumeHandshake(ChannelHandlerContext ctx, T inbound) throws Exception {
+ OutboundRpcMessage msg = new OutboundRpcMessage(RpcMode.RESPONSE, this.handshakeType, coordinationId,
+ getHandshakeResponse(inbound));
+ ctx.writeAndFlush(msg);
+ }
+
+ public abstract MessageLite getHandshakeResponse(T inbound) throws Exception;
+
+ }
+
+ @Override
+ protected MessageLite getResponseDefaultInstance(int rpcType) throws RpcException {
+ return null;
+ }
+
+ @Override
+ protected Response handle(C connection, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException {
+ return null;
+ }
+
+ @Override
+ public <SEND extends MessageLite, RECEIVE extends MessageLite> DrillRpcFuture<RECEIVE> send(C connection, T rpcType,
+ SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
+ return super.send(connection, rpcType, protobufBody, clazz, dataBodies);
+ }
+
+ @Override
+ public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener,
+ C connection, T rpcType, SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
+ super.send(listener, connection, rpcType, protobufBody, clazz, dataBodies);
+ }
+
+ @Override
+ public C initRemoteConnection(SocketChannel channel) {
+ local = channel.localAddress();
+ remote = channel.remoteAddress();
+ return null;
+ }
+
+ public int bind(final int initialPort, boolean allowPortHunting) {
+ int port = initialPort - 1;
+ while (true) {
+ try {
+ b.bind(++port).sync();
+ break;
+ } catch (Exception e) {
+ // TODO(DRILL-3026): Revisit: Exception is not (always) BindException.
+ // One case is "java.io.IOException: bind() failed: Address already in
+ // use".
+ if (e instanceof BindException && allowPortHunting) {
+ continue;
+ }
+ final UserException bindException =
+ UserException
+ .resourceError( e )
+ .addContext( "Server type", getClass().getSimpleName() )
+ .message( "Drillbit could not bind to port %s.", port )
+ .build(logger);
+ throw bindException;
+ }
+ }
+
+ connect = !connect;
+ logger.debug("Server of type {} started on port {}.", getClass().getSimpleName(), port);
+ return port;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ Stopwatch watch = new Stopwatch().start();
+ // this takes 1s to complete
+ // known issue: https://github.com/netty/netty/issues/2545
+ eventLoopGroup.shutdownGracefully(0, 0, TimeUnit.SECONDS).get();
+ long elapsed = watch.elapsed(MILLISECONDS);
+ if (elapsed > 500) {
+ logger.info("closed eventLoopGroup " + eventLoopGroup + " in " + elapsed + " ms");
+ }
+ } catch (final InterruptedException | ExecutionException e) {
+ logger.warn("Failure while shutting down {}. ", this.getClass().getName(), e);
+
+ // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
+ // interruption and respond to it if it wants to.
+ Thread.currentThread().interrupt();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ChannelClosedException.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ChannelClosedException.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ChannelClosedException.java
new file mode 100644
index 0000000..b68efae
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ChannelClosedException.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+public class ChannelClosedException extends RpcException{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ChannelClosedException.class);
+
+ public ChannelClosedException() {
+ super();
+ }
+
+ public ChannelClosedException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ChannelClosedException(String message) {
+ super(message);
+ }
+
+ public ChannelClosedException(Throwable cause) {
+ super(cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ChannelListenerWithCoordinationId.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ChannelListenerWithCoordinationId.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ChannelListenerWithCoordinationId.java
new file mode 100644
index 0000000..3b95dd8
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ChannelListenerWithCoordinationId.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.channel.ChannelFuture;
+import io.netty.util.concurrent.GenericFutureListener;
+
+public interface ChannelListenerWithCoordinationId extends GenericFutureListener<ChannelFuture>{
+ public int getCoordinationId();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ConnectionThrottle.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ConnectionThrottle.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ConnectionThrottle.java
new file mode 100644
index 0000000..e496dcc
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ConnectionThrottle.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+public interface ConnectionThrottle {
+ public void setAutoRead(boolean enableAutoRead);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java
new file mode 100644
index 0000000..5a5bbab
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelFuture;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.drill.common.exceptions.UserRemoteException;
+import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
+
+/**
+ * Manages the creation of rpc futures for a particular socket.
+ */
+public class CoordinationQueue {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoordinationQueue.class);
+
+ private final PositiveAtomicInteger circularInt = new PositiveAtomicInteger();
+ private final Map<Integer, RpcOutcome<?>> map;
+
+ public CoordinationQueue(int segmentSize, int segmentCount) {
+ map = new ConcurrentHashMap<Integer, RpcOutcome<?>>(segmentSize, 0.75f, segmentCount);
+ }
+
+ void channelClosed(Throwable ex) {
+ if (ex != null) {
+ RpcException e;
+ if (ex instanceof RpcException) {
+ e = (RpcException) ex;
+ } else {
+ e = new RpcException(ex);
+ }
+ for (RpcOutcome<?> f : map.values()) {
+ f.setException(e);
+ }
+ }
+ }
+
+ public <V> ChannelListenerWithCoordinationId get(RpcOutcomeListener<V> handler, Class<V> clazz, RemoteConnection connection) {
+ int i = circularInt.getNext();
+ RpcListener<V> future = new RpcListener<V>(handler, clazz, i, connection);
+ Object old = map.put(i, future);
+ if (old != null) {
+ throw new IllegalStateException(
+ "You attempted to reuse a coordination id when the previous coordination id has not been removed. This is likely rpc future callback memory leak.");
+ }
+ return future;
+ }
+
+ private class RpcListener<T> implements ChannelListenerWithCoordinationId, RpcOutcome<T>{
+ final RpcOutcomeListener<T> handler;
+ final Class<T> clazz;
+ final int coordinationId;
+ final RemoteConnection connection;
+
+ public RpcListener(RpcOutcomeListener<T> handler, Class<T> clazz, int coordinationId, RemoteConnection connection) {
+ super();
+ this.handler = handler;
+ this.clazz = clazz;
+ this.coordinationId = coordinationId;
+ this.connection = connection;
+ }
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+
+ if (!future.isSuccess()) {
+ removeFromMap(coordinationId);
+ if (future.channel().isActive()) {
+ throw new RpcException("Future failed") ;
+ } else {
+ setException(new ChannelClosedException());
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void set(Object value, ByteBuf buffer) {
+ assert clazz.isAssignableFrom(value.getClass());
+ handler.success( (T) value, buffer);
+ }
+
+ @Override
+ public void setException(Throwable t) {
+ handler.failed(RpcException.mapException(t));
+ }
+
+ @Override
+ public Class<T> getOutcomeType() {
+ return clazz;
+ }
+
+ @Override
+ public int getCoordinationId() {
+ return coordinationId;
+ }
+
+ }
+
+ private RpcOutcome<?> removeFromMap(int coordinationId) {
+ RpcOutcome<?> rpc = map.remove(coordinationId);
+ if (rpc == null) {
+ throw new IllegalStateException(
+ "Attempting to retrieve an rpc that wasn't first stored in the rpc coordination queue. This would most likely happen if you're opposite endpoint sent multiple messages on the same coordination id.");
+ }
+ return rpc;
+ }
+
+ public <V> RpcOutcome<V> getFuture(int rpcType, int coordinationId, Class<V> clazz) {
+
+ RpcOutcome<?> rpc = removeFromMap(coordinationId);
+ // logger.debug("Got rpc from map {}", rpc);
+ Class<?> outcomeClass = rpc.getOutcomeType();
+
+ if (outcomeClass != clazz) {
+ throw new IllegalStateException(
+ String
+ .format(
+ "RPC Engine had a submission and response configuration mismatch. The RPC request that you submitted was defined with an expected response type of %s. However, "
+ + "when the response returned, a call to getResponseDefaultInstance() with Rpc number %d provided an expected class of %s. This means either your submission uses the wrong type definition"
+ + "or your getResponseDefaultInstance() method responds the wrong instance type ",
+ clazz.getCanonicalName(), rpcType, outcomeClass.getCanonicalName()));
+ }
+
+ @SuppressWarnings("unchecked")
+ RpcOutcome<V> crpc = (RpcOutcome<V>) rpc;
+
+ // logger.debug("Returning casted future");
+ return crpc;
+ }
+
+ public void updateFailedFuture(int coordinationId, DrillPBError failure) {
+ // logger.debug("Updating failed future.");
+ try {
+ RpcOutcome<?> rpc = removeFromMap(coordinationId);
+ rpc.setException(new UserRemoteException(failure));
+ } catch(Exception ex) {
+ logger.warn("Failed to remove from map. Not a problem since we were updating on failed future.", ex);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/DrillRpcFuture.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/DrillRpcFuture.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/DrillRpcFuture.java
new file mode 100644
index 0000000..d044432
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/DrillRpcFuture.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+import com.google.common.util.concurrent.CheckedFuture;
+
+public interface DrillRpcFuture<T> extends CheckedFuture<T,RpcException> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRpcFuture.class);
+
+ public ByteBuf getBuffer();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/DrillRpcFutureImpl.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/DrillRpcFutureImpl.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/DrillRpcFutureImpl.java
new file mode 100644
index 0000000..cbe63c6
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/DrillRpcFutureImpl.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+import com.google.common.util.concurrent.AbstractCheckedFuture;
+import com.google.common.util.concurrent.AbstractFuture;
+
+class DrillRpcFutureImpl<V> extends AbstractCheckedFuture<V, RpcException> implements DrillRpcFuture<V>, RpcOutcomeListener<V>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRpcFutureImpl.class);
+
+ private volatile ByteBuf buffer;
+
+ public DrillRpcFutureImpl() {
+ super(new InnerFuture<V>());
+ }
+
+ @Override
+ protected RpcException mapException(Exception ex) {
+ return RpcException.mapException(ex);
+ }
+
+ private static class InnerFuture<T> extends AbstractFuture<T> {
+ // we rewrite these so that the parent can see them
+
+ void setValue(T value) {
+ super.set(value);
+ }
+
+ @Override
+ protected boolean setException(Throwable t) {
+ return super.setException(t);
+ }
+ }
+
+ @Override
+ public void failed(RpcException ex) {
+ ( (InnerFuture<V>)delegate()).setException(ex);
+ }
+
+ @Override
+ public void success(V value, ByteBuf buffer) {
+ this.buffer = buffer;
+ ( (InnerFuture<V>)delegate()).setValue(value);
+ }
+
+ @Override
+ public void interrupted(final InterruptedException ex) {
+ // Propagate the interrupt to inner future
+ ( (InnerFuture<V>)delegate()).cancel(true);
+ }
+
+ @Override
+ public ByteBuf getBuffer() {
+ return buffer;
+ }
+
+ public void release() {
+ if (buffer != null) {
+ buffer.release();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/FutureBitCommand.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/FutureBitCommand.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/FutureBitCommand.java
new file mode 100644
index 0000000..53d0772
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/FutureBitCommand.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.protobuf.MessageLite;
+
+public abstract class FutureBitCommand<T extends MessageLite, C extends RemoteConnection> implements RpcCommand<T,C> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FutureBitCommand.class);
+
+ protected final SettableFuture<T> settableFuture;
+ private final RpcCheckedFuture<T> parentFuture;
+
+ public FutureBitCommand() {
+ this.settableFuture = SettableFuture.create();
+ this.parentFuture = new RpcCheckedFuture<T>(settableFuture);
+ }
+
+ public abstract void doRpcCall(RpcOutcomeListener<T> outcomeListener, C connection);
+
+ @Override
+ public void connectionAvailable(C connection) {
+
+ doRpcCall(new DeferredRpcOutcome(), connection);
+ }
+
+ @Override
+ public void connectionSucceeded(C connection) {
+ connectionAvailable(connection);
+ }
+
+ private class DeferredRpcOutcome implements RpcOutcomeListener<T> {
+
+ @Override
+ public void failed(RpcException ex) {
+ settableFuture.setException(ex);
+ }
+
+ @Override
+ public void success(T value, ByteBuf buf) {
+ settableFuture.set(value);
+ }
+
+ @Override
+ public void interrupted(final InterruptedException e) {
+ // If we are interrupted while performing the command, consider as failure.
+ logger.warn("Interrupted while running the command", e);
+ failed(new RpcException(e));
+ }
+ }
+
+ public DrillRpcFuture<T> getFuture() {
+ return parentFuture;
+ }
+
+ @Override
+ public void connectionFailed(FailureType type, Throwable t) {
+ settableFuture.setException(RpcException.mapException(
+ String.format("Command failed while establishing connection. Failure type %s.", type), t));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/InboundRpcMessage.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/InboundRpcMessage.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/InboundRpcMessage.java
new file mode 100644
index 0000000..d739034
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/InboundRpcMessage.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
+
+import java.io.InputStream;
+
+import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
+
+public class InboundRpcMessage extends RpcMessage{
+ public ByteBuf pBody;
+ public ByteBuf dBody;
+
+ public InboundRpcMessage(RpcMode mode, int rpcType, int coordinationId, ByteBuf pBody, ByteBuf dBody) {
+ super(mode, rpcType, coordinationId);
+ this.pBody = pBody;
+ this.dBody = dBody;
+ }
+
+ @Override
+ public int getBodySize() {
+ int len = pBody.capacity();
+ if (dBody != null) {
+ len += dBody.capacity();
+ }
+ return len;
+ }
+
+ @Override
+ void release() {
+ if (pBody != null) {
+ pBody.release();
+ }
+ if (dBody != null) {
+ dBody.release();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "InboundRpcMessage [pBody=" + pBody + ", mode=" + mode + ", rpcType=" + rpcType + ", coordinationId="
+ + coordinationId + ", dBody=" + dBody + "]";
+ }
+
+ public InputStream getProtobufBodyAsIS() {
+ return new ByteBufInputStream(pBody);
+ }
+
+}
[35/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
new file mode 100644
index 0000000..c29b9b0
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
@@ -0,0 +1,332 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.math.BigDecimal;
+
+import org.apache.drill.common.expression.IfExpression.IfCondition;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
+import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.joda.time.Period;
+
+import com.google.common.collect.ImmutableList;
+
+public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBuilder, RuntimeException>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionStringBuilder.class);
+
+ static final ExpressionStringBuilder INSTANCE = new ExpressionStringBuilder();
+
+ public static String toString(LogicalExpression expr) {
+ StringBuilder sb = new StringBuilder();
+ expr.accept(INSTANCE, sb);
+ return sb.toString();
+ }
+
+ public static void toString(LogicalExpression expr, StringBuilder sb) {
+ expr.accept(INSTANCE, sb);
+ }
+
+
+ @Override
+ public Void visitFunctionCall(FunctionCall call, StringBuilder sb) throws RuntimeException {
+ ImmutableList<LogicalExpression> args = call.args;
+ sb.append(call.getName());
+ sb.append("(");
+ for (int i = 0; i < args.size(); i++) {
+ if (i != 0) {
+ sb.append(", ");
+ }
+ args.get(i).accept(this, sb);
+ }
+ sb.append(") ");
+ return null;
+ }
+
+ @Override
+ public Void visitBooleanOperator(BooleanOperator op, StringBuilder sb) throws RuntimeException {
+ return visitFunctionCall(op, sb);
+ }
+
+ @Override
+ public Void visitFunctionHolderExpression(FunctionHolderExpression holder, StringBuilder sb) throws RuntimeException {
+ ImmutableList<LogicalExpression> args = holder.args;
+ sb.append(holder.getName());
+ sb.append("(");
+ for (int i = 0; i < args.size(); i++) {
+ if (i != 0) {
+ sb.append(", ");
+ }
+ args.get(i).accept(this, sb);
+ }
+ sb.append(") ");
+ return null;
+ }
+
+ @Override
+ public Void visitIfExpression(IfExpression ifExpr, StringBuilder sb) throws RuntimeException {
+
+ // serialize the if expression
+ sb.append(" ( ");
+ IfCondition c = ifExpr.ifCondition;
+ sb.append("if (");
+ c.condition.accept(this, sb);
+ sb.append(" ) then (");
+ c.expression.accept(this, sb);
+ sb.append(" ) ");
+
+ sb.append(" else (");
+ ifExpr.elseExpression.accept(this, sb);
+ sb.append(" ) ");
+ sb.append(" end ");
+ sb.append(" ) ");
+ return null;
+ }
+
+ @Override
+ public Void visitSchemaPath(SchemaPath path, StringBuilder sb) throws RuntimeException {
+ PathSegment seg = path.getRootSegment();
+ if (seg.isArray()) {
+ throw new IllegalStateException("Drill doesn't currently support top level arrays");
+ }
+ sb.append('`');
+ sb.append(seg.getNameSegment().getPath());
+ sb.append('`');
+
+ while ( (seg = seg.getChild()) != null) {
+ if (seg.isNamed()) {
+ sb.append('.');
+ sb.append('`');
+ sb.append(seg.getNameSegment().getPath());
+ sb.append('`');
+ } else {
+ sb.append('[');
+ sb.append(seg.getArraySegment().getIndex());
+ sb.append(']');
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLongConstant(LongExpression lExpr, StringBuilder sb) throws RuntimeException {
+ sb.append(lExpr.getLong());
+ return null;
+ }
+
+ @Override
+ public Void visitDateConstant(DateExpression lExpr, StringBuilder sb) throws RuntimeException {
+ sb.append("cast( ");
+ sb.append(lExpr.getDate());
+ sb.append(" as DATE)");
+ return null;
+ }
+
+ @Override
+ public Void visitTimeConstant(TimeExpression lExpr, StringBuilder sb) throws RuntimeException {
+ sb.append("cast( ");
+ sb.append(lExpr.getTime());
+ sb.append(" as TIME)");
+ return null;
+ }
+
+ @Override
+ public Void visitTimeStampConstant(TimeStampExpression lExpr, StringBuilder sb) throws RuntimeException {
+ sb.append("cast( ");
+ sb.append(lExpr.getTimeStamp());
+ sb.append(" as TIMESTAMP)");
+ return null;
+ }
+
+ @Override
+ public Void visitIntervalYearConstant(IntervalYearExpression lExpr, StringBuilder sb) throws RuntimeException {
+ sb.append("cast( '");
+ sb.append(Period.months(lExpr.getIntervalYear()).toString());
+ sb.append("' as INTERVALYEAR)");
+ return null;
+ }
+
+ @Override
+ public Void visitIntervalDayConstant(IntervalDayExpression lExpr, StringBuilder sb) throws RuntimeException {
+ sb.append("cast( '");
+ sb.append(Period.days(lExpr.getIntervalDay()).plusMillis(lExpr.getIntervalMillis()).toString());
+ sb.append("' as INTERVALDAY)");
+ return null;
+ }
+
+ @Override
+ public Void visitDecimal9Constant(Decimal9Expression decExpr, StringBuilder sb) throws RuntimeException {
+ BigDecimal value = new BigDecimal(decExpr.getIntFromDecimal());
+ sb.append((value.setScale(decExpr.getScale())).toString());
+ return null;
+ }
+
+ @Override
+ public Void visitDecimal18Constant(Decimal18Expression decExpr, StringBuilder sb) throws RuntimeException {
+ BigDecimal value = new BigDecimal(decExpr.getLongFromDecimal());
+ sb.append((value.setScale(decExpr.getScale())).toString());
+ return null;
+ }
+
+ @Override
+ public Void visitDecimal28Constant(Decimal28Expression decExpr, StringBuilder sb) throws RuntimeException {
+ sb.append(decExpr.toString());
+ return null;
+ }
+
+ @Override
+ public Void visitDecimal38Constant(Decimal38Expression decExpr, StringBuilder sb) throws RuntimeException {
+ sb.append(decExpr.getBigDecimal().toString());
+ return null;
+ }
+
+ @Override
+ public Void visitDoubleConstant(DoubleExpression dExpr, StringBuilder sb) throws RuntimeException {
+ sb.append(dExpr.getDouble());
+ return null;
+ }
+
+ @Override
+ public Void visitBooleanConstant(BooleanExpression e, StringBuilder sb) throws RuntimeException {
+ sb.append(e.getBoolean());
+ return null;
+ }
+
+ @Override
+ public Void visitQuotedStringConstant(QuotedString e, StringBuilder sb) throws RuntimeException {
+ sb.append("'");
+ sb.append(e.value);
+ sb.append("'");
+ return null;
+ }
+
+ @Override
+ public Void visitConvertExpression(ConvertExpression e, StringBuilder sb) throws RuntimeException {
+ sb.append(e.getConvertFunction()).append("(");
+ e.getInput().accept(this, sb);
+ sb.append(", '").append(e.getEncodingType()).append("')");
+ return null;
+ }
+
+ @Override
+ public Void visitCastExpression(CastExpression e, StringBuilder sb) throws RuntimeException {
+ MajorType mt = e.getMajorType();
+
+ sb.append("cast( (");
+ e.getInput().accept(this, sb);
+ sb.append(" ) as ");
+ sb.append(mt.getMinorType().name());
+
+ switch(mt.getMinorType()) {
+ case FLOAT4:
+ case FLOAT8:
+ case BIT:
+ case INT:
+ case TINYINT:
+ case SMALLINT:
+ case BIGINT:
+ case UINT1:
+ case UINT2:
+ case UINT4:
+ case UINT8:
+ case DATE:
+ case TIMESTAMP:
+ case TIMESTAMPTZ:
+ case TIME:
+ case INTERVAL:
+ case INTERVALDAY:
+ case INTERVALYEAR:
+ // do nothing else.
+ break;
+ case VAR16CHAR:
+ case VARBINARY:
+ case VARCHAR:
+ case FIXED16CHAR:
+ case FIXEDBINARY:
+ case FIXEDCHAR:
+
+ // add size in parens
+ sb.append("(");
+ sb.append(mt.getWidth());
+ sb.append(")");
+ break;
+ case DECIMAL9:
+ case DECIMAL18:
+ case DECIMAL28DENSE:
+ case DECIMAL28SPARSE:
+ case DECIMAL38DENSE:
+ case DECIMAL38SPARSE:
+
+ // add scale and precision
+ sb.append("(");
+ sb.append(mt.getPrecision());
+ sb.append(", ");
+ sb.append(mt.getScale());
+ sb.append(")");
+ break;
+ default:
+ throw new UnsupportedOperationException(String.format("Unable to convert cast expression %s into string.", e));
+ }
+ sb.append(" )");
+ return null;
+ }
+
+ @Override
+ public Void visitFloatConstant(FloatExpression fExpr, StringBuilder sb) throws RuntimeException {
+ sb.append(fExpr.getFloat());
+ return null;
+ }
+
+ @Override
+ public Void visitIntConstant(IntExpression intExpr, StringBuilder sb) throws RuntimeException {
+ sb.append(intExpr.getInt());
+ return null;
+ }
+
+ @Override
+ public Void visitNullConstant(TypedNullConstant e, StringBuilder sb) throws RuntimeException {
+ sb.append("NULL");
+ return null;
+ }
+
+ @Override
+ public Void visitNullExpression(NullExpression e, StringBuilder sb) throws RuntimeException {
+ sb.append("NULL");
+ return null;
+ }
+
+ @Override
+ public Void visitUnknown(LogicalExpression e, StringBuilder sb) {
+ sb.append(e.toString());
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java b/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java
new file mode 100644
index 0000000..7d0e86f
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.io.IOException;
+
+import org.apache.drill.common.expression.FieldReference.De;
+import org.apache.drill.common.expression.FieldReference.Se;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+
+@JsonSerialize(using = Se.class)
+@JsonDeserialize(using = De.class)
+public class FieldReference extends SchemaPath {
+ MajorType overrideType;
+
+ public FieldReference(SchemaPath sp) {
+ super(sp);
+ checkData();
+ }
+
+ private void checkData() {
+ if (getRootSegment().getChild() != null) {
+ throw new UnsupportedOperationException("Field references must be singular names.");
+ }
+
+ }
+
+
+ private void checkSimpleString(CharSequence value) {
+ if (value.toString().contains(".")) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Unhandled field reference \"%s\"; a field reference identifier"
+ + " must not have the form of a qualified name (i.e., with \".\").",
+ value));
+ }
+ }
+
+ public FieldReference(CharSequence value) {
+ this(value, ExpressionPosition.UNKNOWN);
+ checkSimpleString(value);
+ }
+
+ public static FieldReference getWithQuotedRef(CharSequence safeString) {
+ return new FieldReference(safeString, ExpressionPosition.UNKNOWN, false);
+ }
+
+
+ public FieldReference(CharSequence value, ExpressionPosition pos) {
+ this(value, pos, true);
+ }
+
+ public FieldReference(CharSequence value, ExpressionPosition pos, boolean check) {
+ super(new NameSegment(value), pos);
+ if (check) {
+ checkData();
+ checkSimpleString(value);
+ }
+
+ }
+
+ public FieldReference(String value, ExpressionPosition pos, MajorType dataType) {
+ this(value, pos);
+ this.overrideType = dataType;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ if (overrideType == null) {
+ return super.getMajorType();
+ } else {
+ return overrideType;
+ }
+ }
+
+ public static class De extends StdDeserializer<FieldReference> {
+
+ public De() {
+ super(FieldReference.class);
+ }
+
+ @Override
+ public FieldReference deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
+ JsonProcessingException {
+ String ref = this._parseString(jp, ctxt);
+ ref = ref.replace("`", "");
+ return new FieldReference(ref, ExpressionPosition.UNKNOWN, false);
+ }
+
+ }
+
+ public static class Se extends StdSerializer<FieldReference> {
+
+ public Se() {
+ super(FieldReference.class);
+ }
+
+ @Override
+ public void serialize(FieldReference value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
+ JsonGenerationException {
+ jgen.writeString('`' + value.getRootSegment().getNameSegment().getPath() + '`');
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/FunctionCall.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/FunctionCall.java b/logical/src/main/java/org/apache/drill/common/expression/FunctionCall.java
new file mode 100644
index 0000000..938ab0a
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/FunctionCall.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+public class FunctionCall extends LogicalExpressionBase implements Iterable<LogicalExpression> {
+ private final String name;
+ public final ImmutableList<LogicalExpression> args;
+ private final ExpressionPosition pos;
+
+ public FunctionCall(String name, List<LogicalExpression> args, ExpressionPosition pos) {
+ super(pos);
+ this.name = name;
+
+ if (args == null) {
+ args = Lists.newArrayList();
+ }
+
+ if (!(args instanceof ImmutableList)) {
+ args = ImmutableList.copyOf(args);
+ }
+ this.args = (ImmutableList<LogicalExpression>) args;
+ this.pos = pos;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public ExpressionPosition getPosition() {
+ return pos;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E{
+ return visitor.visitFunctionCall(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return args.iterator();
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ throw new UnsupportedOperationException(
+ "No return type as FunctionCall is not a materialized expression");
+ }
+
+ @Override
+ public String toString() {
+ final int maxLen = 10;
+ return "FunctionCall [func=" + name + ", args="
+ + (args != null ? args.subList(0, Math.min(args.size(), maxLen)) : null) + ", pos=" + pos + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java b/logical/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
new file mode 100644
index 0000000..9128a78
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import com.google.common.collect.Lists;
+
+public class FunctionCallFactory {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionCallFactory.class);
+
+ private static Map<String, String> opToFuncTable = new HashMap<>();
+
+ static {
+ opToFuncTable.put("+", "add");
+ opToFuncTable.put("-", "subtract");
+ opToFuncTable.put("/", "divide");
+ opToFuncTable.put("*", "multiply");
+ opToFuncTable.put("%", "modulo");
+ opToFuncTable.put("^", "xor");
+ opToFuncTable.put("||", "concatOperator");
+ opToFuncTable.put("or", "booleanOr");
+ opToFuncTable.put("and", "booleanAnd");
+ opToFuncTable.put(">", "greater_than");
+ opToFuncTable.put("<", "less_than");
+ opToFuncTable.put("==", "equal");
+ opToFuncTable.put("=", "equal");
+ opToFuncTable.put("!=", "not_equal");
+ opToFuncTable.put("<>", "not_equal");
+ opToFuncTable.put(">=", "greater_than_or_equal_to");
+ opToFuncTable.put("<=", "less_than_or_equal_to");
+ opToFuncTable.put("is null", "isnull");
+ opToFuncTable.put("is not null", "isnotnull");
+ opToFuncTable.put("is true", "istrue");
+ opToFuncTable.put("is not true", "isnottrue");
+ opToFuncTable.put("is false", "isfalse");
+ opToFuncTable.put("is not false", "isnotfalse");
+ opToFuncTable.put("similar to", "similar_to");
+
+ opToFuncTable.put("!", "not");
+ opToFuncTable.put("u-", "negative");
+ }
+
+ private static String replaceOpWithFuncName(String op) {
+ return (opToFuncTable.containsKey(op)) ? (opToFuncTable.get(op)) : op;
+ }
+
+ public static boolean isBooleanOperator(String funcName) {
+ String opName = replaceOpWithFuncName(funcName);
+ return opName.equals("booleanAnd") || opName.equals("booleanOr");
+ }
+
+ /*
+ * create a cast function.
+ * arguments : type -- targetType
+ * ep -- input expression position
+ * expr -- input expression
+ */
+ public static LogicalExpression createCast(MajorType type, ExpressionPosition ep, LogicalExpression expr){
+ return new CastExpression(expr, type, ep);
+ }
+
+ public static LogicalExpression createConvert(String function, String conversionType, LogicalExpression expr, ExpressionPosition ep) {
+ return new ConvertExpression(function, conversionType, expr, ep);
+ }
+
+ public static LogicalExpression createExpression(String functionName, List<LogicalExpression> args){
+ return createExpression(functionName, ExpressionPosition.UNKNOWN, args);
+ }
+
+ public static LogicalExpression createExpression(String functionName, ExpressionPosition ep, List<LogicalExpression> args){
+ String name = replaceOpWithFuncName(functionName);
+ if (isBooleanOperator(name)) {
+ return new BooleanOperator(name, args, ep);
+ } else {
+ return new FunctionCall(name, args, ep);
+ }
+ }
+
+ public static LogicalExpression createExpression(String functionName, ExpressionPosition ep, LogicalExpression... e){
+ return createExpression(functionName, ep, Lists.newArrayList(e));
+ }
+
+ public static LogicalExpression createBooleanOperator(String functionName, List<LogicalExpression> args){
+ return createBooleanOperator(functionName, ExpressionPosition.UNKNOWN, args);
+ }
+
+ public static LogicalExpression createBooleanOperator(String functionName, ExpressionPosition ep, List<LogicalExpression> args){
+ return new BooleanOperator(replaceOpWithFuncName(functionName), args, ep);
+ }
+
+ public static LogicalExpression createByOp(List<LogicalExpression> args, ExpressionPosition ep, List<String> opTypes) {
+ if (args.size() == 1) {
+ return args.get(0);
+ }
+
+ if (args.size() - 1 != opTypes.size()) {
+ throw new DrillRuntimeException("Must receive one more expression then the provided number of operators.");
+ }
+
+ LogicalExpression first = args.get(0);
+ for (int i = 0; i < opTypes.size(); i++) {
+ List<LogicalExpression> l2 = new ArrayList<LogicalExpression>();
+ l2.add(first);
+ l2.add(args.get(i + 1));
+ first = createExpression(opTypes.get(i), ep, l2);
+ }
+ return first;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java b/logical/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
new file mode 100644
index 0000000..d5497f7
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.List;
+
+import org.apache.drill.common.expression.fn.FuncHolder;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+public abstract class FunctionHolderExpression extends LogicalExpressionBase {
+ public final ImmutableList<LogicalExpression> args;
+ public final String nameUsed;
+
+ public FunctionHolderExpression(String nameUsed, ExpressionPosition pos, List<LogicalExpression> args) {
+ super(pos);
+ if (args == null) {
+ args = Lists.newArrayList();
+ }
+
+ if (!(args instanceof ImmutableList)) {
+ args = ImmutableList.copyOf(args);
+ }
+ this.args = (ImmutableList<LogicalExpression>) args;
+ this.nameUsed = nameUsed;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitFunctionHolderExpression(this, value);
+ }
+
+ /**
+ * A function can have multiple names, it returns the function name used in the query
+ * @return
+ */
+ public String getName() {
+ return nameUsed;
+ }
+
+ /**
+ * constant input expected for i'th argument?
+ * @param i
+ * @return
+ */
+ public abstract boolean argConstantOnly(int i);
+
+ /**
+ * @return aggregating function or not
+ */
+ public abstract boolean isAggregating();
+
+ /**
+ * is the function output non-deterministic?
+ */
+ public abstract boolean isRandom();
+
+ /**
+ * @ return a copy of FunctionHolderExpression, with passed in argument list.
+ */
+ public abstract FunctionHolderExpression copy(List<LogicalExpression> args);
+
+ /** Return the underlying function implementation holder. */
+ public abstract FuncHolder getHolder();
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/FunctionName.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/FunctionName.java b/logical/src/main/java/org/apache/drill/common/expression/FunctionName.java
new file mode 100644
index 0000000..2164cb2
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/FunctionName.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+public @interface FunctionName {
+ String value();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/IfExpression.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/IfExpression.java b/logical/src/main/java/org/apache/drill/common/expression/IfExpression.java
new file mode 100644
index 0000000..e85caa0
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/IfExpression.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class IfExpression extends LogicalExpressionBase {
+ static final Logger logger = LoggerFactory.getLogger(IfExpression.class);
+
+ public final IfCondition ifCondition;
+ public final LogicalExpression elseExpression;
+ public final MajorType outputType;
+
+ private IfExpression(ExpressionPosition pos, IfCondition conditions, LogicalExpression elseExpression, MajorType outputType) {
+ super(pos);
+ this.ifCondition = conditions;
+ this.elseExpression = elseExpression;
+ this.outputType = outputType;
+ }
+
+ public static class IfCondition{
+ public final LogicalExpression condition;
+ public final LogicalExpression expression;
+
+ public IfCondition(LogicalExpression condition, LogicalExpression expression) {
+ //logger.debug("Generating IfCondition {}, {}", condition, expression);
+
+ this.condition = condition;
+ this.expression = expression;
+ }
+
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitIfExpression(this, value);
+ }
+
+ public static class Builder {
+ IfCondition conditions;
+ private LogicalExpression elseExpression;
+ private ExpressionPosition pos = ExpressionPosition.UNKNOWN;
+ private MajorType outputType;
+
+ public Builder setPosition(ExpressionPosition pos) {
+ this.pos = pos;
+ return this;
+ }
+
+ public Builder setElse(LogicalExpression elseExpression) {
+ this.elseExpression = elseExpression;
+ return this;
+ }
+
+ public Builder setIfCondition(IfCondition conditions) {
+ this.conditions = conditions;
+ return this;
+ }
+
+ public Builder setOutputType(MajorType outputType) {
+ this.outputType = outputType;
+ return this;
+ }
+
+ public IfExpression build(){
+ Preconditions.checkNotNull(pos);
+ Preconditions.checkNotNull(conditions);
+ return new IfExpression(pos, conditions, elseExpression, outputType);
+ }
+
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ if (outputType != null) {
+ return outputType;
+ }
+
+ MajorType majorType = elseExpression.getMajorType();
+ if (majorType.getMinorType() == MinorType.UNION) {
+ Set<MinorType> subtypes = Sets.newHashSet();
+ for (MinorType subtype : majorType.getSubTypeList()) {
+ subtypes.add(subtype);
+ }
+ for (MinorType subtype : ifCondition.expression.getMajorType().getSubTypeList()) {
+ subtypes.add(subtype);
+ }
+ MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL);
+ for (MinorType subtype : subtypes) {
+ builder.addSubType(subtype);
+ }
+ return builder.build();
+ }
+ if (majorType.getMode() == DataMode.OPTIONAL) {
+ return majorType;
+ }
+
+ if (ifCondition.expression.getMajorType().getMode() == DataMode.OPTIONAL) {
+ assert ifCondition.expression.getMajorType().getMinorType() == majorType.getMinorType();
+
+ return ifCondition.expression.getMajorType();
+ }
+
+ return majorType;
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ List<LogicalExpression> children = Lists.newLinkedList();
+
+ children.add(ifCondition.condition);
+ children.add(ifCondition.expression);
+ children.add(this.elseExpression);
+ return children.iterator();
+ }
+
+ @Override
+ public int getCumulativeCost() {
+ // return the average cost of operands for a boolean "and" | "or"
+ int cost = this.getSelfCost();
+
+ int i = 0;
+ for (LogicalExpression e : this) {
+ cost += e.getCumulativeCost();
+ i++;
+ }
+
+ return (int) (cost / i) ;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java b/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
new file mode 100644
index 0000000..a74ae29
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.io.IOException;
+
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.parser.ExprLexer;
+import org.apache.drill.common.expression.parser.ExprParser;
+import org.apache.drill.common.expression.parser.ExprParser.parse_return;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+
+//@JsonDeserialize(using = LogicalExpression.De.class) // Excluded as we need to register this with the DrillConfig.
+@JsonSerialize(using = LogicalExpression.Se.class)
+public interface LogicalExpression extends Iterable<LogicalExpression>{
+ static final Logger logger = LoggerFactory.getLogger(LogicalExpression.class);
+
+ public abstract MajorType getMajorType();
+
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E;
+
+ public ExpressionPosition getPosition();
+
+ public int getSelfCost();
+ public int getCumulativeCost();
+
+ public static class De extends StdDeserializer<LogicalExpression> {
+ DrillConfig config;
+
+ public De(DrillConfig config) {
+ super(LogicalExpression.class);
+ this.config = config;
+ }
+
+ @Override
+ public LogicalExpression deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
+ JsonProcessingException {
+ String expr = jp.getText();
+
+ if (expr == null || expr.isEmpty()) {
+ return null;
+ }
+ try {
+ // logger.debug("Parsing expression string '{}'", expr);
+ ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+ ExprParser parser = new ExprParser(tokens);
+
+ //TODO: move functionregistry and error collector to injectables.
+ //ctxt.findInjectableValue(valueId, forProperty, beanInstance)
+ parse_return ret = parser.parse();
+
+ // ret.e.resolveAndValidate(expr, errorCollector);
+ return ret.e;
+ } catch (RecognitionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ public static class Se extends StdSerializer<LogicalExpression> {
+
+ protected Se() {
+ super(LogicalExpression.class);
+ }
+
+ @Override
+ public void serialize(LogicalExpression value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
+ JsonGenerationException {
+ StringBuilder sb = new StringBuilder();
+ ExpressionStringBuilder esb = new ExpressionStringBuilder();
+ value.accept(esb, sb);
+ jgen.writeString(sb.toString());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java b/logical/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java
new file mode 100644
index 0000000..7dfe4a2
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+
+@JsonPropertyOrder({ "type" })
+public abstract class LogicalExpressionBase implements LogicalExpression {
+
+ private final ExpressionPosition pos;
+
+ protected LogicalExpressionBase(ExpressionPosition pos) {
+ super();
+ this.pos = pos;
+ }
+
+ @Override
+ public ExpressionPosition getPosition() {
+ return pos;
+ }
+
+ protected void i(StringBuilder sb, int indent) {
+ for (int i = 0; i < indent; i++){
+ sb.append(" ");
+ }
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ throw new UnsupportedOperationException(String.format("The type of %s doesn't currently support LogicalExpression.getDataType().", this.getClass().getCanonicalName()));
+ }
+
+ @JsonProperty("type")
+ public String getDescription() {
+ return this.getClass().getSimpleName();
+ }
+
+ @JsonIgnore
+ public int getSelfCost() {
+ return 0;
+ }
+
+ @JsonIgnore
+ public int getCumulativeCost() {
+ int cost = this.getSelfCost();
+
+ for (LogicalExpression e : this) {
+ cost += e.getCumulativeCost();
+ }
+
+ return cost;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/NullExpression.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/NullExpression.java b/logical/src/main/java/org/apache/drill/common/expression/NullExpression.java
new file mode 100644
index 0000000..e00399f
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/NullExpression.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+
+import com.google.common.collect.Iterators;
+
+public class NullExpression implements LogicalExpression{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NullExpression.class);
+
+ public static final NullExpression INSTANCE = new NullExpression();
+
+ final MajorType t = Types.optional(MinorType.NULL);
+
+ @Override
+ public MajorType getMajorType() {
+ return t;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitNullExpression(this, value);
+ }
+
+ @Override
+ public ExpressionPosition getPosition() {
+ return ExpressionPosition.UNKNOWN;
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ public int getSelfCost() { return 0 ; }
+
+ public int getCumulativeCost() { return 0; }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java b/logical/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
new file mode 100644
index 0000000..a03524c
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.List;
+
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+
+public interface OutputTypeDeterminer {
+
+ public static OutputTypeDeterminer FIXED_BIT = new FixedType(Types.required(MinorType.BIT));
+ public static OutputTypeDeterminer FIXED_INT = new FixedType(Types.required(MinorType.INT));
+ public static OutputTypeDeterminer FIXED_BIGINT = new FixedType(Types.required(MinorType.BIGINT));
+ public static OutputTypeDeterminer FIXED_FLOAT4 = new FixedType(Types.required(MinorType.FLOAT4));
+ public static OutputTypeDeterminer FIXED_FLOAT8 = new FixedType(Types.required(MinorType.FLOAT8));
+ public static OutputTypeDeterminer FIXED_VARCHAR = new FixedType(Types.required(MinorType.VARCHAR));
+ public static OutputTypeDeterminer FIXED_VARBINARY = new FixedType(Types.required(MinorType.VARBINARY));
+ public static OutputTypeDeterminer FIXED_VAR16CHAR = new FixedType(Types.required(MinorType.VAR16CHAR));
+
+ public MajorType getOutputType(List<LogicalExpression> expressions);
+
+
+ public static class FixedType implements OutputTypeDeterminer{
+ public MajorType outputType;
+
+
+ public FixedType(MajorType outputType) {
+ super();
+ this.outputType = outputType;
+ }
+
+
+ @Override
+ public MajorType getOutputType(List<LogicalExpression> expressions) {
+ return outputType;
+ }
+
+ }
+
+ public static class SameAsFirstInput implements OutputTypeDeterminer{
+ @Override
+ public MajorType getOutputType(List<LogicalExpression> expressions) {
+ return expressions.get(0).getMajorType();
+ }
+ }
+
+ public static class SameAsAnySoft implements OutputTypeDeterminer{
+ @Override
+ public MajorType getOutputType(List<LogicalExpression> expressions) {
+ for(LogicalExpression e : expressions){
+ if(e.getMajorType().getMode() == DataMode.OPTIONAL){
+ return e.getMajorType();
+ }
+ }
+ return expressions.get(0).getMajorType();
+ }
+ }
+
+ public static class NullIfNullType implements OutputTypeDeterminer{
+ public MinorType outputMinorType;
+
+
+ public NullIfNullType(MinorType outputType) {
+ super();
+ this.outputMinorType = outputType;
+ }
+
+ @Override
+ public MajorType getOutputType(List<LogicalExpression> expressions) {
+ for(LogicalExpression e : expressions){
+ if(e.getMajorType().getMode() == DataMode.OPTIONAL){
+ return Types.optional(outputMinorType);
+ }
+ }
+ return Types.required(outputMinorType);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java b/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
new file mode 100644
index 0000000..744a07f
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+public abstract class PathSegment{
+
+ PathSegment child;
+
+ int hash;
+
+ public abstract PathSegment cloneWithNewChild(PathSegment segment);
+ @Override
+ public abstract PathSegment clone();
+
+ public static final class ArraySegment extends PathSegment {
+ private final int index;
+
+ public ArraySegment(String numberAsText, PathSegment child) {
+ this(Integer.parseInt(numberAsText), child);
+ }
+
+ public ArraySegment(int index, PathSegment child) {
+ this.child = child;
+ this.index = index;
+ assert index >=0;
+ }
+
+ public ArraySegment(PathSegment child) {
+ this.child = child;
+ this.index = -1;
+ }
+
+ public boolean hasIndex() {
+ return index != -1;
+ }
+
+ public ArraySegment(int index) {
+ if (index < 0 ) {
+ throw new IllegalArgumentException();
+ }
+ this.index = index;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ @Override
+ public boolean isArray() {
+ return true;
+ }
+
+ @Override
+ public boolean isNamed() {
+ return false;
+ }
+
+ @Override
+ public ArraySegment getArraySegment() {
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "ArraySegment [index=" + index + ", getChild()=" + getChild() + "]";
+ }
+
+ @Override
+ public int segmentHashCode() {
+ return index;
+ }
+
+ @Override
+ public boolean segmentEquals(PathSegment obj) {
+ if (this == obj) {
+ return true;
+ } else if (obj == null) {
+ return false;
+ } else if (obj instanceof ArraySegment) {
+ return index == ((ArraySegment)obj).getIndex();
+ }
+ return false;
+ }
+
+ @Override
+ public PathSegment clone() {
+ PathSegment seg = index < 0 ? new ArraySegment(null) : new ArraySegment(index);
+ if (child != null) {
+ seg.setChild(child.clone());
+ }
+ return seg;
+ }
+
+ @Override
+ public ArraySegment cloneWithNewChild(PathSegment newChild) {
+ ArraySegment seg = index < 0 ? new ArraySegment(null) : new ArraySegment(index);
+ if (child != null) {
+ seg.setChild(child.cloneWithNewChild(newChild));
+ } else {
+ seg.setChild(newChild);
+ }
+ return seg;
+ }
+ }
+
+
+ public static final class NameSegment extends PathSegment {
+ private final String path;
+
+ public NameSegment(CharSequence n, PathSegment child) {
+ this.child = child;
+ this.path = n.toString();
+ }
+
+ public NameSegment(CharSequence n) {
+ this.path = n.toString();
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ @Override
+ public boolean isArray() {
+ return false;
+ }
+
+ @Override
+ public boolean isNamed() {
+ return true;
+ }
+
+ @Override
+ public NameSegment getNameSegment() {
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "NameSegment [path=" + path + ", getChild()=" + getChild() + "]";
+ }
+
+ @Override
+ public int segmentHashCode() {
+ return ((path == null) ? 0 : path.toLowerCase().hashCode());
+ }
+
+ @Override
+ public boolean segmentEquals(PathSegment obj) {
+ if (this == obj) {
+ return true;
+ } else if (obj == null) {
+ return false;
+ } else if (getClass() != obj.getClass()) {
+ return false;
+ }
+
+ NameSegment other = (NameSegment) obj;
+ if (path == null) {
+ return other.path == null;
+ }
+ return path.equalsIgnoreCase(other.path);
+ }
+
+ @Override
+ public NameSegment clone() {
+ NameSegment s = new NameSegment(this.path);
+ if (child != null) {
+ s.setChild(child.clone());
+ }
+ return s;
+ }
+
+ @Override
+ public NameSegment cloneWithNewChild(PathSegment newChild) {
+ NameSegment s = new NameSegment(this.path);
+ if (child != null) {
+ s.setChild(child.cloneWithNewChild(newChild));
+ } else {
+ s.setChild(newChild);
+ }
+ return s;
+ }
+
+ }
+
+ public NameSegment getNameSegment() {
+ throw new UnsupportedOperationException();
+ }
+
+ public ArraySegment getArraySegment() {
+ throw new UnsupportedOperationException();
+ }
+
+ public abstract boolean isArray();
+ public abstract boolean isNamed();
+
+ public boolean isLastPath() {
+ return child == null;
+ }
+
+ public PathSegment getChild() {
+ return child;
+ }
+
+ void setChild(PathSegment child) {
+ this.child = child;
+ }
+
+ protected abstract int segmentHashCode();
+ protected abstract boolean segmentEquals(PathSegment other);
+
+ @Override
+ public int hashCode() {
+ int h = hash;
+ if (h == 0) {
+ h = segmentHashCode();
+ h = 31*h + ((child == null) ? 0 : child.hashCode());
+ hash = h;
+ }
+ return h;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+
+ PathSegment other = (PathSegment) obj;
+ if (!segmentEquals(other)) {
+ return false;
+ } else if (child == null) {
+ return (other.child == null);
+ } else {
+ return child.equals(other.child);
+ }
+ }
+
+ /**
+ * Check if another path is contained in this one. This is useful for 2 cases. The first
+ * is checking if the other is lower down in the tree, below this path. The other is if
+ * a path is actually contained above the current one.
+ *
+ * Examples:
+ * [a] . contains( [a.b.c] ) returns true
+ * [a.b.c] . contains( [a] ) returns true
+ *
+ * This behavior is used for cases like scanning json in an event based fashion, when we arrive at
+ * a node in a complex type, we will know the complete path back to the root. This method can
+ * be used to determine if we need the data below. This is true in both the cases where the
+ * column requested from the user is below the current node (in which case we may ignore other nodes
+ * further down the tree, while keeping others). This is also the case if the requested path is further
+ * up the tree, if we know we are at position a.b.c and a.b was a requested column, we need to scan
+ * all of the data at and below the current a.b.c node.
+ *
+ * @param otherSeg - path segment to check if it is contained below this one.
+ * @return - is this a match
+ */
+ public boolean contains(PathSegment otherSeg) {
+ if (this == otherSeg) {
+ return true;
+ }
+ if (otherSeg == null) {
+ return false;
+ }
+ // TODO - fix this in the future to match array segments are part of the path
+ // the current behavior to always return true when we hit an array may be useful in some cases,
+ // but we can get better performance in the JSON reader if we avoid reading unwanted elements in arrays
+ if (otherSeg.isArray() || this.isArray()) {
+ return true;
+ }
+ if (getClass() != otherSeg.getClass()) {
+ return false;
+ }
+
+ if (!segmentEquals(otherSeg)) {
+ return false;
+ }
+ else if (child == null || otherSeg.child == null) {
+ return true;
+ } else {
+ return child.contains(otherSeg.child);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
new file mode 100644
index 0000000..f5fc687
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.PathSegment.ArraySegment;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
+import org.apache.drill.common.expression.parser.ExprLexer;
+import org.apache.drill.common.expression.parser.ExprParser;
+import org.apache.drill.common.expression.parser.ExprParser.parse_return;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.proto.UserBitShared.NamePart;
+import org.apache.drill.exec.proto.UserBitShared.NamePart.Type;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+
+public class SchemaPath extends LogicalExpressionBase {
+
+ private final NameSegment rootSegment;
+
+ public static SchemaPath getSimplePath(String name) {
+ return getCompoundPath(name);
+ }
+
+ public static SchemaPath getCompoundPath(String... strings) {
+ NameSegment s = null;
+ // loop through strings in reverse order
+ for (int i = strings.length - 1; i >= 0; i--) {
+ s = new NameSegment(strings[i], s);
+ }
+ return new SchemaPath(s);
+ }
+
+ @SuppressWarnings("unused")
+ public PathSegment getLastSegment() {
+ PathSegment s= rootSegment;
+ while (s.getChild() != null) {
+ s = s.getChild();
+ }
+ return s;
+ }
+
+ @Deprecated
+ public SchemaPath(String simpleName, ExpressionPosition pos) {
+ super(pos);
+ this.rootSegment = new NameSegment(simpleName);
+ if (simpleName.contains(".")) {
+ throw new IllegalStateException("This is deprecated and only supports simpe paths.");
+ }
+ }
+
+
+ public NamePart getAsNamePart() {
+ return getNamePart(rootSegment);
+ }
+
+ private static NamePart getNamePart(PathSegment s) {
+ if (s == null) {
+ return null;
+ }
+ NamePart.Builder b = NamePart.newBuilder();
+ if (s.getChild() != null) {
+ b.setChild(getNamePart(s.getChild()));
+ }
+
+ if (s.isArray()) {
+ if (s.getArraySegment().hasIndex()) {
+ throw new IllegalStateException("You cannot convert a indexed schema path to a NamePart. NameParts can only reference Vectors, not individual records or values.");
+ }
+ b.setType(Type.ARRAY);
+ } else {
+ b.setType(Type.NAME);
+ b.setName(s.getNameSegment().getPath());
+ }
+ return b.build();
+ }
+
+ private static PathSegment getPathSegment(NamePart n) {
+ PathSegment child = n.hasChild() ? getPathSegment(n.getChild()) : null;
+ if (n.getType() == Type.ARRAY) {
+ return new ArraySegment(child);
+ } else {
+ return new NameSegment(n.getName(), child);
+ }
+ }
+
+ public static SchemaPath create(NamePart namePart) {
+ Preconditions.checkArgument(namePart.getType() == NamePart.Type.NAME);
+ return new SchemaPath((NameSegment) getPathSegment(namePart));
+ }
+
+ /**
+ * A simple is a path where there are no repeated elements outside the lowest level of the path.
+ * @return Whether this path is a simple path.
+ */
+ public boolean isSimplePath() {
+ PathSegment seg = rootSegment;
+ while (seg != null) {
+ if (seg.isArray() && !seg.isLastPath()) {
+ return false;
+ }
+ seg = seg.getChild();
+ }
+ return true;
+ }
+
+
+ public SchemaPath(SchemaPath path) {
+ super(path.getPosition());
+ this.rootSegment = path.rootSegment;
+ }
+
+ public SchemaPath(NameSegment rootSegment) {
+ super(ExpressionPosition.UNKNOWN);
+ this.rootSegment = rootSegment;
+ }
+
+ public SchemaPath(NameSegment rootSegment, ExpressionPosition pos) {
+ super(pos);
+ this.rootSegment = rootSegment;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitSchemaPath(this, value);
+ }
+
+ public SchemaPath getChild(String childPath) {
+ NameSegment newRoot = rootSegment.cloneWithNewChild(new NameSegment(childPath));
+ return new SchemaPath(newRoot);
+ }
+
+ @SuppressWarnings("unused")
+ public SchemaPath getUnindexedArrayChild() {
+ NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(null));
+ return new SchemaPath(newRoot);
+ }
+
+ public SchemaPath getChild(int index) {
+ NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(index));
+ return new SchemaPath(newRoot);
+ }
+
+ public NameSegment getRootSegment() {
+ return rootSegment;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return Types.LATE_BIND_TYPE;
+ }
+
+ @Override
+ public int hashCode() {
+ return ((rootSegment == null) ? 0 : rootSegment.hashCode());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof SchemaPath)) {
+ return false;
+ }
+
+ SchemaPath other = (SchemaPath) obj;
+ if (rootSegment == null) {
+ return (other.rootSegment == null);
+ }
+ return rootSegment.equals(other.rootSegment);
+ }
+
+ public boolean contains(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof SchemaPath)) {
+ return false;
+ }
+
+ SchemaPath other = (SchemaPath) obj;
+ return rootSegment == null || rootSegment.contains(other.rootSegment);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ @Override
+ public String toString() {
+ return ExpressionStringBuilder.toString(this);
+ }
+
+ public String toExpr() {
+ return ExpressionStringBuilder.toString(this);
+ }
+
+ public String getAsUnescapedPath() {
+ StringBuilder sb = new StringBuilder();
+ PathSegment seg = getRootSegment();
+ if (seg.isArray()) {
+ throw new IllegalStateException("Drill doesn't currently support top level arrays");
+ }
+ sb.append(seg.getNameSegment().getPath());
+
+ while ( (seg = seg.getChild()) != null) {
+ if (seg.isNamed()) {
+ sb.append('.');
+ sb.append(seg.getNameSegment().getPath());
+ } else {
+ sb.append('[');
+ sb.append(seg.getArraySegment().getIndex());
+ sb.append(']');
+ }
+ }
+ return sb.toString();
+ }
+
+ public static class De extends StdDeserializer<SchemaPath> {
+
+ public De() {
+ super(LogicalExpression.class);
+ }
+
+ @Override
+ public SchemaPath deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
+ String expr = jp.getText();
+
+ if (expr == null || expr.isEmpty()) {
+ return null;
+ }
+ try {
+ // logger.debug("Parsing expression string '{}'", expr);
+ ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+ ExprParser parser = new ExprParser(tokens);
+
+ //TODO: move functionregistry and error collector to injectables.
+ //ctxt.findInjectableValue(valueId, forProperty, beanInstance)
+ parse_return ret = parser.parse();
+
+ // ret.e.resolveAndValidate(expr, errorCollector);
+ if (ret.e instanceof SchemaPath) {
+ return (SchemaPath) ret.e;
+ } else {
+ throw new IllegalStateException("Schema path is not a valid format.");
+ }
+ } catch (RecognitionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java b/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java
new file mode 100644
index 0000000..bd129d6
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import com.google.common.collect.Iterators;
+
+public class TypedNullConstant extends LogicalExpressionBase {
+
+ private final MajorType type;
+
+ public TypedNullConstant(MajorType type) {
+ super(null);
+ this.type = TypeProtos.MajorType.newBuilder().mergeFrom(type).setMode(DataMode.OPTIONAL).build();
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return this.type;
+ }
+
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitNullConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/ValidationError.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ValidationError.java b/logical/src/main/java/org/apache/drill/common/expression/ValidationError.java
new file mode 100644
index 0000000..195a6d8
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/ValidationError.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+public class ValidationError {
+
+}
[43/45] drill git commit: DRILL-4065: Maintain a cache of created
UserGroupInformation objects so that we don't accidentally generate thousands
of FileSystem objects & threads.
Posted by ja...@apache.org.
DRILL-4065: Maintain a cache of created UserGroupInformation objects so that we don't accidentally generate thousands of FileSystem objects & threads.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bdd2386e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bdd2386e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bdd2386e
Branch: refs/heads/master
Commit: bdd2386e284b21727b3498624a01fd1fbf429dea
Parents: 494492f
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Nov 10 16:04:09 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 18:33:37 2015 -0800
----------------------------------------------------------------------
.../drill/exec/util/ImpersonationUtil.java | 80 ++++++++++++++++++--
1 file changed, 72 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/bdd2386e/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
index 37fdc40..38d2797 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
@@ -17,18 +17,24 @@
*/
package org.apache.drill.exec.util;
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Set;
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Sets;
/**
* Utilities for impersonation purpose.
@@ -36,8 +42,66 @@ import java.util.Set;
public class ImpersonationUtil {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ImpersonationUtil.class);
+ private static final LoadingCache<Key, UserGroupInformation> CACHE = CacheBuilder.newBuilder()
+ .maximumSize(100)
+ .expireAfterAccess(60, TimeUnit.MINUTES)
+ .build(new CacheLoader<Key, UserGroupInformation>() {
+ @Override
+ public UserGroupInformation load(Key key) throws Exception {
+ return UserGroupInformation.createProxyUser(key.proxyUserName, key.loginUser);
+ }
+ });
+
private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+ private static class Key {
+ final String proxyUserName;
+ final UserGroupInformation loginUser;
+
+ public Key(String proxyUserName, UserGroupInformation loginUser) {
+ super();
+ this.proxyUserName = proxyUserName;
+ this.loginUser = loginUser;
+ }
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((loginUser == null) ? 0 : loginUser.hashCode());
+ result = prime * result + ((proxyUserName == null) ? 0 : proxyUserName.hashCode());
+ return result;
+ }
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ Key other = (Key) obj;
+ if (loginUser == null) {
+ if (other.loginUser != null) {
+ return false;
+ }
+ } else if (!loginUser.equals(other.loginUser)) {
+ return false;
+ }
+ if (proxyUserName == null) {
+ if (other.proxyUserName != null) {
+ return false;
+ }
+ } else if (!proxyUserName.equals(other.proxyUserName)) {
+ return false;
+ }
+ return true;
+ }
+
+
+ }
/**
* Create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation} of operator owner if operator
* owner is valid. Otherwise create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation} for
@@ -83,8 +147,8 @@ public class ImpersonationUtil {
return getProcessUserUGI();
}
- return UserGroupInformation.createProxyUser(proxyUserName, UserGroupInformation.getLoginUser());
- } catch(IOException e) {
+ return CACHE.get(new Key(proxyUserName, UserGroupInformation.getLoginUser()));
+ } catch (IOException | ExecutionException e) {
final String errMsg = "Failed to create proxy user UserGroupInformation object: " + e.getMessage();
logger.error(errMsg, e);
throw new DrillRuntimeException(errMsg, e);
[31/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
----------------------------------------------------------------------
diff --git a/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
new file mode 100644
index 0000000..7cf1477
--- /dev/null
+++ b/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.parser;
+
+import java.io.IOException;
+
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.apache.drill.common.expression.ExpressionStringBuilder;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.parser.ExprParser.parse_return;
+import org.apache.drill.test.DrillTest;
+import org.junit.Test;
+
+public class TreeTest extends DrillTest {
+
+ @Test
+ public void testIfWithCase() throws Exception{
+ testExpressionParsing("if ($F1) then case when (_MAP.R_NAME = 'AFRICA') then 2 else 4 end else if(4==3) then 1 else if(x==3) then 7 else (if(2==1) then 6 else 4 end) end");
+ }
+
+ @Test
+ public void testAdd() throws Exception{
+ testExpressionParsing("2+2");
+ }
+
+ @Test
+ public void testIf() throws Exception{
+ testExpressionParsing("if ('blue.red') then 'orange' else if (false) then 1 else 0 end");
+ }
+
+ @Test
+ public void testQuotedIdentifier() throws Exception{
+ testExpressionParsing("`hello friend`.`goodbye`");
+ }
+
+ @Test
+ public void testSpecialQuoted() throws Exception{
+ testExpressionParsing("`*0` + `*` ");
+ }
+
+ @Test
+ public void testQuotedIdentifier2() throws Exception{
+ testExpressionParsing("`hello friend`.goodbye");
+ }
+
+ @Test
+ public void testComplexIdentifier() throws Exception{
+ testExpressionParsing("goodbye[4].`hello`");
+ }
+
+ @Test // DRILL-2606
+ public void testCastToBooleanExpr() throws Exception{
+ testExpressionParsing("cast( (cast( (`bool_col` ) as VARCHAR(100) ) ) as BIT )");
+ }
+
+ private LogicalExpression parseExpression(String expr) throws RecognitionException, IOException{
+
+ ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+// tokens.fill();
+// for(Token t : (List<Token>) tokens.getTokens()){
+// System.out.println(t + "" + t.getType());
+// }
+// tokens.rewind();
+
+ ExprParser parser = new ExprParser(tokens);
+ parse_return ret = parser.parse();
+
+ return ret.e;
+
+ }
+
+ private String serializeExpression(LogicalExpression expr){
+
+ ExpressionStringBuilder b = new ExpressionStringBuilder();
+ StringBuilder sb = new StringBuilder();
+ expr.accept(b, sb);
+ return sb.toString();
+ }
+
+ /**
+ * Attempt to parse an expression. Once parsed, convert it to a string and then parse it again to make sure serialization works.
+ * @param expr
+ * @throws RecognitionException
+ * @throws IOException
+ */
+ private void testExpressionParsing(String expr) throws RecognitionException, IOException{
+ System.out.println("-----" + expr + "-----");
+ LogicalExpression e = parseExpression(expr);
+
+ String newStringExpr = serializeExpression(e);
+ System.out.println(newStringExpr);
+ LogicalExpression e2 = parseExpression(newStringExpr);
+ //Assert.assertEquals(e, e2);
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/test/java/org/apache/drill/common/logical/data/OrderTest.java
----------------------------------------------------------------------
diff --git a/logical/src/test/java/org/apache/drill/common/logical/data/OrderTest.java b/logical/src/test/java/org/apache/drill/common/logical/data/OrderTest.java
new file mode 100644
index 0000000..80b4e3c
--- /dev/null
+++ b/logical/src/test/java/org/apache/drill/common/logical/data/OrderTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import static org.junit.Assert.*;
+
+import java.sql.SQLException;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.Order.Ordering;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelFieldCollation.Direction;
+import org.apache.calcite.rel.RelFieldCollation.NullDirection;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.*;
+
+public class OrderTest {
+
+ //////////
+ // Order.Ordering tests:
+
+ // "Round trip" tests that strings from output work as input:
+
+ @Test
+ public void test_Ordering_roundTripAscAndNullsFirst() {
+ Ordering src = new Ordering( Direction.ASCENDING, null, NullDirection.FIRST);
+ Ordering reconstituted =
+ new Ordering( (LogicalExpression) null, src.getOrder(), src.getNullDirection().toString() );
+ assertThat( reconstituted.getDirection(), equalTo( RelFieldCollation.Direction.ASCENDING ) );
+ assertThat( reconstituted.getNullDirection(), equalTo( NullDirection.FIRST ) );
+ }
+
+ @Test
+ public void test_Ordering_roundTripDescAndNullsLast() {
+ Ordering src = new Ordering( Direction.DESCENDING, null, NullDirection.LAST);
+ Ordering reconstituted =
+ new Ordering( (LogicalExpression) null, src.getOrder(), src.getNullDirection().toString() );
+ assertThat( reconstituted.getDirection(), equalTo( RelFieldCollation.Direction.DESCENDING ) );
+ assertThat( reconstituted.getNullDirection(), equalTo( NullDirection.LAST ) );
+ }
+
+ @Test
+ public void test_Ordering_roundTripDescAndNullsUnspecified() {
+ Ordering src = new Ordering( Direction.DESCENDING, null, NullDirection.UNSPECIFIED);
+ Ordering reconstituted =
+ new Ordering( (LogicalExpression) null, src.getOrder(), src.getNullDirection().toString() );
+ assertThat( reconstituted.getDirection(), equalTo( RelFieldCollation.Direction.DESCENDING ) );
+ assertThat( reconstituted.getNullDirection(), equalTo( NullDirection.UNSPECIFIED ) );
+ }
+
+
+ // Basic input validation:
+
+ @Test( expected = DrillRuntimeException.class ) // (Currently.)
+ public void test_Ordering_garbageOrderRejected() {
+ new Ordering( (LogicalExpression) null, "AS CE ND IN G", (String) null );
+ }
+
+ @Test( expected = DrillRuntimeException.class ) // (Currently.)
+ public void test_Ordering_garbageNullOrderingRejected() {
+ new Ordering( (LogicalExpression) null, (String) null, "HIGH" );
+ }
+
+
+ // Defaults-value/null-strings test:
+
+ @Test
+ public void testOrdering_nullStrings() {
+ Ordering ordering = new Ordering( (LogicalExpression) null, null, null );
+ assertThat( ordering.getDirection(), equalTo( RelFieldCollation.Direction.ASCENDING ) );
+ assertThat( ordering.getNullDirection(), equalTo( RelFieldCollation.NullDirection.UNSPECIFIED ) );
+ assertThat( ordering.getOrder(), equalTo( "ASC" ) );
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
----------------------------------------------------------------------
diff --git a/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java b/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
new file mode 100644
index 0000000..090b8f4
--- /dev/null
+++ b/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.storage;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Collection;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.config.LogicalPlanPersistence;
+import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.common.scanner.ClassPathScanner;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.test.DrillTest;
+import org.junit.Test;
+
+public class CheckStorageConfig extends DrillTest {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CheckStorageConfig.class);
+
+ @Test
+ public void ensureStorageEnginePickup() {
+ DrillConfig config = DrillConfig.create();
+ ScanResult scan = ClassPathScanner.fromPrescan(config);
+ Collection<?> engines = scan.getImplementations(StoragePluginConfig.class);
+ assertEquals(engines.size(), 1);
+ }
+
+ @Test
+ public void checkPlanParsing() throws Exception{
+ DrillConfig config = DrillConfig.create();
+ ScanResult scan = ClassPathScanner.fromPrescan(config);
+ LogicalPlan plan = LogicalPlan.parse(new LogicalPlanPersistence(config, scan), FileUtils.getResourceAsString("/storage_engine_plan.json"));
+ }
+}
[03/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
new file mode 100644
index 0000000..42384cb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.DrillBuf;
+
+import java.math.BigDecimal;
+
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.Decimal18Holder;
+import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal9Holder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.IntervalDayHolder;
+import org.apache.drill.exec.expr.holders.IntervalYearHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.TimeHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.util.DecimalUtility;
+
+import com.google.common.base.Charsets;
+
+
+public class ValueHolderHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
+
+ public static IntHolder getIntHolder(int value) {
+ IntHolder holder = new IntHolder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static BigIntHolder getBigIntHolder(long value) {
+ BigIntHolder holder = new BigIntHolder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static Float4Holder getFloat4Holder(float value) {
+ Float4Holder holder = new Float4Holder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static Float8Holder getFloat8Holder(double value) {
+ Float8Holder holder = new Float8Holder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static DateHolder getDateHolder(long value) {
+ DateHolder holder = new DateHolder();
+ holder.value = value;
+ return holder;
+ }
+
+ public static TimeHolder getTimeHolder(int value) {
+ TimeHolder holder = new TimeHolder();
+ holder.value = value;
+ return holder;
+ }
+
+ public static TimeStampHolder getTimeStampHolder(long value) {
+ TimeStampHolder holder = new TimeStampHolder();
+ holder.value = value;
+ return holder;
+ }
+
+ public static BitHolder getBitHolder(int value) {
+ BitHolder holder = new BitHolder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static NullableBitHolder getNullableBitHolder(boolean isNull, int value) {
+ NullableBitHolder holder = new NullableBitHolder();
+ holder.isSet = isNull? 0 : 1;
+ if (! isNull) {
+ holder.value = value;
+ }
+
+ return holder;
+ }
+
+ public static VarCharHolder getVarCharHolder(DrillBuf buf, String s){
+ VarCharHolder vch = new VarCharHolder();
+
+ byte[] b = s.getBytes(Charsets.UTF_8);
+ vch.start = 0;
+ vch.end = b.length;
+ vch.buffer = buf.reallocIfNeeded(b.length);
+ vch.buffer.setBytes(0, b);
+ return vch;
+ }
+
+ public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
+ VarCharHolder vch = new VarCharHolder();
+
+ byte[] b = s.getBytes(Charsets.UTF_8);
+ vch.start = 0;
+ vch.end = b.length;
+ vch.buffer = a.buffer(b.length); //
+ vch.buffer.setBytes(0, b);
+ return vch;
+ }
+
+
+ public static IntervalYearHolder getIntervalYearHolder(int intervalYear) {
+ IntervalYearHolder holder = new IntervalYearHolder();
+
+ holder.value = intervalYear;
+ return holder;
+ }
+
+ public static IntervalDayHolder getIntervalDayHolder(int days, int millis) {
+ IntervalDayHolder dch = new IntervalDayHolder();
+
+ dch.days = days;
+ dch.milliseconds = millis;
+ return dch;
+ }
+
+ public static Decimal9Holder getDecimal9Holder(int decimal, int scale, int precision) {
+ Decimal9Holder dch = new Decimal9Holder();
+
+ dch.scale = scale;
+ dch.precision = precision;
+ dch.value = decimal;
+
+ return dch;
+ }
+
+ public static Decimal18Holder getDecimal18Holder(long decimal, int scale, int precision) {
+ Decimal18Holder dch = new Decimal18Holder();
+
+ dch.scale = scale;
+ dch.precision = precision;
+ dch.value = decimal;
+
+ return dch;
+ }
+
+ public static Decimal28SparseHolder getDecimal28Holder(DrillBuf buf, String decimal) {
+
+ Decimal28SparseHolder dch = new Decimal28SparseHolder();
+
+ BigDecimal bigDecimal = new BigDecimal(decimal);
+
+ dch.scale = bigDecimal.scale();
+ dch.precision = bigDecimal.precision();
+ Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
+ dch.start = 0;
+ dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.integerSize);
+ DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+
+ return dch;
+ }
+
+ public static Decimal38SparseHolder getDecimal38Holder(DrillBuf buf, String decimal) {
+
+ Decimal38SparseHolder dch = new Decimal38SparseHolder();
+
+ BigDecimal bigDecimal = new BigDecimal(decimal);
+
+ dch.scale = bigDecimal.scale();
+ dch.precision = bigDecimal.precision();
+ Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
+ dch.start = 0;
+ dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.integerSize);
+ DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+
+ return dch;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
new file mode 100644
index 0000000..8627d1a
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import java.io.Closeable;
+
+import io.netty.buffer.DrillBuf;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+/**
+ * An abstraction that is used to store a sequence of values in an individual column.
+ *
+ * A {@link ValueVector value vector} stores underlying data in-memory in a columnar fashion that is compact and
+ * efficient. The column whose data is stored, is referred by {@link #getField()}.
+ *
+ * A vector when instantiated, relies on a {@link org.apache.drill.exec.record.DeadBuf dead buffer}. It is important
+ * that vector is allocated before attempting to read or write.
+ *
+ * There are a few "rules" around vectors:
+ *
+ * <ul>
+ * <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
+ * <li>null vectors start with all values as null before writing anything</li>
+ * <li>for variable width types, the offset vector should be all zeros before writing</li>
+ * <li>you must call setValueCount before a vector can be read</li>
+ * <li>you should never write to a vector once it has been read.</li>
+ * </ul>
+ *
+ * Please note that the current implementation doesn't enfore those rules, hence we may find few places that
+ * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)
+ *
+ * This interface "should" strive to guarantee this order of operation:
+ * <blockquote>
+ * allocate > mutate > setvaluecount > access > clear (or allocate to start the process over).
+ * </blockquote>
+ */
+public interface ValueVector extends Closeable, Iterable<ValueVector> {
+ /**
+ * Allocate new buffers. ValueVector implements logic to determine how much to allocate.
+ * @throws OutOfMemoryRuntimeException Thrown if no memory can be allocated.
+ */
+ void allocateNew() throws OutOfMemoryRuntimeException;
+
+ /**
+ * Allocates new buffers. ValueVector implements logic to determine how much to allocate.
+ * @return Returns true if allocation was succesful.
+ */
+ boolean allocateNewSafe();
+
+ /**
+ * Set the initial record capacity
+ * @param numRecords
+ */
+ void setInitialCapacity(int numRecords);
+
+ /**
+ * Returns the maximum number of values that can be stored in this vector instance.
+ */
+ int getValueCapacity();
+
+ /**
+ * Alternative to clear(). Allows use as an AutoCloseable in try-with-resources.
+ */
+ @Override
+ void close();
+
+ /**
+ * Release the underlying DrillBuf and reset the ValueVector to empty.
+ */
+ void clear();
+
+ /**
+ * Get information about how this field is materialized.
+ */
+ MaterializedField getField();
+
+ /**
+ * Returns a {@link org.apache.drill.exec.record.TransferPair transfer pair}, creating a new target vector of
+ * the same type.
+ */
+ TransferPair getTransferPair();
+
+ TransferPair getTransferPair(FieldReference ref);
+
+ /**
+ * Returns a new {@link org.apache.drill.exec.record.TransferPair transfer pair} that is used to transfer underlying
+ * buffers into the target vector.
+ */
+ TransferPair makeTransferPair(ValueVector target);
+
+ /**
+ * Returns an {@link org.apache.drill.exec.vector.ValueVector.Accessor accessor} that is used to read from this vector
+ * instance.
+ */
+ Accessor getAccessor();
+
+ /**
+ * Returns an {@link org.apache.drill.exec.vector.ValueVector.Mutator mutator} that is used to write to this vector
+ * instance.
+ */
+ Mutator getMutator();
+
+ /**
+ * Returns a {@link org.apache.drill.exec.vector.complex.reader.FieldReader field reader} that supports reading values
+ * from this vector.
+ */
+ FieldReader getReader();
+
+ /**
+ * Get the metadata for this field. Used in serialization
+ *
+ * @return FieldMetadata for this field.
+ */
+ SerializedField getMetadata();
+
+ /**
+ * Returns the number of bytes that is used by this vector instance.
+ */
+ int getBufferSize();
+
+ /**
+ * Returns the number of bytes that is used by this vector if it holds the given number
+ * of values. The result will be the same as if Mutator.setValueCount() were called, followed
+ * by calling getBufferSize(), but without any of the closing side-effects that setValueCount()
+ * implies wrt finishing off the population of a vector. Some operations might wish to use
+ * this to determine how much memory has been used by a vector so far, even though it is
+ * not finished being populated.
+ *
+ * @param valueCount the number of values to assume this vector contains
+ * @return the buffer size if this vector is holding valueCount values
+ */
+ int getBufferSizeFor(int valueCount);
+
+ /**
+ * Return the underlying buffers associated with this vector. Note that this doesn't impact the reference counts for
+ * this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus
+ * external classes shouldn't hold a reference to it (unless they change it).
+ * @param clear Whether to clear vector before returning; the buffers will still be refcounted;
+ * but the returned array will be the only reference to them
+ *
+ * @return The underlying {@link io.netty.buffer.DrillBuf buffers} that is used by this vector instance.
+ */
+ DrillBuf[] getBuffers(boolean clear);
+
+ /**
+ * Load the data provided in the buffer. Typically used when deserializing from the wire.
+ *
+ * @param metadata
+ * Metadata used to decode the incoming buffer.
+ * @param buffer
+ * The buffer that contains the ValueVector.
+ */
+ void load(SerializedField metadata, DrillBuf buffer);
+
+ /**
+ * An abstraction that is used to read from this vector instance.
+ */
+ interface Accessor {
+ /**
+ * Get the Java Object representation of the element at the specified position. Useful for testing.
+ *
+ * @param index
+ * Index of the value to get
+ */
+ Object getObject(int index);
+
+ /**
+ * Returns the number of values that is stored in this vector.
+ */
+ int getValueCount();
+
+ /**
+ * Returns true if the value at the given index is null, false otherwise.
+ */
+ boolean isNull(int index);
+ }
+
+ /**
+ * An abstractiong that is used to write into this vector instance.
+ */
+ interface Mutator {
+ /**
+ * Sets the number of values that is stored in this vector to the given value count.
+ *
+ * @param valueCount value count to set.
+ */
+ void setValueCount(int valueCount);
+
+ /**
+ * Resets the mutator to pristine state.
+ */
+ void reset();
+
+ /**
+ * @deprecated this has nothing to do with value vector abstraction and should be removed.
+ */
+ @Deprecated
+ void generateTestData(int values);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
new file mode 100644
index 0000000..ee9c039
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.DrillBuf;
+
+public interface VariableWidthVector extends ValueVector{
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Desired size of the underlying data buffer.
+ * @param valueCount Number of values in the vector.
+ */
+ void allocateNew(int totalBytes, int valueCount);
+
+ /**
+ * Provide the maximum amount of variable width bytes that can be stored int his vector.
+ * @return
+ */
+ int getByteCapacity();
+
+ VariableWidthMutator getMutator();
+
+ VariableWidthAccessor getAccessor();
+
+ interface VariableWidthAccessor extends Accessor {
+ int getValueLength(int index);
+ }
+
+ int getCurrentSizeInBytes();
+
+ interface VariableWidthMutator extends Mutator {
+ void setValueLengthSafe(int index, int length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
new file mode 100644
index 0000000..2ecaef6
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import java.util.Collection;
+
+import com.google.common.base.Preconditions;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.record.MaterializedField;
+
+public class VectorDescriptor {
+ private static final String DEFAULT_NAME = "NONE";
+
+ private final MaterializedField field;
+
+ public VectorDescriptor(final TypeProtos.MajorType type) {
+ this(DEFAULT_NAME, type);
+ }
+
+ public VectorDescriptor(final String name, final TypeProtos.MajorType type) {
+ this(MaterializedField.create(name, type));
+ }
+
+ public VectorDescriptor(final MaterializedField field) {
+ this.field = Preconditions.checkNotNull(field, "field cannot be null");
+ }
+
+ public MaterializedField getField() {
+ return field;
+ }
+
+ public TypeProtos.MajorType getType() {
+ return field.getType();
+ }
+
+ public String getName() {
+ return field.getLastName();
+ }
+
+ public Collection<MaterializedField> getChildren() {
+ return field.getChildren();
+ }
+
+ public boolean hasName() {
+ return getName() != DEFAULT_NAME;
+ }
+
+ public VectorDescriptor withName(final String name) {
+ return new VectorDescriptor(field.withPath(new FieldReference(name)));
+ }
+
+ public VectorDescriptor withType(final TypeProtos.MajorType type) {
+ return new VectorDescriptor(field.withType(type));
+ }
+
+ public static VectorDescriptor create(final String name, final TypeProtos.MajorType type) {
+ return new VectorDescriptor(name, type);
+ }
+
+ public static VectorDescriptor create(final TypeProtos.MajorType type) {
+ return new VectorDescriptor(type);
+ }
+
+ public static VectorDescriptor create(final MaterializedField field) {
+ return new VectorDescriptor(field);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
new file mode 100644
index 0000000..7f977f2
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
+public class VectorTrimmer {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorTrimmer.class);
+
+ public static void trim(ByteBuf data, int idx) {
+ data.writerIndex(idx);
+ if (data instanceof DrillBuf) {
+ // data.capacity(idx);
+ data.writerIndex(idx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
new file mode 100644
index 0000000..9fb151f
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import java.util.Iterator;
+
+import com.google.common.collect.Iterators;
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.complex.impl.NullReader;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class ZeroVector implements ValueVector {
+ public final static ZeroVector INSTANCE = new ZeroVector();
+
+ private final MaterializedField field = MaterializedField.create("[DEFAULT]", Types.LATE_BIND_TYPE);
+
+ private final TransferPair defaultPair = new TransferPair() {
+ @Override
+ public void transfer() { }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) { }
+
+ @Override
+ public ValueVector getTo() {
+ return ZeroVector.this;
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) { }
+ };
+
+ private final Accessor defaultAccessor = new Accessor() {
+ @Override
+ public Object getObject(int index) {
+ return null;
+ }
+
+ @Override
+ public int getValueCount() {
+ return 0;
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return true;
+ }
+ };
+
+ private final Mutator defaultMutator = new Mutator() {
+ @Override
+ public void setValueCount(int valueCount) { }
+
+ @Override
+ public void reset() { }
+
+ @Override
+ public void generateTestData(int values) { }
+ };
+
+ public ZeroVector() { }
+
+ @Override
+ public void close() { }
+
+ @Override
+ public void clear() { }
+
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ return defaultPair;
+ }
+
+ @Override
+ public UserBitShared.SerializedField getMetadata() {
+ return getField()
+ .getAsBuilder()
+ .setBufferLength(getBufferSize())
+ .setValueCount(getAccessor().getValueCount())
+ .build();
+ }
+
+ @Override
+ public Iterator iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ @Override
+ public int getBufferSize() {
+ return 0;
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ return 0;
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ return new DrillBuf[0];
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryRuntimeException {
+ allocateNewSafe();
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ return true;
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) { }
+
+ @Override
+ public int getValueCapacity() {
+ return 0;
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ return defaultPair;
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector target) {
+ return defaultPair;
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return defaultAccessor;
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return defaultMutator;
+ }
+
+ @Override
+ public FieldReader getReader() {
+ return NullReader.INSTANCE;
+ }
+
+ @Override
+ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
new file mode 100644
index 0000000..5f6d0d2
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import javax.annotation.Nullable;
+import javax.validation.constraints.DecimalMin.List;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.exec.vector.ValueVector;
+
+/**
+ * Base class for composite vectors.
+ *
+ * This class implements common functionality of composite vectors.
+ */
+public abstract class AbstractContainerVector implements ValueVector {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
+
+ protected final MaterializedField field;
+ protected final BufferAllocator allocator;
+ protected final CallBack callBack;
+
+ protected AbstractContainerVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+ this.field = Preconditions.checkNotNull(field);
+ this.allocator = allocator;
+ this.callBack = callBack;
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryRuntimeException {
+ if (!allocateNewSafe()) {
+ throw new OutOfMemoryRuntimeException();
+ }
+ }
+
+ /**
+ * Returns the field definition of this instance.
+ */
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ /**
+ * Returns a {@link org.apache.drill.exec.vector.ValueVector} corresponding to the given field name if exists or null.
+ */
+ public ValueVector getChild(String name) {
+ return getChild(name, ValueVector.class);
+ }
+
+ /**
+ * Returns a sequence of field names in the order that they show up in the schema.
+ */
+ protected Collection<String> getChildFieldNames() {
+ return Sets.newLinkedHashSet(Iterables.transform(field.getChildren(), new Function<MaterializedField, String>() {
+ @Nullable
+ @Override
+ public String apply(MaterializedField field) {
+ return Preconditions.checkNotNull(field).getLastName();
+ }
+ }));
+ }
+
+ /**
+ * Clears out all underlying child vectors.
+ */
+ @Override
+ public void close() {
+ for (ValueVector vector:(Iterable<ValueVector>)this) {
+ vector.close();
+ }
+ }
+
+ protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
+ if (clazz.isAssignableFrom(v.getClass())) {
+ return (T) v;
+ }
+ throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
+ }
+
+ public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
+ return FieldIdUtil.getFieldIdIfMatches(this, builder, addToBreadCrumb, seg);
+ }
+
+ MajorType getLastPathType() {
+ if((this.getField().getType().getMinorType() == MinorType.LIST &&
+ this.getField().getType().getMode() == DataMode.REPEATED)) { // Use Repeated scalar type instead of Required List.
+ VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
+ ValueVector v = vord.vector;
+ if (! (v instanceof AbstractContainerVector)) {
+ return v.getField().getType();
+ }
+ } else if (this.getField().getType().getMinorType() == MinorType.MAP &&
+ this.getField().getType().getMode() == DataMode.REPEATED) { // Use Required Map
+ return this.getField().getType().toBuilder().setMode(DataMode.REQUIRED).build();
+ }
+
+ return this.getField().getType();
+ }
+
+ protected boolean supportsDirectRead() {
+ return false;
+ }
+
+ // return the number of child vectors
+ public abstract int size();
+
+ // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
+ public abstract <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz);
+
+ // return the child vector with the input name
+ public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
+
+ // return the child vector's ordinal in the composite container
+ public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
new file mode 100644
index 0000000..35df691
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
@@ -0,0 +1,276 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.common.collections.MapWithOrdinal;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.exec.vector.ValueVector;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+/*
+ * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
+ */
+public abstract class AbstractMapVector extends AbstractContainerVector {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
+
+ // Maintains a map with key as field name and value is the vector itself
+ private final MapWithOrdinal<String, ValueVector> vectors = new MapWithOrdinal<>();
+
+ protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+ super(field.clone(), allocator, callBack);
+ MaterializedField clonedField = field.clone();
+ // create the hierarchy of the child vectors based on the materialized field
+ for (MaterializedField child : clonedField.getChildren()) {
+ if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
+ final String fieldName = child.getLastName();
+ final ValueVector v = TypeHelper.getNewVector(child, allocator, callBack);
+ putVector(fieldName, v);
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ for(final ValueVector valueVector : vectors.values()) {
+ valueVector.close();
+ }
+ vectors.clear();
+
+ super.close();
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ for (final ValueVector v : vectors.values()) {
+ if (!v.allocateNewSafe()) {
+ return false;
+ }
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
+ * {@link org.apache.drill.exec.vector.ValueVector}.
+ *
+ * Execution takes place in the following order:
+ * <ul>
+ * <li>
+ * if field is new, create and insert a new vector of desired type.
+ * </li>
+ * <li>
+ * if field exists and existing vector is of desired vector type, return the vector.
+ * </li>
+ * <li>
+ * if field exists and null filled, clear the existing vector; create and insert a new vector of desired type.
+ * </li>
+ * <li>
+ * otherwise, throw an {@link java.lang.IllegalStateException}
+ * </li>
+ * </ul>
+ *
+ * @param name name of the field
+ * @param type type of the field
+ * @param clazz class of expected vector type
+ * @param <T> class type of expected vector type
+ * @throws java.lang.IllegalStateException raised if there is a hard schema change
+ *
+ * @return resultant {@link org.apache.drill.exec.vector.ValueVector}
+ */
+ @Override
+ public <T extends ValueVector> T addOrGet(String name, TypeProtos.MajorType type, Class<T> clazz) {
+ final ValueVector existing = getChild(name);
+ boolean create = false;
+ if (existing == null) {
+ create = true;
+ } else if (clazz.isAssignableFrom(existing.getClass())) {
+ return (T) existing;
+ } else if (nullFilled(existing)) {
+ existing.clear();
+ create = true;
+ }
+ if (create) {
+ final T vector = (T) TypeHelper.getNewVector(field.getPath(), name, allocator, type, callBack);
+ putChild(name, vector);
+ if (callBack!=null) {
+ callBack.doWork();
+ }
+ return vector;
+ }
+ final String message = "Drill does not support schema change yet. Existing[{}] and desired[{}] vector types mismatch";
+ throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(), clazz.getSimpleName()));
+ }
+
+ private boolean nullFilled(ValueVector vector) {
+ for (int r = 0; r < vector.getAccessor().getValueCount(); r++) {
+ if (!vector.getAccessor().isNull(r)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns a {@link org.apache.drill.exec.vector.ValueVector} corresponding to the given ordinal identifier.
+ */
+ public ValueVector getChildByOrdinal(int id) {
+ return vectors.getByOrdinal(id);
+ }
+
+ /**
+ * Returns a {@link org.apache.drill.exec.vector.ValueVector} instance of subtype of <T> corresponding to the given
+ * field name if exists or null.
+ */
+ @Override
+ public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+ final ValueVector v = vectors.get(name.toLowerCase());
+ if (v == null) {
+ return null;
+ }
+ return typeify(v, clazz);
+ }
+
+ /**
+ * Inserts the vector with the given name if it does not exist else replaces it with the new value.
+ *
+ * Note that this method does not enforce any vector type check nor throws a schema change exception.
+ */
+ protected void putChild(String name, ValueVector vector) {
+ putVector(name, vector);
+ field.addChild(vector.getField());
+ }
+
+ /**
+ * Inserts the input vector into the map if it does not exist, replaces if it exists already
+ * @param name field name
+ * @param vector vector to be inserted
+ */
+ protected void putVector(String name, ValueVector vector) {
+ final ValueVector old = vectors.put(
+ Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
+ Preconditions.checkNotNull(vector, "vector cannot be null")
+ );
+ if (old != null && old != vector) {
+ logger.debug("Field [{}] mutated from [{}] to [{}]", name, old.getClass().getSimpleName(),
+ vector.getClass().getSimpleName());
+ }
+ }
+
+ /**
+ * Returns a sequence of underlying child vectors.
+ */
+ protected Collection<ValueVector> getChildren() {
+ return vectors.values();
+ }
+
+ /**
+ * Returns the number of underlying child vectors.
+ */
+ @Override
+ public int size() {
+ return vectors.size();
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ return vectors.values().iterator();
+ }
+
+ /**
+ * Returns a list of scalar child vectors recursing the entire vector hierarchy.
+ */
+ public List<ValueVector> getPrimitiveVectors() {
+ final List<ValueVector> primitiveVectors = Lists.newArrayList();
+ for (final ValueVector v : vectors.values()) {
+ if (v instanceof AbstractMapVector) {
+ AbstractMapVector mapVector = (AbstractMapVector) v;
+ primitiveVectors.addAll(mapVector.getPrimitiveVectors());
+ } else {
+ primitiveVectors.add(v);
+ }
+ }
+ return primitiveVectors;
+ }
+
+ /**
+ * Returns a vector with its corresponding ordinal mapping if field exists or null.
+ */
+ @Override
+ public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
+ final int ordinal = vectors.getOrdinal(name.toLowerCase());
+ if (ordinal < 0) {
+ return null;
+ }
+ final ValueVector vector = vectors.getByOrdinal(ordinal);
+ return new VectorWithOrdinal(vector, ordinal);
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ final List<DrillBuf> buffers = Lists.newArrayList();
+
+ for (final ValueVector vector : vectors.values()) {
+ for (final DrillBuf buf : vector.getBuffers(false)) {
+ buffers.add(buf);
+ if (clear) {
+ buf.retain(1);
+ }
+ }
+ if (clear) {
+ vector.clear();
+ }
+ }
+
+ return buffers.toArray(new DrillBuf[buffers.size()]);
+ }
+
+ @Override
+ public int getBufferSize() {
+ int actualBufSize = 0 ;
+
+ for (final ValueVector v : vectors.values()) {
+ for (final DrillBuf buf : v.getBuffers(false)) {
+ actualBufSize += buf.writerIndex();
+ }
+ }
+ return actualBufSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
new file mode 100644
index 0000000..de4ad95
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.exception.SchemaChangeRuntimeException;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.AddOrGetResult;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+import org.apache.drill.exec.vector.ZeroVector;
+
+public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
+
+ public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
+ public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+ public final static String DATA_VECTOR_NAME = "$data$";
+
+ public final static MaterializedField OFFSETS_FIELD =
+ MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(TypeProtos.MinorType.UINT4));
+
+ protected final UInt4Vector offsets;
+ protected ValueVector vector;
+
+ protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
+ this(field, allocator, DEFAULT_DATA_VECTOR);
+ }
+
+ protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
+ super(field, allocator);
+ this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
+ this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if (!offsets.allocateNewSafe()) {
+ return false;
+ }
+ success = vector.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ offsets.zeroVector();
+ return success;
+ }
+
+
+ @Override
+ public UInt4Vector getOffsetVector() {
+ return offsets;
+ }
+
+ @Override
+ public ValueVector getDataVector() {
+ return vector;
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ offsets.setInitialCapacity(numRecords + 1);
+ vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+ }
+
+ @Override
+ public int getValueCapacity() {
+ final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
+ if (vector == DEFAULT_DATA_VECTOR) {
+ return offsetValueCapacity;
+ }
+ return Math.min(vector.getValueCapacity(), offsetValueCapacity);
+ }
+
+ @Override
+ protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+ return super.getMetadataBuilder()
+ .addChild(offsets.getMetadata())
+ .addChild(vector.getMetadata());
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (getAccessor().getValueCount() == 0) {
+ return 0;
+ }
+ return offsets.getBufferSize() + vector.getBufferSize();
+ }
+
+ @Override
+ public int getBufferSizeFor(int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount);
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ return Collections.singleton(getDataVector()).iterator();
+ }
+
+ @Override
+ public void clear() {
+ offsets.clear();
+ vector.clear();
+ super.clear();
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), DrillBuf.class);
+ if (clear) {
+ for (DrillBuf buffer:buffers) {
+ buffer.retain();
+ }
+ clear();
+ }
+ return buffers;
+ }
+
+ @Override
+ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+ final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+ offsets.load(offsetMetadata, buffer);
+
+ final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
+ if (getDataVector() == DEFAULT_DATA_VECTOR) {
+ addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+ }
+
+ final int offsetLength = offsetMetadata.getBufferLength();
+ final int vectorLength = vectorMetadata.getBufferLength();
+ vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
+ }
+
+ /**
+ * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
+ *
+ * @see {@link ContainerVectorLike#size}
+ */
+ @Override
+ public int size() {
+ return vector == DEFAULT_DATA_VECTOR ? 0:1;
+ }
+
+ @Override
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ boolean created = false;
+ if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) {
+ final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
+ vector = TypeHelper.getNewVector(field, allocator);
+ // returned vector must have the same field
+ assert field.equals(vector.getField());
+ getField().addChild(field);
+ created = true;
+ }
+
+ final TypeProtos.MajorType actual = vector.getField().getType();
+ if (!actual.equals(descriptor.getType())) {
+ final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
+ descriptor.getType(), actual);
+ throw new SchemaChangeRuntimeException(msg);
+ }
+
+ return new AddOrGetResult<>((T)vector, created);
+ }
+
+ protected void replaceDataVector(ValueVector v) {
+ vector.clear();
+ vector = v;
+ }
+
+ public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
+
+ @Override
+ public int getValueCount() {
+ return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
+ }
+
+ @Override
+ public int getInnerValueCount() {
+ return vector.getAccessor().getValueCount();
+ }
+
+ @Override
+ public int getInnerValueCountAt(int index) {
+ return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return false;
+ }
+
+ @Override
+ public boolean isEmpty(int index) {
+ return false;
+ }
+ }
+
+ public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
+
+ @Override
+ public void startNewValue(int index) {
+ while (offsets.getValueCapacity() <= index) {
+ offsets.reAlloc();
+ }
+ offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
+ setValueCount(index+1);
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ // TODO: populate offset end points
+ offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+ final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+ vector.getMutator().setValueCount(childValueCount);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
new file mode 100644
index 0000000..999b47d
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ContainerVectorLike.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import org.apache.drill.exec.vector.AddOrGetResult;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+
+/**
+ * A mix-in used for introducing container vector-like behaviour.
+ */
+public interface ContainerVectorLike {
+
+ /**
+ * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
+ *
+ * @param descriptor vector descriptor
+ * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
+ * @throws org.apache.drill.common.exceptions.DrillRuntimeException
+ * if schema change is not permissible between the given and existing data vector types.
+ */
+ <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
+
+ /**
+ * Returns the number of child vectors in this container vector-like instance.
+ */
+ int size();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/EmptyValuePopulator.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/EmptyValuePopulator.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/EmptyValuePopulator.java
new file mode 100644
index 0000000..fa1de66
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/EmptyValuePopulator.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import com.google.common.base.Preconditions;
+import org.apache.drill.exec.vector.UInt4Vector;
+
+/**
+ * A helper class that is used to track and populate empty values in repeated value vectors.
+ */
+public class EmptyValuePopulator {
+ private final UInt4Vector offsets;
+
+ public EmptyValuePopulator(UInt4Vector offsets) {
+ this.offsets = Preconditions.checkNotNull(offsets, "offsets cannot be null");
+ }
+
+ /**
+ * Marks all values since the last set as empty. The last set value is obtained from underlying offsets vector.
+ *
+ * @param lastIndex the last index (inclusive) in the offsets vector until which empty population takes place
+ * @throws java.lang.IndexOutOfBoundsException if lastIndex is negative or greater than offsets capacity.
+ */
+ public void populate(int lastIndex) {
+ if (lastIndex < 0) {
+ throw new IndexOutOfBoundsException("index cannot be negative");
+ }
+ final UInt4Vector.Accessor accessor = offsets.getAccessor();
+ final UInt4Vector.Mutator mutator = offsets.getMutator();
+ final int lastSet = Math.max(accessor.getValueCount() - 1, 0);
+ final int previousEnd = accessor.get(lastSet);//0 ? 0 : accessor.get(lastSet);
+ for (int i = lastSet; i < lastIndex; i++) {
+ mutator.setSafe(i + 1, previousEnd);
+ }
+ mutator.setValueCount(lastIndex+1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
new file mode 100644
index 0000000..468a7f3
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.vector.ValueVector;
+
+public class FieldIdUtil {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FieldIdUtil.class);
+ public static TypedFieldId getFieldIdIfMatches(ValueVector vector, TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
+ if (seg == null) {
+ if (addToBreadCrumb) {
+ builder.intermediateType(vector.getField().getType());
+ }
+ return builder.finalType(vector.getField().getType()).build();
+ }
+
+ if (seg.isArray()) {
+ if (seg.isLastPath()) {
+ MajorType type;
+ if (vector instanceof AbstractContainerVector) {
+ type = ((AbstractContainerVector) vector).getLastPathType();
+ } else if (vector instanceof ListVector) {
+ type = ((ListVector) vector).getDataVector().getField().getType();
+ builder.listVector();
+ } else {
+ throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
+ }
+ builder //
+ .withIndex() //
+ .finalType(type);
+
+ // remainder starts with the 1st array segment in SchemaPath.
+ // only set remainder when it's the only array segment.
+ if (addToBreadCrumb) {
+ addToBreadCrumb = false;
+ builder.remainder(seg);
+ }
+ return builder.build();
+ } else {
+ if (addToBreadCrumb) {
+ addToBreadCrumb = false;
+ builder.remainder(seg);
+ }
+ }
+ } else {
+ if (vector instanceof ListVector) {
+ return null;
+ }
+ }
+
+ ValueVector v;
+ if (vector instanceof AbstractContainerVector) {
+ VectorWithOrdinal vord = ((AbstractContainerVector) vector).getChildVectorWithOrdinal(seg.isArray() ? null : seg.getNameSegment().getPath());
+ if (vord == null) {
+ return null;
+ }
+ v = vord.vector;
+ if (addToBreadCrumb) {
+ builder.intermediateType(v.getField().getType());
+ builder.addId(vord.ordinal);
+ }
+ } else if (vector instanceof ListVector) {
+ v = ((ListVector) vector).getDataVector();
+ } else {
+ throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
+ }
+
+ if (v instanceof AbstractContainerVector) {
+ // we're looking for a multi path.
+ AbstractContainerVector c = (AbstractContainerVector) v;
+ return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
+ } else if(v instanceof ListVector) {
+ ListVector list = (ListVector) v;
+ return list.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
+ } else if (v instanceof UnionVector) {
+ return ((UnionVector) v).getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild());
+ } else {
+ if (seg.isNamed()) {
+ if(addToBreadCrumb) {
+ builder.intermediateType(v.getField().getType());
+ }
+ builder.finalType(v.getField().getType());
+ } else {
+ builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
+ }
+
+ if (seg.isLastPath()) {
+ return builder.build();
+ } else {
+ PathSegment child = seg.getChild();
+ if (child.isLastPath() && child.isArray()) {
+ if (addToBreadCrumb) {
+ builder.remainder(child);
+ }
+ builder.withIndex();
+ builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build());
+ return builder.build();
+ } else {
+ logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong.");
+ return null;
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
new file mode 100644
index 0000000..e5d9443
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
@@ -0,0 +1,327 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector.complex;
+
+import com.google.common.collect.ObjectArrays;
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.vector.AddOrGetResult;
+import org.apache.drill.exec.vector.UInt1Vector;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+import org.apache.drill.exec.vector.ZeroVector;
+import org.apache.drill.exec.vector.complex.impl.ComplexCopier;
+import org.apache.drill.exec.vector.complex.impl.UnionListReader;
+import org.apache.drill.exec.vector.complex.impl.UnionListWriter;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
+
+import java.util.List;
+
+public class ListVector extends BaseRepeatedValueVector {
+
+ private UInt4Vector offsets;
+ private final UInt1Vector bits;
+ private Mutator mutator = new Mutator();
+ private Accessor accessor = new Accessor();
+ private UnionListWriter writer;
+ private UnionListReader reader;
+ private CallBack callBack;
+
+ public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+ super(field, allocator);
+ this.bits = new UInt1Vector(MaterializedField.create("$bits$", Types.required(MinorType.UINT1)), allocator);
+ offsets = getOffsetVector();
+ this.field.addChild(getDataVector().getField());
+ this.writer = new UnionListWriter(this);
+ this.reader = new UnionListReader(this);
+ this.callBack = callBack;
+ }
+
+ public UnionListWriter getWriter() {
+ return writer;
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryRuntimeException {
+ super.allocateNewSafe();
+ }
+
+ public void transferTo(ListVector target) {
+ offsets.makeTransferPair(target.offsets).transfer();
+ bits.makeTransferPair(target.bits).transfer();
+ if (target.getDataVector() instanceof ZeroVector) {
+ target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ }
+ getDataVector().makeTransferPair(target.getDataVector()).transfer();
+ }
+
+ public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
+ copyFrom(inIndex, outIndex, from);
+ }
+
+ public void copyFrom(int inIndex, int outIndex, ListVector from) {
+ FieldReader in = from.getReader();
+ in.setPosition(inIndex);
+ FieldWriter out = getWriter();
+ out.setPosition(outIndex);
+ ComplexCopier.copy(in, out);
+ }
+
+ @Override
+ public ValueVector getDataVector() {
+ return vector;
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ return new TransferImpl(field.withPath(ref));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector target) {
+ return new TransferImpl((ListVector) target);
+ }
+
+ private class TransferImpl implements TransferPair {
+
+ ListVector to;
+
+ public TransferImpl(MaterializedField field) {
+ to = new ListVector(field, allocator, null);
+ to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ }
+
+ public TransferImpl(ListVector to) {
+ this.to = to;
+ to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ }
+
+ @Override
+ public void transfer() {
+ transferTo(to);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ to.allocateNew();
+ for (int i = 0; i < length; i++) {
+ copyValueSafe(startIndex + i, i);
+ }
+ }
+
+ @Override
+ public ValueVector getTo() {
+ return to;
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) {
+ this.to.copyFrom(from, to, ListVector.this);
+ }
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ @Override
+ public FieldReader getReader() {
+ return reader;
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if (!offsets.allocateNewSafe()) {
+ return false;
+ }
+ success = vector.allocateNewSafe();
+ success = success && bits.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ if (success) {
+ offsets.zeroVector();
+ bits.zeroVector();
+ }
+ return success;
+ }
+
+ @Override
+ protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+ return getField().getAsBuilder()
+ .setValueCount(getAccessor().getValueCount())
+ .setBufferLength(getBufferSize())
+ .addChild(offsets.getMetadata())
+ .addChild(bits.getMetadata())
+ .addChild(vector.getMetadata());
+ }
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+ reader = new UnionListReader(this);
+ return result;
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (getAccessor().getValueCount() == 0) {
+ return 0;
+ }
+ return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize();
+ }
+
+ @Override
+ public void clear() {
+ offsets.clear();
+ vector.clear();
+ bits.clear();
+ lastSet = 0;
+ super.clear();
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ final DrillBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false),
+ vector.getBuffers(false), DrillBuf.class), DrillBuf.class);
+ if (clear) {
+ for (DrillBuf buffer:buffers) {
+ buffer.retain();
+ }
+ clear();
+ }
+ return buffers;
+ }
+
+ @Override
+ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+ final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+ offsets.load(offsetMetadata, buffer);
+
+ final int offsetLength = offsetMetadata.getBufferLength();
+ final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
+ final int bitLength = bitMetadata.getBufferLength();
+ bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
+
+ final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
+ if (getDataVector() == DEFAULT_DATA_VECTOR) {
+ addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+ }
+
+ final int vectorLength = vectorMetadata.getBufferLength();
+ vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
+ }
+
+ public UnionVector promoteToUnion() {
+ MaterializedField newField = MaterializedField.create(getField().getPath(), Types.optional(MinorType.UNION));
+ UnionVector vector = new UnionVector(newField, allocator, null);
+ replaceDataVector(vector);
+ reader = new UnionListReader(this);
+ return vector;
+ }
+
+ public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
+ return FieldIdUtil.getFieldIdIfMatches(this, builder, addToBreadCrumb, seg);
+ }
+
+ private int lastSet;
+
+ public class Accessor extends BaseRepeatedAccessor {
+
+ @Override
+ public Object getObject(int index) {
+ if (isNull(index)) {
+ return null;
+ }
+ final List<Object> vals = new JsonStringArrayList<>();
+ final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
+ final int start = offsetsAccessor.get(index);
+ final int end = offsetsAccessor.get(index + 1);
+ final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor();
+ for(int i = start; i < end; i++) {
+ vals.add(valuesAccessor.getObject(i));
+ }
+ return vals;
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return bits.getAccessor().get(index) == 0;
+ }
+ }
+
+ public class Mutator extends BaseRepeatedMutator {
+ public void setNotNull(int index) {
+ bits.getMutator().setSafe(index, 1);
+ lastSet = index + 1;
+ }
+
+ @Override
+ public void startNewValue(int index) {
+ for (int i = lastSet; i <= index; i++) {
+ offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
+ }
+ setNotNull(index);
+ lastSet = index + 1;
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ // TODO: populate offset end points
+ if (valueCount == 0) {
+ offsets.getMutator().setValueCount(0);
+ } else {
+ for (int i = lastSet; i < valueCount; i++) {
+ offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
+ }
+ offsets.getMutator().setValueCount(valueCount + 1);
+ }
+ final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+ vector.getMutator().setValueCount(childValueCount);
+ bits.getMutator().setValueCount(valueCount);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
new file mode 100644
index 0000000..048358c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -0,0 +1,380 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
+
+import io.netty.buffer.DrillBuf;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.holders.ComplexHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.exec.util.JsonStringHashMap;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier;
+import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+import com.google.common.base.Preconditions;
+
+public class MapVector extends AbstractMapVector {
+ //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
+
+ public final static MajorType TYPE = Types.required(MinorType.MAP);
+
+ private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
+ private final Accessor accessor = new Accessor();
+ private final Mutator mutator = new Mutator();
+ private int valueCount;
+
+ public MapVector(String path, BufferAllocator allocator, CallBack callBack){
+ this(MaterializedField.create(SchemaPath.getSimplePath(path), TYPE), allocator, callBack);
+ }
+
+ public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+ super(field, allocator, callBack);
+ }
+
+ @Override
+ public FieldReader getReader() {
+ //return new SingleMapReaderImpl(MapVector.this);
+ return reader;
+ }
+
+ transient private MapTransferPair ephPair;
+ transient private MapSingleCopier ephPair2;
+
+ public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
+ if(ephPair == null || ephPair.from != from) {
+ ephPair = (MapTransferPair) from.makeTransferPair(this);
+ }
+ ephPair.copyValueSafe(fromIndex, thisIndex);
+ }
+
+ public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
+ if(ephPair2 == null || ephPair2.from != from) {
+ ephPair2 = from.makeSingularCopier(this);
+ }
+ ephPair2.copySafe(fromSubIndex, thisIndex);
+ }
+
+ @Override
+ protected boolean supportsDirectRead() {
+ return true;
+ }
+
+ public Iterator<String> fieldNameIterator() {
+ return getChildFieldNames().iterator();
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ v.setInitialCapacity(numRecords);
+ }
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (valueCount == 0 || size() == 0) {
+ return 0;
+ }
+ long buffer = 0;
+ for (final ValueVector v : (Iterable<ValueVector>)this) {
+ buffer += v.getBufferSize();
+ }
+
+ return (int) buffer;
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ long bufferSize = 0;
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ bufferSize += v.getBufferSizeFor(valueCount);
+ }
+
+ return (int) bufferSize;
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ int expectedSize = getBufferSize();
+ int actualSize = super.getBufferSize();
+
+ Preconditions.checkArgument(expectedSize == actualSize);
+ return super.getBuffers(clear);
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ return new MapTransferPair(this, getField().getPath());
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new MapTransferPair(this, (MapVector) to);
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ return new MapTransferPair(this, ref);
+ }
+
+ protected static class MapTransferPair implements TransferPair{
+ private final TransferPair[] pairs;
+ private final MapVector from;
+ private final MapVector to;
+
+ public MapTransferPair(MapVector from, SchemaPath path) {
+ this(from, new MapVector(MaterializedField.create(path, TYPE), from.allocator, from.callBack), false);
+ }
+
+ public MapTransferPair(MapVector from, MapVector to) {
+ this(from, to, true);
+ }
+
+ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
+ this.from = from;
+ this.to = to;
+ this.pairs = new TransferPair[from.size()];
+ this.to.ephPair = null;
+ this.to.ephPair2 = null;
+
+ int i = 0;
+ ValueVector vector;
+ for (String child:from.getChildFieldNames()) {
+ int preSize = to.size();
+ vector = from.getChild(child);
+ if (vector == null) {
+ continue;
+ }
+ //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
+ // the child fields may be nested fields of the top level child. For example if the structure
+ // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
+ // But the children member of a Materialized field is a HashSet. If the fields are added in the
+ // children HashSet, and the hashCode of the Materialized field includes the hash code of the
+ // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
+ // (This is similar to what happens in ScanBatch where the children cannot be added till they are
+ // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
+ // include the hashCode of the children but is based only on MaterializedField$key.
+ final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+ if (allocate && to.size() != preSize) {
+ newVector.allocateNew();
+ }
+ pairs[i++] = vector.makeTransferPair(newVector);
+ }
+ }
+
+ @Override
+ public void transfer() {
+ for (final TransferPair p : pairs) {
+ p.transfer();
+ }
+ to.valueCount = from.valueCount;
+ from.clear();
+ }
+
+ @Override
+ public ValueVector getTo() {
+ return to;
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) {
+ for (TransferPair p : pairs) {
+ p.copyValueSafe(from, to);
+ }
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ for (TransferPair p : pairs) {
+ p.splitAndTransfer(startIndex, length);
+ }
+ to.getMutator().setValueCount(length);
+ }
+ }
+
+ @Override
+ public int getValueCapacity() {
+ if (size() == 0) {
+ return 0;
+ }
+
+ final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
+ @Override
+ public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
+ return Ints.compare(
+ Preconditions.checkNotNull(left).getValueCapacity(),
+ Preconditions.checkNotNull(right).getValueCapacity()
+ );
+ }
+ };
+
+ return natural.min(getChildren()).getValueCapacity();
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public void load(SerializedField metadata, DrillBuf buf) {
+ final List<SerializedField> fields = metadata.getChildList();
+ valueCount = metadata.getValueCount();
+
+ int bufOffset = 0;
+ for (final SerializedField child : fields) {
+ final MaterializedField fieldDef = MaterializedField.create(child);
+
+ ValueVector vector = getChild(fieldDef.getLastName());
+ if (vector == null) {
+ // if we arrive here, we didn't have a matching vector.
+ vector = TypeHelper.getNewVector(fieldDef, allocator);
+ putChild(fieldDef.getLastName(), vector);
+ }
+ if (child.getValueCount() == 0) {
+ vector.clear();
+ } else {
+ vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
+ }
+ bufOffset += child.getBufferLength();
+ }
+
+ assert bufOffset == buf.capacity();
+ }
+
+ @Override
+ public SerializedField getMetadata() {
+ SerializedField.Builder b = getField() //
+ .getAsBuilder() //
+ .setBufferLength(getBufferSize()) //
+ .setValueCount(valueCount);
+
+
+ for(ValueVector v : getChildren()) {
+ b.addChild(v.getMetadata());
+ }
+ return b.build();
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ public class Accessor extends BaseValueVector.BaseAccessor {
+
+ @Override
+ public Object getObject(int index) {
+ Map<String, Object> vv = new JsonStringHashMap<>();
+ for (String child:getChildFieldNames()) {
+ ValueVector v = getChild(child);
+ // TODO(DRILL-4001): Resolve this hack:
+ // The index/value count check in the following if statement is a hack
+ // to work around the current fact that RecordBatchLoader.load and
+ // MapVector.load leave child vectors with a length of zero (as opposed
+ // to matching the lengths of siblings and the parent map vector)
+ // because they don't remove (or set the lengths of) vectors from
+ // previous batches that aren't in the current batch.
+ if (v != null && index < v.getAccessor().getValueCount()) {
+ Object value = v.getAccessor().getObject(index);
+ if (value != null) {
+ vv.put(child, value);
+ }
+ }
+ }
+ return vv;
+ }
+
+ public void get(int index, ComplexHolder holder) {
+ reader.setPosition(index);
+ holder.reader = reader;
+ }
+
+ @Override
+ public int getValueCount() {
+ return valueCount;
+ }
+ }
+
+ public ValueVector getVectorById(int id) {
+ return getChildByOrdinal(id);
+ }
+
+ public class Mutator extends BaseValueVector.BaseMutator {
+
+ @Override
+ public void setValueCount(int valueCount) {
+ for (final ValueVector v : getChildren()) {
+ v.getMutator().setValueCount(valueCount);
+ }
+ MapVector.this.valueCount = valueCount;
+ }
+
+ @Override
+ public void reset() { }
+
+ @Override
+ public void generateTestData(int values) { }
+ }
+
+ @Override
+ public void clear() {
+ for (final ValueVector v : getChildren()) {
+ v.clear();
+ }
+ valueCount = 0;
+ }
+
+ @Override
+ public void close() {
+ final Collection<ValueVector> vectors = getChildren();
+ for (final ValueVector v : vectors) {
+ v.close();
+ }
+ vectors.clear();
+ valueCount = 0;
+
+ super.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/Positionable.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/Positionable.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/Positionable.java
new file mode 100644
index 0000000..6d86a64
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/Positionable.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+public interface Positionable {
+ public void setPosition(int index);
+}
[36/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
deleted file mode 100644
index 7cf1477..0000000
--- a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.parser;
-
-import java.io.IOException;
-
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.RecognitionException;
-import org.apache.drill.common.expression.ExpressionStringBuilder;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.parser.ExprParser.parse_return;
-import org.apache.drill.test.DrillTest;
-import org.junit.Test;
-
-public class TreeTest extends DrillTest {
-
- @Test
- public void testIfWithCase() throws Exception{
- testExpressionParsing("if ($F1) then case when (_MAP.R_NAME = 'AFRICA') then 2 else 4 end else if(4==3) then 1 else if(x==3) then 7 else (if(2==1) then 6 else 4 end) end");
- }
-
- @Test
- public void testAdd() throws Exception{
- testExpressionParsing("2+2");
- }
-
- @Test
- public void testIf() throws Exception{
- testExpressionParsing("if ('blue.red') then 'orange' else if (false) then 1 else 0 end");
- }
-
- @Test
- public void testQuotedIdentifier() throws Exception{
- testExpressionParsing("`hello friend`.`goodbye`");
- }
-
- @Test
- public void testSpecialQuoted() throws Exception{
- testExpressionParsing("`*0` + `*` ");
- }
-
- @Test
- public void testQuotedIdentifier2() throws Exception{
- testExpressionParsing("`hello friend`.goodbye");
- }
-
- @Test
- public void testComplexIdentifier() throws Exception{
- testExpressionParsing("goodbye[4].`hello`");
- }
-
- @Test // DRILL-2606
- public void testCastToBooleanExpr() throws Exception{
- testExpressionParsing("cast( (cast( (`bool_col` ) as VARCHAR(100) ) ) as BIT )");
- }
-
- private LogicalExpression parseExpression(String expr) throws RecognitionException, IOException{
-
- ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
- CommonTokenStream tokens = new CommonTokenStream(lexer);
-
-// tokens.fill();
-// for(Token t : (List<Token>) tokens.getTokens()){
-// System.out.println(t + "" + t.getType());
-// }
-// tokens.rewind();
-
- ExprParser parser = new ExprParser(tokens);
- parse_return ret = parser.parse();
-
- return ret.e;
-
- }
-
- private String serializeExpression(LogicalExpression expr){
-
- ExpressionStringBuilder b = new ExpressionStringBuilder();
- StringBuilder sb = new StringBuilder();
- expr.accept(b, sb);
- return sb.toString();
- }
-
- /**
- * Attempt to parse an expression. Once parsed, convert it to a string and then parse it again to make sure serialization works.
- * @param expr
- * @throws RecognitionException
- * @throws IOException
- */
- private void testExpressionParsing(String expr) throws RecognitionException, IOException{
- System.out.println("-----" + expr + "-----");
- LogicalExpression e = parseExpression(expr);
-
- String newStringExpr = serializeExpression(e);
- System.out.println(newStringExpr);
- LogicalExpression e2 = parseExpression(newStringExpr);
- //Assert.assertEquals(e, e2);
-
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/test/java/org/apache/drill/common/logical/data/OrderTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/common/logical/data/OrderTest.java b/common/src/test/java/org/apache/drill/common/logical/data/OrderTest.java
deleted file mode 100644
index 80b4e3c..0000000
--- a/common/src/test/java/org/apache/drill/common/logical/data/OrderTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import static org.junit.Assert.*;
-
-import java.sql.SQLException;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.Order.Ordering;
-import org.apache.calcite.rel.RelFieldCollation;
-import org.apache.calcite.rel.RelFieldCollation.Direction;
-import org.apache.calcite.rel.RelFieldCollation.NullDirection;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.*;
-
-public class OrderTest {
-
- //////////
- // Order.Ordering tests:
-
- // "Round trip" tests that strings from output work as input:
-
- @Test
- public void test_Ordering_roundTripAscAndNullsFirst() {
- Ordering src = new Ordering( Direction.ASCENDING, null, NullDirection.FIRST);
- Ordering reconstituted =
- new Ordering( (LogicalExpression) null, src.getOrder(), src.getNullDirection().toString() );
- assertThat( reconstituted.getDirection(), equalTo( RelFieldCollation.Direction.ASCENDING ) );
- assertThat( reconstituted.getNullDirection(), equalTo( NullDirection.FIRST ) );
- }
-
- @Test
- public void test_Ordering_roundTripDescAndNullsLast() {
- Ordering src = new Ordering( Direction.DESCENDING, null, NullDirection.LAST);
- Ordering reconstituted =
- new Ordering( (LogicalExpression) null, src.getOrder(), src.getNullDirection().toString() );
- assertThat( reconstituted.getDirection(), equalTo( RelFieldCollation.Direction.DESCENDING ) );
- assertThat( reconstituted.getNullDirection(), equalTo( NullDirection.LAST ) );
- }
-
- @Test
- public void test_Ordering_roundTripDescAndNullsUnspecified() {
- Ordering src = new Ordering( Direction.DESCENDING, null, NullDirection.UNSPECIFIED);
- Ordering reconstituted =
- new Ordering( (LogicalExpression) null, src.getOrder(), src.getNullDirection().toString() );
- assertThat( reconstituted.getDirection(), equalTo( RelFieldCollation.Direction.DESCENDING ) );
- assertThat( reconstituted.getNullDirection(), equalTo( NullDirection.UNSPECIFIED ) );
- }
-
-
- // Basic input validation:
-
- @Test( expected = DrillRuntimeException.class ) // (Currently.)
- public void test_Ordering_garbageOrderRejected() {
- new Ordering( (LogicalExpression) null, "AS CE ND IN G", (String) null );
- }
-
- @Test( expected = DrillRuntimeException.class ) // (Currently.)
- public void test_Ordering_garbageNullOrderingRejected() {
- new Ordering( (LogicalExpression) null, (String) null, "HIGH" );
- }
-
-
- // Defaults-value/null-strings test:
-
- @Test
- public void testOrdering_nullStrings() {
- Ordering ordering = new Ordering( (LogicalExpression) null, null, null );
- assertThat( ordering.getDirection(), equalTo( RelFieldCollation.Direction.ASCENDING ) );
- assertThat( ordering.getNullDirection(), equalTo( RelFieldCollation.NullDirection.UNSPECIFIED ) );
- assertThat( ordering.getOrder(), equalTo( "ASC" ) );
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
deleted file mode 100644
index 090b8f4..0000000
--- a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.storage;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Collection;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.config.LogicalPlanPersistence;
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.common.scanner.ClassPathScanner;
-import org.apache.drill.common.scanner.persistence.ScanResult;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.test.DrillTest;
-import org.junit.Test;
-
-public class CheckStorageConfig extends DrillTest {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CheckStorageConfig.class);
-
- @Test
- public void ensureStorageEnginePickup() {
- DrillConfig config = DrillConfig.create();
- ScanResult scan = ClassPathScanner.fromPrescan(config);
- Collection<?> engines = scan.getImplementations(StoragePluginConfig.class);
- assertEquals(engines.size(), 1);
- }
-
- @Test
- public void checkPlanParsing() throws Exception{
- DrillConfig config = DrillConfig.create();
- ScanResult scan = ClassPathScanner.fromPrescan(config);
- LogicalPlan plan = LogicalPlan.parse(new LogicalPlanPersistence(config, scan), FileUtils.getResourceAsString("/storage_engine_plan.json"));
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 7299b3e..c5eb2b0 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -273,6 +273,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-logical</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-rpc</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/exec/rpc/src/main/java/org/apache/drill/common/SerializedExecutor.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/common/SerializedExecutor.java b/exec/rpc/src/main/java/org/apache/drill/common/SerializedExecutor.java
new file mode 100644
index 0000000..6a3a823
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/common/SerializedExecutor.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+import io.netty.util.Recycler;
+import io.netty.util.Recycler.Handle;
+
+import java.util.LinkedList;
+import java.util.concurrent.Executor;
+
+/**
+ * Serializes execution of multiple submissions to a single target, while still
+ * using a thread pool to execute those submissions. Provides an implicit
+ * queueing capability for a single target that requires any commands that
+ * execute against it to be serialized.
+ */
+public abstract class SerializedExecutor implements Executor {
+
+ private final Recycler<RunnableProcessor> processors = new Recycler<RunnableProcessor>() {
+ @Override
+ protected RunnableProcessor newObject(Handle handle) {
+ return new RunnableProcessor(handle);
+ }
+ };
+
+ private boolean isProcessing = false;
+ private final LinkedList<Runnable> queuedRunnables = new LinkedList<>();
+ private final Executor underlyingExecutor;
+ private final String name;
+
+ /**
+ * Constructor.
+ *
+ * @param underlyingExecutor
+ * underlying executor to use to execute commands submitted to this
+ * SerializedExecutor
+ */
+ public SerializedExecutor(String name, Executor underlyingExecutor) {
+ this.underlyingExecutor = underlyingExecutor;
+ this.name = name;
+ }
+
+ /**
+ * An exception occurred in the last command executed; this reports that to
+ * the subclass of SerializedExecutor.
+ *
+ * <p>
+ * The default implementation of this method throws an exception, which is
+ * considered an error (see below). Implementors have two alternatives:
+ * Arrange not to throw from your commands' run(), or if they do, provide an
+ * override of this method that handles any exception that is thrown.
+ * </p>
+ *
+ * <p>
+ * It is an error for this to throw an exception, and doing so will terminate
+ * the thread with an IllegalStateException. Derivers must handle any reported
+ * exceptions in other ways.
+ * </p>
+ *
+ * @param command
+ * the command that caused the exception
+ * @param t
+ * the exception
+ */
+ protected abstract void runException(Runnable command, Throwable t);
+
+ private class RunnableProcessor implements Runnable {
+ private final Handle handle;
+
+ private Runnable command;
+
+ public RunnableProcessor(Handle handle) {
+ this.handle = handle;
+ }
+
+ public Runnable set(Runnable command) {
+ this.command = command;
+ return this;
+ }
+
+ @Override
+ public void run() {
+ final Thread currentThread = Thread.currentThread();
+ final String originalThreadName = currentThread.getName();
+ currentThread.setName(name);
+
+ try {
+ while (true) {
+ try {
+ command.run();
+ } catch (Exception | AssertionError e) {
+ try {
+ runException(command, e);
+ } catch (Exception | AssertionError ee) {
+ throw new IllegalStateException("Exception handler threw an exception", ee);
+ }
+ }
+
+ synchronized (queuedRunnables) {
+ if (queuedRunnables.isEmpty()) {
+ isProcessing = false;
+ break;
+ }
+
+ command = queuedRunnables.removeFirst();
+ }
+ }
+ } finally {
+ currentThread.setName(originalThreadName);
+ command = null;
+ processors.recycle(this, handle);
+ }
+ }
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ synchronized (queuedRunnables) {
+ if (isProcessing) {
+ queuedRunnables.addLast(command);
+ return;
+ }
+
+ isProcessing = true;
+ }
+
+ underlyingExecutor.execute(processors.get().set(command));
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/exec/vector/pom.xml
----------------------------------------------------------------------
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 1639912..0109371 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -33,18 +33,30 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-logical</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.drill.memory</groupId>
<artifactId>drill-memory-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.drill</groupId>
- <artifactId>drill-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.9</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.4.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.4.3</version>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/pom.xml
----------------------------------------------------------------------
diff --git a/logical/pom.xml b/logical/pom.xml
new file mode 100644
index 0000000..f592657
--- /dev/null
+++ b/logical/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>drill-root</artifactId>
+ <groupId>org.apache.drill</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>drill-logical</artifactId>
+ <packaging>jar</packaging>
+ <name>Logical Plan, Base expressions</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-protocol</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <!-- add as provided scope so that we can compile TestTools. Should only be ever used in a test scenario where someone else is bringing JUnit in. -->
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${dep.junit.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.typesafe</groupId>
+ <artifactId>config</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.msgpack</groupId>
+ <artifactId>msgpack</artifactId>
+ <version>0.6.6</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.8</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.4.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.4.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-servlets</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ <version>3.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.9</version>
+ </dependency>
+
+ </dependencies>
+
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>java</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <mainClass>org.apache.drill.common.scanner.BuildTimeScan</mainClass>
+ <arguments>
+ <argument>${project.build.outputDirectory}</argument>
+ </arguments>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.15</version>
+ <configuration>
+ <useSystemClassLoader>false</useSystemClassLoader>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr3-maven-plugin</artifactId>
+ <version>3.4</version>
+ <configuration>
+
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>antlr</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
----------------------------------------------------------------------
diff --git a/logical/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g b/logical/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
new file mode 100644
index 0000000..a881b5c
--- /dev/null
+++ b/logical/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
@@ -0,0 +1,154 @@
+lexer grammar ExprLexer;
+
+options {
+ language=Java;
+}
+
+@header {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.common.expression.parser;
+}
+
+If : 'if';
+Else : 'else';
+Return : 'return';
+Then : 'then';
+End : 'end';
+In : 'in';
+Case : 'case';
+When : 'when';
+
+Cast: 'cast';
+Convert : 'convert_' ('from' | 'to');
+Nullable: 'nullable';
+Repeat: 'repeat';
+As: 'as';
+
+BIT : 'bit' | 'BIT';
+INT : 'int' | 'INT';
+BIGINT : 'bigint' | 'BIGINT';
+FLOAT4 : 'float4' | 'FLOAT4';
+FLOAT8 : 'float8' | 'FLOAT8';
+VARCHAR : 'varchar' | 'VARCHAR';
+VARBINARY: 'varbinary' | 'VARBINARY';
+DATE : 'date' | 'DATE';
+TIMESTAMP: 'timestamp' | 'TIMESTAMP';
+TIME : 'time' | 'TIME';
+TIMESTAMPTZ: 'timestamptz' | 'TIMESTAMPTZ';
+INTERVAL : 'interval' | 'INTERVAL';
+INTERVALYEAR : 'intervalyear' | 'INTERVALYEAR';
+INTERVALDAY : 'intervalday' | 'INTERVALDAY';
+Period : '.';
+DECIMAL9 : 'decimal9' | 'DECIMAL9';
+DECIMAL18 : 'decimal18' | 'DECIMAL18';
+DECIMAL28DENSE : 'decimal28dense' | 'DECIMAL28DENSE';
+DECIMAL28SPARSE : 'decimal28sparse' | 'DECIMAL28SPARSE';
+DECIMAL38DENSE : 'decimal38dense' | 'DECIMAL38DENSE';
+DECIMAL38SPARSE : 'decimal38sparse' | 'DECIMAL38SPARSE';
+Or : 'or' | 'OR' | 'Or';
+And : 'and' | 'AND' ;
+Equals : '==' | '=';
+NEquals : '<>' | '!=';
+GTEquals : '>=';
+LTEquals : '<=';
+Caret : '^';
+Excl : '!';
+GT : '>';
+LT : '<';
+Plus : '+';
+Minus : '-';
+Asterisk : '*';
+ForwardSlash : '/';
+Percent : '%';
+OBrace : '{';
+CBrace : '}';
+OBracket : '[';
+CBracket : ']';
+OParen : '(';
+CParen : ')';
+SColon : ';';
+Comma : ',';
+QMark : '?';
+Colon : ':';
+SingleQuote: '\'';
+
+Bool
+ : 'true'
+ | 'false'
+ ;
+
+Number
+ : Int ('.' Digit*)? (('e' | 'E') ('+' | '-')? Digit*)?
+ ;
+
+//Float
+// : Int ('.' Digit*)? ('e' ('+' | '-')? Digit*)?
+// ;
+//
+//Integer
+// : Digit Digit*
+// ;
+
+Identifier
+ : ('a'..'z' | 'A'..'Z' | '_' | '$') ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)*
+ ;
+
+QuotedIdentifier
+@after {
+ setText(getText().substring(1, getText().length()-1).replaceAll("\\\\(.)", "$1"));
+}
+ : '`' (~('`' | '\\') | '\\' ('\\' | '`'))* '`'
+ ;
+
+String
+@after {
+ setText(getText().substring(1, getText().length()-1).replaceAll("\\\\(.)", "$1"));
+}
+ : '\'' (~('\'' | '\\') | '\\' ('\\' | '\''))* '\''
+ ;
+
+Comment
+ : '//' ~('\r' | '\n')* {skip();}
+ | '/*' .* '*/' {skip();}
+ ;
+
+Space
+ : (' ' | '\t' | '\r' | '\n' | '\u000C') {skip();}
+ ;
+
+fragment Int
+ : '1'..'9' Digit*
+ | '0'
+ ;
+
+fragment Digit
+ : '0'..'9'
+ ;
+
+FallThrough
+ @after{
+ throw new RuntimeException(java.lang.String.format(
+ "Encountered an illegal char on line \%d, column \%d: '\%s'",
+ getLine(), getCharPositionInLine(), getText()
+ )
+ );
+ }
+ :
+ ;
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
----------------------------------------------------------------------
diff --git a/logical/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g b/logical/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
new file mode 100644
index 0000000..600b791
--- /dev/null
+++ b/logical/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
@@ -0,0 +1,323 @@
+parser grammar ExprParser;
+
+options{
+ output=AST;
+ language=Java;
+ tokenVocab=ExprLexer;
+ backtrack=true;
+ memoize=true;
+}
+
+
+
+@header {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.common.expression.parser;
+
+//Explicit import...
+import org.antlr.runtime.BitSet;
+import java.util.*;
+import org.apache.drill.common.expression.*;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
+import org.apache.drill.common.expression.PathSegment.ArraySegment;
+import org.apache.drill.common.types.*;
+import org.apache.drill.common.types.TypeProtos.*;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.exceptions.ExpressionParsingException;
+}
+
+@members{
+ private String fullExpression;
+ private int tokenPos;
+
+ public static void p(String s){
+ System.out.println(s);
+ }
+
+ public ExpressionPosition pos(Token token){
+ return new ExpressionPosition(fullExpression, token.getTokenIndex());
+ }
+
+ @Override
+ public void displayRecognitionError(String[] tokenNames, RecognitionException e) {
+ String hdr = getErrorHeader(e);
+ String msg = getErrorMessage(e, tokenNames);
+ throw new ExpressionParsingException("Expression has syntax error! " + hdr + ":" + msg);
+ }
+}
+
+parse returns [LogicalExpression e]
+ : expression EOF {
+ $e = $expression.e;
+ if(fullExpression == null) fullExpression = $expression.text;
+ tokenPos = $expression.start.getTokenIndex();
+ }
+ ;
+
+functionCall returns [LogicalExpression e]
+ : Identifier OParen exprList? CParen {$e = FunctionCallFactory.createExpression($Identifier.text, pos($Identifier), $exprList.listE); }
+ ;
+
+convertCall returns [LogicalExpression e]
+ : Convert OParen expression Comma String CParen
+ { $e = FunctionCallFactory.createConvert($Convert.text, $String.text, $expression.e, pos($Convert));}
+ ;
+
+castCall returns [LogicalExpression e]
+ @init{
+ List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
+ ExpressionPosition p = null;
+ }
+ : Cast OParen expression As dataType repeat? CParen
+ { if ($repeat.isRep!=null && $repeat.isRep.compareTo(Boolean.TRUE)==0)
+ $e = FunctionCallFactory.createCast(TypeProtos.MajorType.newBuilder().mergeFrom($dataType.type).setMode(DataMode.REPEATED).build(), pos($Cast), $expression.e);
+ else
+ $e = FunctionCallFactory.createCast($dataType.type, pos($Cast), $expression.e);}
+ ;
+
+repeat returns [Boolean isRep]
+ : Repeat { $isRep = Boolean.TRUE;}
+ ;
+
+dataType returns [MajorType type]
+ : numType {$type =$numType.type;}
+ | charType {$type =$charType.type;}
+ | dateType {$type =$dateType.type;}
+ | booleanType {$type =$booleanType.type;}
+ ;
+
+booleanType returns [MajorType type]
+ : BIT { $type = Types.required(TypeProtos.MinorType.BIT); }
+ ;
+
+numType returns [MajorType type]
+ : INT { $type = Types.required(TypeProtos.MinorType.INT); }
+ | BIGINT { $type = Types.required(TypeProtos.MinorType.BIGINT); }
+ | FLOAT4 { $type = Types.required(TypeProtos.MinorType.FLOAT4); }
+ | FLOAT8 { $type = Types.required(TypeProtos.MinorType.FLOAT8); }
+ | DECIMAL9 OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL9).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
+ | DECIMAL18 OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL18).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
+ | DECIMAL28DENSE OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL28DENSE).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
+ | DECIMAL28SPARSE OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL28SPARSE).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
+ | DECIMAL38DENSE OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL38DENSE).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
+ | DECIMAL38SPARSE OParen precision Comma scale CParen { $type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.DECIMAL38SPARSE).setMode(DataMode.REQUIRED).setPrecision($precision.value.intValue()).setScale($scale.value.intValue()).build(); }
+ ;
+
+charType returns [MajorType type]
+ : VARCHAR typeLen {$type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.VARCHAR).setMode(DataMode.REQUIRED).setWidth($typeLen.length.intValue()).build(); }
+ | VARBINARY typeLen {$type = TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.VARBINARY).setMode(DataMode.REQUIRED).setWidth($typeLen.length.intValue()).build();}
+ ;
+
+precision returns [Integer value]
+ : Number {$value = Integer.parseInt($Number.text); }
+ ;
+
+scale returns [Integer value]
+ : Number {$value = Integer.parseInt($Number.text); }
+ ;
+
+dateType returns [MajorType type]
+ : DATE { $type = Types.required(TypeProtos.MinorType.DATE); }
+ | TIMESTAMP { $type = Types.required(TypeProtos.MinorType.TIMESTAMP); }
+ | TIME { $type = Types.required(TypeProtos.MinorType.TIME); }
+ | TIMESTAMPTZ { $type = Types.required(TypeProtos.MinorType.TIMESTAMPTZ); }
+ | INTERVAL { $type = Types.required(TypeProtos.MinorType.INTERVAL); }
+ | INTERVALYEAR { $type = Types.required(TypeProtos.MinorType.INTERVALYEAR); }
+ | INTERVALDAY { $type = Types.required(TypeProtos.MinorType.INTERVALDAY); }
+ ;
+
+typeLen returns [Integer length]
+ : OParen Number CParen {$length = Integer.parseInt($Number.text);}
+ ;
+
+ifStatement returns [LogicalExpression e]
+ @init {
+ IfExpression.Builder s = IfExpression.newBuilder();
+ }
+ @after {
+ $e = s.build();
+ }
+ : i1=ifStat {s.setIfCondition($i1.i); s.setPosition(pos($i1.start)); } (elseIfStat { s.setIfCondition($elseIfStat.i); } )* Else expression { s.setElse($expression.e); }End
+ ;
+
+ifStat returns [IfExpression.IfCondition i]
+ : If e1=expression Then e2=expression { $i = new IfExpression.IfCondition($e1.e, $e2.e); }
+ ;
+elseIfStat returns [IfExpression.IfCondition i]
+ : Else If e1=expression Then e2=expression { $i = new IfExpression.IfCondition($e1.e, $e2.e); }
+ ;
+
+caseStatement returns [LogicalExpression e]
+ @init {
+ IfExpression.Builder s = IfExpression.newBuilder();
+ }
+ @after {
+ $e = s.build();
+ }
+ : Case (caseWhenStat {s.setIfCondition($caseWhenStat.e); }) + caseElseStat { s.setElse($caseElseStat.e); } End
+ ;
+
+caseWhenStat returns [IfExpression.IfCondition e]
+ : When e1=expression Then e2=expression {$e = new IfExpression.IfCondition($e1.e, $e2.e); }
+ ;
+
+caseElseStat returns [LogicalExpression e]
+ : Else expression {$e = $expression.e; }
+ ;
+
+exprList returns [List<LogicalExpression> listE]
+ @init{
+ $listE = new ArrayList<LogicalExpression>();
+ }
+ : e1=expression {$listE.add($e1.e); } (Comma e2=expression {$listE.add($e2.e); } )*
+ ;
+
+expression returns [LogicalExpression e]
+ : ifStatement {$e = $ifStatement.e; }
+ | caseStatement {$e = $caseStatement.e; }
+ | condExpr {$e = $condExpr.e; }
+ ;
+
+condExpr returns [LogicalExpression e]
+ : orExpr {$e = $orExpr.e; }
+ ;
+
+orExpr returns [LogicalExpression e]
+ @init{
+ List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
+ ExpressionPosition p = null;
+ }
+ @after{
+ if(exprs.size() == 1){
+ $e = exprs.get(0);
+ }else{
+ $e = FunctionCallFactory.createBooleanOperator("or", p, exprs);
+ }
+ }
+ : a1=andExpr { exprs.add($a1.e); p = pos( $a1.start );} (Or a2=andExpr { exprs.add($a2.e); })*
+ ;
+
+andExpr returns [LogicalExpression e]
+ @init{
+ List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
+ ExpressionPosition p = null;
+ }
+ @after{
+ if(exprs.size() == 1){
+ $e = exprs.get(0);
+ }else{
+ $e = FunctionCallFactory.createBooleanOperator("and", p, exprs);
+ }
+ }
+ : e1=equExpr { exprs.add($e1.e); p = pos( $e1.start ); } ( And e2=equExpr { exprs.add($e2.e); })*
+ ;
+
+equExpr returns [LogicalExpression e]
+ @init{
+ List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
+ List<String> cmps = new ArrayList();
+ ExpressionPosition p = null;
+ }
+ @after{
+ $e = FunctionCallFactory.createByOp(exprs, p, cmps);
+ }
+ : r1=relExpr { exprs.add($r1.e); p = pos( $r1.start );
+ } ( cmpr= ( Equals | NEquals ) r2=relExpr {exprs.add($r2.e); cmps.add($cmpr.text); })*
+ ;
+
+relExpr returns [LogicalExpression e]
+ : left=addExpr {$e = $left.e; } (cmpr = (GTEquals | LTEquals | GT | LT) right=addExpr {$e = FunctionCallFactory.createExpression($cmpr.text, pos($left.start), $left.e, $right.e); } )?
+ ;
+
+addExpr returns [LogicalExpression e]
+ @init{
+ List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
+ List<String> ops = new ArrayList();
+ ExpressionPosition p = null;
+ }
+ @after{
+ $e = FunctionCallFactory.createByOp(exprs, p, ops);
+ }
+ : m1=mulExpr {exprs.add($m1.e); p = pos($m1.start); } ( op=(Plus|Minus) m2=mulExpr {exprs.add($m2.e); ops.add($op.text); })*
+ ;
+
+mulExpr returns [LogicalExpression e]
+ @init{
+ List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
+ List<String> ops = new ArrayList();
+ ExpressionPosition p = null;
+ }
+ @after{
+ $e = FunctionCallFactory.createByOp(exprs, p, ops);
+ }
+ : p1=xorExpr {exprs.add($p1.e); p = pos($p1.start);} (op=(Asterisk|ForwardSlash|Percent) p2=xorExpr {exprs.add($p2.e); ops.add($op.text); } )*
+ ;
+
+xorExpr returns [LogicalExpression e]
+ @init{
+ List<LogicalExpression> exprs = new ArrayList<LogicalExpression>();
+ List<String> ops = new ArrayList();
+ ExpressionPosition p = null;
+ }
+ @after{
+ $e = FunctionCallFactory.createByOp(exprs, p, ops);
+ }
+ : u1=unaryExpr {exprs.add($u1.e); p = pos($u1.start);} (Caret u2=unaryExpr {exprs.add($u2.e); ops.add($Caret.text);} )*
+ ;
+
+unaryExpr returns [LogicalExpression e]
+ : sign=(Plus|Minus)? Number {$e = ValueExpressions.getNumericExpression($sign.text, $Number.text, pos(($sign != null) ? $sign : $Number)); }
+ | Minus atom {$e = FunctionCallFactory.createExpression("u-", pos($Minus), $atom.e); }
+ | Excl atom {$e= FunctionCallFactory.createExpression("!", pos($Excl), $atom.e); }
+ | atom {$e = $atom.e; }
+ ;
+
+atom returns [LogicalExpression e]
+ : Bool {$e = new ValueExpressions.BooleanExpression($Bool.text, pos($Bool)); }
+ | lookup {$e = $lookup.e; }
+ ;
+
+pathSegment returns [NameSegment seg]
+ : s1=nameSegment {$seg = $s1.seg;}
+ ;
+
+nameSegment returns [NameSegment seg]
+ : QuotedIdentifier ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new NameSegment($QuotedIdentifier.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+ | Identifier ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new NameSegment($Identifier.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+ ;
+
+arraySegment returns [PathSegment seg]
+ : OBracket Number CBracket ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new ArraySegment($Number.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+ ;
+
+
+lookup returns [LogicalExpression e]
+ : functionCall {$e = $functionCall.e ;}
+ | convertCall {$e = $convertCall.e; }
+ | castCall {$e = $castCall.e; }
+ | pathSegment {$e = new SchemaPath($pathSegment.seg, pos($pathSegment.start) ); }
+ | String {$e = new ValueExpressions.QuotedString($String.text, pos($String) ); }
+ | OParen expression CParen {$e = $expression.e; }
+ | SingleQuote Identifier SingleQuote {$e = new SchemaPath($Identifier.text, pos($Identifier) ); }
+ ;
+
+
+
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/JSONOptions.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/JSONOptions.java b/logical/src/main/java/org/apache/drill/common/JSONOptions.java
new file mode 100644
index 0000000..e432135
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/JSONOptions.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+import java.io.IOException;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+
+import org.apache.drill.common.JSONOptions.De;
+import org.apache.drill.common.JSONOptions.Se;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.config.LogicalPlanPersistence;
+import org.apache.drill.common.exceptions.LogicalPlanParsingException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonLocation;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.TreeNode;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.node.TreeTraversingParser;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import com.google.common.base.Preconditions;
+
+@JsonSerialize(using = Se.class)
+@JsonDeserialize(using = De.class)
+public class JSONOptions {
+
+ private final static Logger logger = LoggerFactory.getLogger(JSONOptions.class);
+
+ private JsonNode root;
+ private JsonLocation location;
+ private Object opaque;
+
+ public JSONOptions(Object opaque) {
+ this.opaque = opaque;
+ }
+
+ public JSONOptions(JsonNode n, JsonLocation location) {
+ this.root = n;
+ this.location = location;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> T getWith(LogicalPlanPersistence lpPersistance, Class<T> c) {
+ try {
+ if (opaque != null) {
+ final Class<?> opaqueClass = opaque.getClass();
+ if (opaqueClass.equals(c)) {
+ return (T) opaque;
+ } else {
+ // Enum values that override methods are given $1, $2 ... extensions. Ignore the extension.
+ // e.g. SystemTable$1 for SystemTable.OPTION
+ if (c.isEnum()) {
+ final String opaqueName = opaqueClass.getName().replaceAll("\\$\\d+$", "");
+ final String cName = c.getName();
+ if(opaqueName.equals(cName)) {
+ return (T) opaque;
+ }
+ }
+ throw new IllegalArgumentException(String.format("Attempted to retrieve a option with type of %s. " +
+ "However, the JSON options carried an opaque value of type %s.", c.getName(), opaqueClass.getName()));
+ }
+ }
+
+ //logger.debug("Read tree {}", root);
+ return lpPersistance.getMapper().treeToValue(root, c);
+ } catch (JsonProcessingException e) {
+ throw new LogicalPlanParsingException(String.format("Failure while trying to convert late bound " +
+ "json options to type of %s. Reference was originally located at line %d, column %d.",
+ c.getCanonicalName(), location.getLineNr(), location.getColumnNr()), e);
+ }
+ }
+
+ public <T> T getListWith(LogicalPlanPersistence config, TypeReference<T> t) throws IOException {
+ return getListWith(config.getMapper(), t);
+ }
+
+ public JsonNode asNode(){
+ Preconditions.checkArgument(this.root != null, "Attempted to grab JSONOptions as JsonNode when no root node was stored. You can only convert non-opaque JSONOptions values to JsonNodes.");
+ return root;
+ }
+
+ public JsonParser asParser(){
+ Preconditions.checkArgument(this.root != null, "Attempted to grab JSONOptions as Parser when no root node was stored. You can only convert non-opaque JSONOptions values to parsers.");
+ return new TreeTraversingParser(root);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> T getListWith(ObjectMapper mapper, TypeReference<T> t) throws IOException {
+ if (opaque != null) {
+ Type c = t.getType();
+ if (c instanceof ParameterizedType) {
+ c = ((ParameterizedType)c).getRawType();
+ }
+ if ( c.equals(opaque.getClass())) {
+ return (T) opaque;
+ } else {
+ throw new IOException(String.format("Attempted to retrieve a list with type of %s. However, the JSON " +
+ "options carried an opaque value of type %s.", t.getType(), opaque.getClass().getName()));
+ }
+ }
+ if (root == null) {
+ return null;
+ }
+ return mapper.treeAsTokens(root).readValueAs(t);
+ }
+
+ public JsonNode path(String name) {
+ return root.path(name);
+ }
+
+ public JsonNode getRoot() {
+ return root;
+ }
+
+ public static class De extends StdDeserializer<JSONOptions> {
+
+ public De() {
+ super(JSONOptions.class);
+ logger.debug("Creating Deserializer.");
+ }
+
+ @Override
+ public JSONOptions deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
+ JsonProcessingException {
+ JsonLocation l = jp.getTokenLocation();
+// logger.debug("Reading tree.");
+ TreeNode n = jp.readValueAsTree();
+// logger.debug("Tree {}", n);
+ if (n instanceof JsonNode) {
+ return new JSONOptions( (JsonNode) n, l);
+ } else {
+ throw new IllegalArgumentException(String.format("Received something other than a JsonNode %s", n));
+ }
+ }
+
+ }
+
+ public static class Se extends StdSerializer<JSONOptions> {
+
+ public Se() {
+ super(JSONOptions.class);
+ }
+
+ @Override
+ public void serialize(JSONOptions value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
+ JsonGenerationException {
+ if (value.opaque != null) {
+ jgen.writeObject(value.opaque);
+ } else {
+ jgen.writeTree(value.root);
+ }
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java b/logical/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
new file mode 100644
index 0000000..cd7a8d0
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.config;
+
+import java.util.Set;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.FormatPluginConfigBase;
+import org.apache.drill.common.logical.StoragePluginConfigBase;
+import org.apache.drill.common.logical.data.LogicalOperatorBase;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+
+public class LogicalPlanPersistence {
+ private ObjectMapper mapper;
+
+ public ObjectMapper getMapper() {
+ return mapper;
+ }
+
+ public LogicalPlanPersistence(DrillConfig conf, ScanResult scanResult) {
+ mapper = new ObjectMapper();
+
+ SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
+ .addDeserializer(LogicalExpression.class, new LogicalExpression.De(conf))
+ .addDeserializer(SchemaPath.class, new SchemaPath.De());
+
+ mapper.registerModule(deserModule);
+ mapper.enable(SerializationFeature.INDENT_OUTPUT);
+ mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+ mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
+ mapper.configure(Feature.ALLOW_COMMENTS, true);
+ registerSubtypes(LogicalOperatorBase.getSubTypes(scanResult));
+ registerSubtypes(StoragePluginConfigBase.getSubTypes(scanResult));
+ registerSubtypes(FormatPluginConfigBase.getSubTypes(scanResult));
+ }
+
+ private <T> void registerSubtypes(Set<Class<? extends T>> types) {
+ for (Class<? extends T> type : types) {
+ mapper.registerSubtypes(type);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/exceptions/ExecutionSetupException.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/exceptions/ExecutionSetupException.java b/logical/src/main/java/org/apache/drill/common/exceptions/ExecutionSetupException.java
new file mode 100644
index 0000000..ae70ffa
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/exceptions/ExecutionSetupException.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.exceptions;
+
+import java.lang.reflect.InvocationTargetException;
+
+public class ExecutionSetupException extends DrillException {
+
+ public static ExecutionSetupException fromThrowable(String message, Throwable cause) {
+ Throwable t = cause instanceof InvocationTargetException
+ ? ((InvocationTargetException)cause).getTargetException() : cause;
+ if (t instanceof ExecutionSetupException) {
+ return ((ExecutionSetupException) t);
+ }
+ return new ExecutionSetupException(message, t);
+ }
+ public ExecutionSetupException() {
+ super();
+
+ }
+
+ public ExecutionSetupException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+
+ }
+
+ public ExecutionSetupException(String message, Throwable cause) {
+ super(message, cause);
+
+ }
+
+ public ExecutionSetupException(String message) {
+ super(message);
+
+ }
+
+ public ExecutionSetupException(Throwable cause) {
+ super(cause);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/exceptions/ExpressionParsingException.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/exceptions/ExpressionParsingException.java b/logical/src/main/java/org/apache/drill/common/exceptions/ExpressionParsingException.java
new file mode 100644
index 0000000..835182d
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/exceptions/ExpressionParsingException.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.exceptions;
+
+public class ExpressionParsingException extends LogicalPlanParsingException {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionParsingException.class);
+
+ public ExpressionParsingException() {
+ super();
+ }
+
+ public ExpressionParsingException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public ExpressionParsingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ExpressionParsingException(String message) {
+ super(message);
+ }
+
+ public ExpressionParsingException(Throwable cause) {
+ super(cause);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/exceptions/LogicalOperatorValidationException.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/exceptions/LogicalOperatorValidationException.java b/logical/src/main/java/org/apache/drill/common/exceptions/LogicalOperatorValidationException.java
new file mode 100644
index 0000000..3debf28
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/exceptions/LogicalOperatorValidationException.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.exceptions;
+
+public class LogicalOperatorValidationException extends LogicalPlanParsingException {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalOperatorValidationException.class);
+
+ public LogicalOperatorValidationException() {
+ super();
+ }
+
+ public LogicalOperatorValidationException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public LogicalOperatorValidationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public LogicalOperatorValidationException(String message) {
+ super(message);
+ }
+
+ public LogicalOperatorValidationException(Throwable cause) {
+ super(cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java b/logical/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
new file mode 100644
index 0000000..c8c0afa
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.exceptions;
+import org.apache.drill.common.logical.data.LogicalOperator;
+
+public class LogicalPlanParsingException extends DrillRuntimeException{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalPlanParsingException.class);
+
+ private LogicalOperator operator;
+
+ public LogicalPlanParsingException() {
+ super();
+ }
+
+ public LogicalPlanParsingException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public LogicalPlanParsingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public LogicalPlanParsingException(String message) {
+ super(message);
+ }
+
+ public LogicalPlanParsingException(Throwable cause) {
+ super(cause);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/BooleanOperator.java b/logical/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
new file mode 100644
index 0000000..809d3e2
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.common.expression;
+
+import java.util.List;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.Types;
+
+public class BooleanOperator extends FunctionCall{
+
+ public BooleanOperator(String name, List<LogicalExpression> args, ExpressionPosition pos) {
+ super(name, args, pos);
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E{
+ return visitor.visitBooleanOperator(this, value);
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ // If any of argumgnet of a boolean "and"/"or" is nullable, the result is nullable bit.
+ // Otherwise, it's non-nullable bit.
+ for (LogicalExpression e : args) {
+ if (e.getMajorType().getMode() == DataMode.OPTIONAL) {
+ return Types.OPTIONAL_BIT;
+ }
+ }
+ return Types.REQUIRED_BIT;
+
+ }
+
+ @Override
+ public int getCumulativeCost() {
+ // return the average cost of operands for a boolean "and" | "or"
+ int cost = this.getSelfCost();
+
+ int i = 0;
+ for (LogicalExpression e : this) {
+ cost += e.getCumulativeCost();
+ i++;
+ }
+
+ return (int) (cost / i) ;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/CastExpression.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/CastExpression.java b/logical/src/main/java/org/apache/drill/common/expression/CastExpression.java
new file mode 100644
index 0000000..5d09a24
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/CastExpression.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class CastExpression extends LogicalExpressionBase implements Iterable<LogicalExpression>{
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CastExpression.class);
+
+ private final LogicalExpression input;
+ private final MajorType type;
+
+ public CastExpression(LogicalExpression input, MajorType type, ExpressionPosition pos) {
+ super(pos);
+ this.input = input;
+ this.type = checkNotNull(type, "Major type cannot be null");
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitCastExpression(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Collections.singleton(input).iterator();
+ }
+
+ public LogicalExpression getInput() {
+ return input;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return type;
+ }
+
+ @Override
+ public String toString() {
+ return "CastExpression [input=" + input + ", type=" + type + "]";
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/ConvertExpression.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ConvertExpression.java b/logical/src/main/java/org/apache/drill/common/expression/ConvertExpression.java
new file mode 100644
index 0000000..dda97e6
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/ConvertExpression.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.Types;
+
+public class ConvertExpression extends LogicalExpressionBase implements Iterable<LogicalExpression>{
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ConvertExpression.class);
+
+ public static final String CONVERT_FROM = "convert_from";
+ public static final String CONVERT_TO = "convert_to";
+
+ private final LogicalExpression input;
+ private final MajorType type;
+ private final String convertFunction;
+ private final String encodingType;
+
+ /**
+ * @param encodingType
+ * @param convertFunction
+ * @param input
+ * @param pos
+ */
+ public ConvertExpression(String convertFunction, String encodingType, LogicalExpression input, ExpressionPosition pos) {
+ super(pos);
+ this.input = input;
+ this.convertFunction = CONVERT_FROM.equals(convertFunction.toLowerCase()) ? CONVERT_FROM : CONVERT_TO;
+ this.encodingType = encodingType.toUpperCase();
+ this.type = Types.getMajorTypeFromName(encodingType.split("_", 2)[0].toLowerCase());
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitConvertExpression(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Collections.singleton(input).iterator();
+ }
+
+ public String getConvertFunction() {
+ return convertFunction;
+ }
+
+ public LogicalExpression getInput() {
+ return input;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return type;
+ }
+
+ public String getEncodingType() {
+ return encodingType;
+ }
+
+ @Override
+ public String toString() {
+ return "ConvertExpression [input=" + input + ", type=" + Types.toString(type) + ", convertFunction="
+ + convertFunction + ", conversionType=" + encodingType + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/ErrorCollector.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ErrorCollector.java b/logical/src/main/java/org/apache/drill/common/expression/ErrorCollector.java
new file mode 100644
index 0000000..f229e53
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/ErrorCollector.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import com.google.common.collect.Range;
+
+public interface ErrorCollector {
+
+ public void addGeneralError(ExpressionPosition expr, String s);
+
+ public void addUnexpectedArgumentType(ExpressionPosition expr, String name, MajorType actual, MajorType[] expected, int argumentIndex);
+
+ public void addUnexpectedArgumentCount(ExpressionPosition expr, int actual, Range<Integer> expected);
+
+ public void addUnexpectedArgumentCount(ExpressionPosition expr, int actual, int expected);
+
+ public void addNonNumericType(ExpressionPosition expr, MajorType actual);
+
+ public void addUnexpectedType(ExpressionPosition expr, int index, MajorType actual);
+
+ public void addExpectedConstantValue(ExpressionPosition expr, int actual, String s);
+
+ boolean hasErrors();
+
+ public int getErrorCount();
+
+ String toErrorString();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java b/logical/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
new file mode 100644
index 0000000..4f958ec
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.drill.common.expression.visitors.ExpressionValidationError;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+
+public class ErrorCollectorImpl implements ErrorCollector {
+ List<ExpressionValidationError> errors;
+
+ public ErrorCollectorImpl() {
+ errors = Lists.newArrayList();
+ }
+
+ private String addExpr(ExpressionPosition expr, String message) {
+ return String.format("Error in expression at index %d. Error: %s. Full expression: %s.", expr.getCharIndex(), message, expr.getExpression());
+ }
+
+ @Override
+ public void addGeneralError(ExpressionPosition expr, String s) {
+ errors.add(new ExpressionValidationError(addExpr(expr, s)));
+ }
+
+ @Override
+ public void addUnexpectedArgumentType(ExpressionPosition expr, String name, MajorType actual, MajorType[] expected, int argumentIndex) {
+ errors.add(
+ new ExpressionValidationError(
+ addExpr(expr, String.format(
+ "Unexpected argument type. Index :%d Name: %s, Type: %s, Expected type(s): %s",
+ argumentIndex, name, actual, Arrays.toString(expected)
+ ))
+ )
+ );
+ }
+
+ @Override
+ public void addUnexpectedArgumentCount(ExpressionPosition expr, int actual, Range<Integer> expected) {
+ errors.add(new ExpressionValidationError(
+ addExpr(expr, String.format("Unexpected argument count. Actual argument count: %d, Expected range: %s", actual, expected))
+ ));
+ }
+
+ @Override
+ public void addUnexpectedArgumentCount(ExpressionPosition expr, int actual, int expected) {
+ errors.add(new ExpressionValidationError(
+ addExpr(expr, String.format("Unexpected argument count. Actual argument count: %d, Expected count: %d", actual, expected))
+ ));
+ }
+
+ @Override
+ public void addNonNumericType(ExpressionPosition expr, MajorType actual) {
+ errors.add(new ExpressionValidationError(
+ addExpr(expr, String.format("Unexpected numeric type. Actual type: %s", actual))
+ ));
+ }
+
+ @Override
+ public void addUnexpectedType(ExpressionPosition expr, int index, MajorType actual) {
+ errors.add(new ExpressionValidationError(
+ addExpr(expr, String.format("Unexpected argument type. Actual type: %s, Index: %d", actual, index))
+ ));
+ }
+
+ @Override
+ public void addExpectedConstantValue(ExpressionPosition expr, int actual, String s) {
+ errors.add(new ExpressionValidationError(
+ addExpr(expr, String.format("Unexpected constant value. Name: %s, Actual: %s", s, actual))
+ ));
+ }
+
+ @Override
+ public boolean hasErrors() {
+ return !errors.isEmpty();
+ }
+
+
+ @Override
+ public int getErrorCount() {
+ return errors.size();
+ }
+
+ @Override
+ public String toErrorString() {
+ return "\n" + Joiner.on("\n").join(errors);
+ }
+
+ @Override
+ public String toString() {
+ return toErrorString();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
new file mode 100644
index 0000000..bc478dd
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Type;
+import java.util.List;
+
+import org.apache.drill.common.expression.visitors.ExpressionValidationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.reflect.TypeToken;
+
+@SuppressWarnings("unchecked")
+public class ExpressionFunction {
+ final static Logger logger = LoggerFactory.getLogger(ExpressionFunction.class);
+
+ @SuppressWarnings("serial")
+ final static Type EXPR_LIST = (new TypeToken<List<LogicalExpression>>() {
+ }).getType();
+
+ private static final Class<?>[] FUNCTIONS = {};
+
+ private static final ImmutableMap<String, Constructor<LogicalExpression>> FUNCTION_MAP;
+
+ static {
+ ImmutableMap.Builder<String, Constructor<LogicalExpression>> builder = ImmutableMap.builder();
+ for (Class<?> c : FUNCTIONS) {
+
+ // logger.debug("Adding {} and function", c);
+ if (!LogicalExpression.class.isAssignableFrom(c)) {
+ logger.error(
+ "The provided Class [{}] does not derive from LogicalExpression. Skipping inclusion in registry.", c);
+ continue;
+ }
+ FunctionName fn = c.getAnnotation(FunctionName.class);
+ if (fn == null) {
+ logger.error(
+ "The provided Class [{}] did not have a FunctionName annotation. Skipping inclusion in registry.", c);
+ continue;
+ }
+ String name = fn.value();
+ try {
+ Constructor<LogicalExpression> m = (Constructor<LogicalExpression>) c.getConstructor(List.class);
+ if (!EXPR_LIST.equals(m.getGenericParameterTypes()[0])) {
+ logger
+ .error(
+ "The constructor for each function must have a argument list that only contains a List<LogicalExpression>. The class[{}] has an inccorect List<{}> argument.",
+ c, m.getGenericParameterTypes()[0]);
+ continue;
+ }
+
+ builder.put(name, m);
+ } catch (Exception e) {
+ logger
+ .error(
+ "Failure while attempting to retrieve Logical Expression list constructor on class [{}]. Functions must have one of these.",
+ c, e);
+ }
+ }
+
+ FUNCTION_MAP = builder.build();
+ }
+
+ public static LogicalExpression create(String functionName, List<LogicalExpression> expressions)
+ throws ExpressionValidationException {
+ // logger.debug("Requesting generation of new function with name {}.",
+ // functionName);
+ if (!FUNCTION_MAP.containsKey(functionName)) {
+ throw new ExpressionValidationException(String.format("Unknown function with name '%s'", functionName));
+ }
+ try {
+ return FUNCTION_MAP.get(functionName).newInstance(expressions);
+ } catch (Exception e) {
+ throw new ExpressionValidationException("Failure while attempting to build type of " + functionName, e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/ExpressionPosition.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionPosition.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionPosition.java
new file mode 100644
index 0000000..e3f34f4
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionPosition.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+public class ExpressionPosition {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionPosition.class);
+
+ public static final ExpressionPosition UNKNOWN = new ExpressionPosition("--UNKNOWN EXPRESSION--", -1);
+
+ private final String expression;
+ private final int charIndex;
+
+ public ExpressionPosition(String expression, int charIndex) {
+ super();
+ this.expression = expression;
+ this.charIndex = charIndex;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString()
+ + "[charIndex = " + charIndex + ", expression = " + expression + "]";
+ }
+
+ public String getExpression() {
+ return expression;
+ }
+
+ public int getCharIndex() {
+ return charIndex;
+ }
+
+
+
+}
[12/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java b/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
deleted file mode 100644
index c676769..0000000
--- a/exec/java-exec/src/main/codegen/templates/UnionListWriter.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.UnsupportedOperationException;
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionListWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using freemarker and the ${.template_name} template.
- */
-
-@SuppressWarnings("unused")
-public class UnionListWriter extends AbstractFieldWriter {
-
- private ListVector vector;
- private UInt4Vector offsets;
- private PromotableWriter writer;
- private boolean inMap = false;
- private String mapName;
- private int lastIndex = 0;
-
- public UnionListWriter(ListVector vector) {
- super(null);
- this.vector = vector;
- this.writer = new PromotableWriter(vector.getDataVector(), vector);
- this.offsets = vector.getOffsetVector();
- }
-
- public UnionListWriter(ListVector vector, AbstractFieldWriter parent) {
- this(vector);
- }
-
- @Override
- public void allocate() {
- vector.allocateNew();
- }
-
- @Override
- public void clear() {
- vector.clear();
- }
-
- @Override
- public MaterializedField getField() {
- return null;
- }
-
- @Override
- public int getValueCapacity() {
- return vector.getValueCapacity();
- }
-
- @Override
- public void close() throws Exception {
-
- }
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
-
- <#if !minor.class?starts_with("Decimal")>
-
- @Override
- public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>() {
- return this;
- }
-
- @Override
- public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
- assert inMap;
- mapName = name;
- final int nextOffset = offsets.getAccessor().get(idx() + 1);
- vector.getMutator().setNotNull(idx());
- writer.setPosition(nextOffset);
- ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name);
- return ${uncappedName}Writer;
- }
-
- </#if>
-
- </#list></#list>
-
- @Override
- public MapWriter map() {
- inMap = true;
- return this;
- }
-
- @Override
- public ListWriter list() {
- final int nextOffset = offsets.getAccessor().get(idx() + 1);
- vector.getMutator().setNotNull(idx());
- offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
- writer.setPosition(nextOffset);
- return writer;
- }
-
- @Override
- public ListWriter list(String name) {
- final int nextOffset = offsets.getAccessor().get(idx() + 1);
- vector.getMutator().setNotNull(idx());
- writer.setPosition(nextOffset);
- ListWriter listWriter = writer.list(name);
- return listWriter;
- }
-
- @Override
- public MapWriter map(String name) {
- MapWriter mapWriter = writer.map(name);
- return mapWriter;
- }
-
- @Override
- public void startList() {
- vector.getMutator().startNewValue(idx());
- }
-
- @Override
- public void endList() {
-
- }
-
- @Override
- public void start() {
- assert inMap;
- final int nextOffset = offsets.getAccessor().get(idx() + 1);
- vector.getMutator().setNotNull(idx());
- offsets.getMutator().setSafe(idx() + 1, nextOffset);
- writer.setPosition(nextOffset);
- }
-
- @Override
- public void end() {
- if (inMap) {
- inMap = false;
- final int nextOffset = offsets.getAccessor().get(idx() + 1);
- offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
- }
- }
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
-
- <#if !minor.class?starts_with("Decimal")>
-
- @Override
- public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- assert !inMap;
- final int nextOffset = offsets.getAccessor().get(idx() + 1);
- vector.getMutator().setNotNull(idx());
- writer.setPosition(nextOffset);
- writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
- offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
- }
-
- </#if>
-
- </#list></#list>
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/UnionReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionReader.java b/exec/java-exec/src/main/codegen/templates/UnionReader.java
deleted file mode 100644
index 46a32ee..0000000
--- a/exec/java-exec/src/main/codegen/templates/UnionReader.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.vector.complex.impl.NullReader;
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionReader.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-@SuppressWarnings("unused")
-public class UnionReader extends AbstractFieldReader {
-
- private BaseReader[] readers = new BaseReader[43];
- public UnionVector data;
-
- public UnionReader(UnionVector data) {
- this.data = data;
- }
-
- private static MajorType[] TYPES = new MajorType[43];
-
- static {
- for (MinorType minorType : MinorType.values()) {
- TYPES[minorType.getNumber()] = Types.optional(minorType);
- }
- }
-
- public MajorType getType() {
- return TYPES[data.getTypeValue(idx())];
- }
-
- public boolean isSet(){
- return !data.getAccessor().isNull(idx());
- }
-
- public void read(UnionHolder holder) {
- holder.reader = this;
- holder.isSet = this.isSet() ? 1 : 0;
- }
-
- public void read(int index, UnionHolder holder) {
- getList().read(index, holder);
- }
-
- private FieldReader getReaderForIndex(int index) {
- int typeValue = data.getTypeValue(index);
- FieldReader reader = (FieldReader) readers[typeValue];
- if (reader != null) {
- return reader;
- }
- switch (typeValue) {
- case 0:
- return NullReader.INSTANCE;
- case MinorType.MAP_VALUE:
- return (FieldReader) getMap();
- case MinorType.LIST_VALUE:
- return (FieldReader) getList();
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
- case MinorType.${name?upper_case}_VALUE:
- return (FieldReader) get${name}();
- </#if>
- </#list></#list>
- default:
- throw new UnsupportedOperationException("Unsupported type: " + MinorType.valueOf(typeValue));
- }
- }
-
- private SingleMapReaderImpl mapReader;
-
- private MapReader getMap() {
- if (mapReader == null) {
- mapReader = (SingleMapReaderImpl) data.getMap().getReader();
- mapReader.setPosition(idx());
- readers[MinorType.MAP_VALUE] = mapReader;
- }
- return mapReader;
- }
-
- private UnionListReader listReader;
-
- private FieldReader getList() {
- if (listReader == null) {
- listReader = new UnionListReader(data.getList());
- listReader.setPosition(idx());
- readers[MinorType.LIST_VALUE] = listReader;
- }
- return listReader;
- }
-
- @Override
- public java.util.Iterator<String> iterator() {
- return getMap().iterator();
- }
-
- @Override
- public void copyAsValue(UnionWriter writer) {
- writer.data.copyFrom(idx(), writer.idx(), data);
- }
-
- <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
- "Character", "DateTime", "Period", "Double", "Float",
- "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
- <#assign safeType=friendlyType />
- <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-
- @Override
- public ${friendlyType} read${safeType}() {
- return getReaderForIndex(idx()).read${safeType}();
- }
-
- </#list>
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign uncappedName = name?uncap_first/>
- <#assign boxedType = (minor.boxedType!type.boxedType) />
- <#assign javaType = (minor.javaType!type.javaType) />
- <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
- <#assign safeType=friendlyType />
- <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
- <#if !minor.class?starts_with("Decimal")>
-
- private Nullable${name}ReaderImpl ${uncappedName}Reader;
-
- private Nullable${name}ReaderImpl get${name}() {
- if (${uncappedName}Reader == null) {
- ${uncappedName}Reader = new Nullable${name}ReaderImpl(data.get${name}Vector());
- ${uncappedName}Reader.setPosition(idx());
- readers[MinorType.${name?upper_case}_VALUE] = ${uncappedName}Reader;
- }
- return ${uncappedName}Reader;
- }
-
- public void read(Nullable${name}Holder holder){
- getReaderForIndex(idx()).read(holder);
- }
-
- public void copyAsValue(${name}Writer writer){
- getReaderForIndex(idx()).copyAsValue(writer);
- }
- </#if>
- </#list></#list>
-
- @Override
- public void copyAsValue(ListWriter writer) {
- ComplexCopier.copy(this, (FieldWriter) writer);
- }
-
- @Override
- public void setPosition(int index) {
- super.setPosition(index);
- for (BaseReader reader : readers) {
- if (reader != null) {
- reader.setPosition(index);
- }
- }
- }
-
- public FieldReader reader(String name){
- return getMap().reader(name);
- }
-
- public FieldReader reader() {
- return getList().reader();
- }
-
- public boolean next() {
- return getReaderForIndex(idx()).next();
- }
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionVector.java b/exec/java-exec/src/main/codegen/templates/UnionVector.java
deleted file mode 100644
index dfcccb2..0000000
--- a/exec/java-exec/src/main/codegen/templates/UnionVector.java
+++ /dev/null
@@ -1,479 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/UnionVector.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex;
-
-<#include "/@includes/vv_imports.ftl" />
-import java.util.Iterator;
-import org.apache.drill.exec.vector.complex.impl.ComplexCopier;
-import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.common.expression.PathSegment;
-
-/*
- * This class is generated using freemarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-
-
-/**
- * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each
- * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods,
- * as well as the addOrGet method.
- *
- * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
- * each time the vector is accessed.
- */
-public class UnionVector implements ValueVector {
-
- private MaterializedField field;
- private BufferAllocator allocator;
- private Accessor accessor = new Accessor();
- private Mutator mutator = new Mutator();
- private int valueCount;
-
- private MapVector internalMap;
- private UInt1Vector typeVector;
-
- private MapVector mapVector;
- private ListVector listVector;
-
- private FieldReader reader;
- private NullableBitVector bit;
-
- private int singleType = 0;
- private ValueVector singleVector;
- private MajorType majorType;
-
- private final CallBack callBack;
-
- public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- this.field = field.clone();
- this.allocator = allocator;
- this.internalMap = new MapVector("internal", allocator, callBack);
- this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class);
- this.field.addChild(internalMap.getField().clone());
- this.majorType = field.getType();
- this.callBack = callBack;
- }
-
- public List<MinorType> getSubTypes() {
- return majorType.getSubTypeList();
- }
-
- private void addSubType(MinorType type) {
- majorType = MajorType.newBuilder(this.majorType).addSubType(type).build();
- if (callBack != null) {
- callBack.doWork();
- }
- }
-
- private static final MajorType MAP_TYPE = Types.optional(MinorType.MAP);
-
- public MapVector getMap() {
- if (mapVector == null) {
- int vectorCount = internalMap.size();
- mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class);
- addSubType(MinorType.MAP);
- if (internalMap.size() > vectorCount) {
- mapVector.allocateNew();
- }
- }
- return mapVector;
- }
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
-
- private Nullable${name}Vector ${uncappedName}Vector;
- private static final MajorType ${name?upper_case}_TYPE = Types.optional(MinorType.${name?upper_case});
-
- public Nullable${name}Vector get${name}Vector() {
- if (${uncappedName}Vector == null) {
- int vectorCount = internalMap.size();
- ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class);
- addSubType(MinorType.${name?upper_case});
- if (internalMap.size() > vectorCount) {
- ${uncappedName}Vector.allocateNew();
- }
- }
- return ${uncappedName}Vector;
- }
-
- </#if>
-
- </#list></#list>
-
- private static final MajorType LIST_TYPE = Types.optional(MinorType.LIST);
-
- public ListVector getList() {
- if (listVector == null) {
- int vectorCount = internalMap.size();
- listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class);
- addSubType(MinorType.LIST);
- if (internalMap.size() > vectorCount) {
- listVector.allocateNew();
- }
- }
- return listVector;
- }
-
- public int getTypeValue(int index) {
- return typeVector.getAccessor().get(index);
- }
-
- public UInt1Vector getTypeVector() {
- return typeVector;
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryRuntimeException {
- internalMap.allocateNew();
- if (typeVector != null) {
- typeVector.zeroVector();
- }
- }
-
- @Override
- public boolean allocateNewSafe() {
- boolean safe = internalMap.allocateNewSafe();
- if (safe) {
- if (typeVector != null) {
- typeVector.zeroVector();
- }
- }
- return safe;
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- }
-
- @Override
- public int getValueCapacity() {
- return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity());
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public void clear() {
- internalMap.clear();
- }
-
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- @Override
- public TransferPair getTransferPair() {
- return new TransferImpl(field);
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref) {
- return new TransferImpl(field.withPath(ref));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector target) {
- return new TransferImpl((UnionVector) target);
- }
-
- public void transferTo(UnionVector target) {
- internalMap.makeTransferPair(target.internalMap).transfer();
- target.valueCount = valueCount;
- target.majorType = majorType;
- }
-
- public void copyFrom(int inIndex, int outIndex, UnionVector from) {
- from.getReader().setPosition(inIndex);
- getWriter().setPosition(outIndex);
- ComplexCopier.copy(from.reader, mutator.writer);
- }
-
- public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
- copyFrom(inIndex, outIndex, from);
- }
-
- public void addVector(ValueVector v) {
- String name = v.getField().getType().getMinorType().name().toLowerCase();
- Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
- internalMap.putChild(name, v);
- addSubType(v.getField().getType().getMinorType());
- }
-
- private class TransferImpl implements TransferPair {
-
- UnionVector to;
-
- public TransferImpl(MaterializedField field) {
- to = new UnionVector(field, allocator, null);
- }
-
- public TransferImpl(UnionVector to) {
- this.to = to;
- }
-
- @Override
- public void transfer() {
- transferTo(to);
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
-
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- this.to.copyFrom(from, to, UnionVector.this);
- }
- }
-
- @Override
- public Accessor getAccessor() {
- return accessor;
- }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- @Override
- public FieldReader getReader() {
- if (reader == null) {
- reader = new UnionReader(this);
- }
- return reader;
- }
-
- public FieldWriter getWriter() {
- if (mutator.writer == null) {
- mutator.writer = new UnionWriter(this);
- }
- return mutator.writer;
- }
-
- @Override
- public UserBitShared.SerializedField getMetadata() {
- SerializedField.Builder b = getField() //
- .getAsBuilder() //
- .setBufferLength(getBufferSize()) //
- .setValueCount(valueCount);
-
- b.addChild(internalMap.getMetadata());
- return b.build();
- }
-
- @Override
- public int getBufferSize() {
- return internalMap.getBufferSize();
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- long bufferSize = 0;
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSizeFor(valueCount);
- }
-
- return (int) bufferSize;
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- return internalMap.getBuffers(clear);
- }
-
- @Override
- public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
- valueCount = metadata.getValueCount();
-
- internalMap.load(metadata.getChild(0), buffer);
- }
-
- @Override
- public Iterator<ValueVector> iterator() {
- List<ValueVector> vectors = Lists.newArrayList(internalMap.iterator());
- vectors.add(typeVector);
- return vectors.iterator();
- }
-
- public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
- if (seg.isNamed()) {
- ValueVector v = getMap();
- if (v != null) {
- return ((AbstractContainerVector) v).getFieldIdIfMatches(builder, addToBreadCrumb, seg);
- } else {
- return null;
- }
- } else if (seg.isArray()) {
- ValueVector v = getList();
- if (v != null) {
- return ((ListVector) v).getFieldIdIfMatches(builder, addToBreadCrumb, seg);
- }
- else return null;
- }
- return null;
- }
-
- public class Accessor extends BaseValueVector.BaseAccessor {
-
-
- @Override
- public Object getObject(int index) {
- int type = typeVector.getAccessor().get(index);
- switch (type) {
- case 0:
- return null;
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
- case MinorType.${name?upper_case}_VALUE:
- return get${name}Vector().getAccessor().getObject(index);
- </#if>
-
- </#list></#list>
- case MinorType.MAP_VALUE:
- return getMap().getAccessor().getObject(index);
- case MinorType.LIST_VALUE:
- return getList().getAccessor().getObject(index);
- default:
- throw new UnsupportedOperationException("Cannot support type: " + MinorType.valueOf(type));
- }
- }
-
- public byte[] get(int index) {
- return null;
- }
-
- public void get(int index, ComplexHolder holder) {
- }
-
- public void get(int index, UnionHolder holder) {
- if (reader == null) {
- reader = new UnionReader(UnionVector.this);
- }
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- @Override
- public int getValueCount() {
- return valueCount;
- }
-
- @Override
- public boolean isNull(int index) {
- return typeVector.getAccessor().get(index) == 0;
- }
-
- public int isSet(int index) {
- return isNull(index) ? 0 : 1;
- }
- }
-
- public class Mutator extends BaseValueVector.BaseMutator {
-
- UnionWriter writer;
-
- @Override
- public void setValueCount(int valueCount) {
- UnionVector.this.valueCount = valueCount;
- internalMap.getMutator().setValueCount(valueCount);
- }
-
- public void setSafe(int index, UnionHolder holder) {
- FieldReader reader = holder.reader;
- if (writer == null) {
- writer = new UnionWriter(UnionVector.this);
- }
- writer.setPosition(index);
- MinorType type = reader.getType().getMinorType();
- switch (type) {
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
- case ${name?upper_case}:
- Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
- reader.read(${uncappedName}Holder);
- setSafe(index, ${uncappedName}Holder);
- break;
- </#if>
- </#list></#list>
- case MAP: {
- ComplexCopier.copy(reader, writer);
- break;
- }
- case LIST: {
- ComplexCopier.copy(reader, writer);
- break;
- }
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
- public void setSafe(int index, Nullable${name}Holder holder) {
- setType(index, MinorType.${name?upper_case});
- get${name}Vector().getMutator().setSafe(index, holder);
- }
-
- </#if>
- </#list></#list>
-
- public void setType(int index, MinorType type) {
- typeVector.getMutator().setSafe(index, type.getNumber());
- }
-
- @Override
- public void reset() { }
-
- @Override
- public void generateTestData(int values) { }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/UnionWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/UnionWriter.java b/exec/java-exec/src/main/codegen/templates/UnionWriter.java
deleted file mode 100644
index 7a123b4..0000000
--- a/exec/java-exec/src/main/codegen/templates/UnionWriter.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using freemarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
-
- UnionVector data;
- private MapWriter mapWriter;
- private UnionListWriter listWriter;
- private List<BaseWriter> writers = Lists.newArrayList();
-
- public UnionWriter(BufferAllocator allocator) {
- super(null);
- }
-
- public UnionWriter(UnionVector vector) {
- super(null);
- data = vector;
- }
-
- public UnionWriter(UnionVector vector, FieldWriter parent) {
- super(null);
- data = vector;
- }
-
- @Override
- public void setPosition(int index) {
- super.setPosition(index);
- for (BaseWriter writer : writers) {
- writer.setPosition(index);
- }
- }
-
-
- @Override
- public void start() {
- data.getMutator().setType(idx(), MinorType.MAP);
- getMapWriter().start();
- }
-
- @Override
- public void end() {
- getMapWriter().end();
- }
-
- @Override
- public void startList() {
- getListWriter().startList();
- data.getMutator().setType(idx(), MinorType.LIST);
- }
-
- @Override
- public void endList() {
- getListWriter().endList();
- }
-
- private MapWriter getMapWriter() {
- if (mapWriter == null) {
- mapWriter = new SingleMapWriter(data.getMap(), null, true);
- mapWriter.setPosition(idx());
- writers.add(mapWriter);
- }
- return mapWriter;
- }
-
- public MapWriter asMap() {
- data.getMutator().setType(idx(), MinorType.MAP);
- return getMapWriter();
- }
-
- private ListWriter getListWriter() {
- if (listWriter == null) {
- listWriter = new UnionListWriter(data.getList());
- listWriter.setPosition(idx());
- writers.add(listWriter);
- }
- return listWriter;
- }
-
- public ListWriter asList() {
- data.getMutator().setType(idx(), MinorType.LIST);
- return getListWriter();
- }
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
-
- <#if !minor.class?starts_with("Decimal")>
-
- private ${name}Writer ${name?uncap_first}Writer;
-
- private ${name}Writer get${name}Writer() {
- if (${uncappedName}Writer == null) {
- ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null);
- ${uncappedName}Writer.setPosition(idx());
- writers.add(${uncappedName}Writer);
- }
- return ${uncappedName}Writer;
- }
-
- public ${name}Writer as${name}() {
- data.getMutator().setType(idx(), MinorType.${name?upper_case});
- return get${name}Writer();
- }
-
- @Override
- public void write(${name}Holder holder) {
- data.getMutator().setType(idx(), MinorType.${name?upper_case});
- get${name}Writer().setPosition(idx());
- get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
- }
-
- public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- data.getMutator().setType(idx(), MinorType.${name?upper_case});
- get${name}Writer().setPosition(idx());
- get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
- }
- </#if>
-
- </#list></#list>
-
- public void writeNull() {
- }
-
- @Override
- public MapWriter map() {
- data.getMutator().setType(idx(), MinorType.LIST);
- getListWriter().setPosition(idx());
- return getListWriter().map();
- }
-
- @Override
- public ListWriter list() {
- data.getMutator().setType(idx(), MinorType.LIST);
- getListWriter().setPosition(idx());
- return getListWriter().list();
- }
-
- @Override
- public ListWriter list(String name) {
- data.getMutator().setType(idx(), MinorType.MAP);
- getMapWriter().setPosition(idx());
- return getMapWriter().list(name);
- }
-
- @Override
- public MapWriter map(String name) {
- data.getMutator().setType(idx(), MinorType.MAP);
- getMapWriter().setPosition(idx());
- return getMapWriter().map(name);
- }
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#if !minor.class?starts_with("Decimal")>
- @Override
- public ${capName}Writer ${lowerName}(String name) {
- data.getMutator().setType(idx(), MinorType.MAP);
- getMapWriter().setPosition(idx());
- return getMapWriter().${lowerName}(name);
- }
-
- @Override
- public ${capName}Writer ${lowerName}() {
- data.getMutator().setType(idx(), MinorType.LIST);
- getListWriter().setPosition(idx());
- return getListWriter().${lowerName}();
- }
- </#if>
- </#list></#list>
-
- @Override
- public void allocate() {
- data.allocateNew();
- }
-
- @Override
- public void clear() {
- data.clear();
- }
-
- @Override
- public void close() throws Exception {
- data.close();
- }
-
- @Override
- public MaterializedField getField() {
- return data.getField();
- }
-
- @Override
- public int getValueCapacity() {
- return data.getValueCapacity();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ValueHolders.java b/exec/java-exec/src/main/codegen/templates/ValueHolders.java
deleted file mode 100644
index 4151bbe..0000000
--- a/exec/java-exec/src/main/codegen/templates/ValueHolders.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-<@pp.dropOutputFile />
-<#list vv.modes as mode>
-<#list vv.types as type>
-<#list type.minor as minor>
-
-<#assign className="${mode.prefix}${minor.class}Holder" />
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/holders/${className}.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.expr.holders;
-
-<#include "/@includes/vv_imports.ftl" />
-
-public final class ${className} implements ValueHolder{
-
- public static final MajorType TYPE = Types.${mode.name?lower_case}(MinorType.${minor.class?upper_case});
-
- public MajorType getType() {return TYPE;}
-
- <#if mode.name == "Repeated">
-
- /** The first index (inclusive) into the Vector. **/
- public int start;
-
- /** The last index (exclusive) into the Vector. **/
- public int end;
-
- /** The Vector holding the actual values. **/
- public ${minor.class}Vector vector;
-
- <#else>
- public static final int WIDTH = ${type.width};
-
- <#if mode.name == "Optional">public int isSet;</#if>
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- public ${field.type} ${field.name};
- </#list>
-
- <#if minor.class.startsWith("Decimal")>
- public static final int maxPrecision = ${minor.maxPrecisionDigits};
- <#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")>
- public static final int nDecimalDigits = ${minor.nDecimalDigits};
-
- public static int getInteger(int index, int start, DrillBuf buffer) {
- int value = buffer.getInt(start + (index * 4));
-
- if (index == 0) {
- /* the first byte contains sign bit, return value without it */
- <#if minor.class.endsWith("Sparse")>
- value = (value & 0x7FFFFFFF);
- <#elseif minor.class.endsWith("Dense")>
- value = (value & 0x0000007F);
- </#if>
- }
- return value;
- }
-
- public static void setInteger(int index, int value, int start, DrillBuf buffer) {
- buffer.setInt(start + (index * 4), value);
- }
-
- public static void setSign(boolean sign, int start, DrillBuf buffer) {
- // Set MSB to 1 if sign is negative
- if (sign == true) {
- int value = getInteger(0, start, buffer);
- setInteger(0, (value | 0x80000000), start, buffer);
- }
- }
-
- public static boolean getSign(int start, DrillBuf buffer) {
- return ((buffer.getInt(start) & 0x80000000) != 0);
- }
- </#if></#if>
-
- @Deprecated
- public int hashCode(){
- throw new UnsupportedOperationException();
- }
-
- /*
- * Reason for deprecation is that ValueHolders are potential scalar replacements
- * and hence we don't want any methods to be invoked on them.
- */
- @Deprecated
- public String toString(){
- throw new UnsupportedOperationException();
- }
- </#if>
-
-
-
-
-}
-
-</#list>
-</#list>
-</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
deleted file mode 100644
index bba39fe..0000000
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ /dev/null
@@ -1,645 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.Override;
-
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
-import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.BaseValueVector;
-import org.apache.drill.exec.vector.VariableWidthVector;
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-
-<#if type.major == "VarLen">
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/**
- * ${minor.class}Vector implements a vector of variable width values. Elements in the vector
- * are accessed by position from the logical start of the vector. A fixed width offsetVector
- * is used to convert an element's position to it's offset from the start of the (0-based)
- * DrillBuf. Size is inferred by adjacent elements.
- * The width of each element is ${type.width} byte(s)
- * The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
-
- private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
- private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
- private static final int MIN_BYTE_COUNT = 4096;
-
- public final static String OFFSETS_VECTOR_NAME = "$offsets$";
- private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4));
- private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
- private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
-
- private final Accessor accessor;
- private final Mutator mutator;
-
- private final UInt${type.width}Vector.Accessor oAccessor;
-
- private int allocationSizeInBytes = INITIAL_BYTE_COUNT;
- private int allocationMonitor = 0;
-
- public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- this.oAccessor = offsetVector.getAccessor();
- this.accessor = new Accessor();
- this.mutator = new Mutator();
- }
-
- @Override
- public FieldReader getReader(){
- return reader;
- }
-
- @Override
- public int getBufferSize(){
- if (getAccessor().getValueCount() == 0) {
- return 0;
- }
- return offsetVector.getBufferSize() + data.writerIndex();
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- final int idx = offsetVector.getAccessor().get(valueCount);
- return offsetVector.getBufferSizeFor(valueCount + 1) + idx;
- }
-
- @Override
- public int getValueCapacity(){
- return Math.max(offsetVector.getValueCapacity() - 1, 0);
- }
-
- @Override
- public int getByteCapacity(){
- return data.capacity();
- }
-
- @Override
- public int getCurrentSizeInBytes() {
- return offsetVector.getAccessor().get(getAccessor().getValueCount());
- }
-
- /**
- * Return the number of bytes contained in the current var len byte vector.
- * @return
- */
- public int getVarByteLength(){
- final int valueCount = getAccessor().getValueCount();
- if(valueCount == 0) {
- return 0;
- }
- return offsetVector.getAccessor().get(valueCount);
- }
-
- @Override
- public SerializedField getMetadata() {
- return getMetadataBuilder() //
- .addChild(offsetVector.getMetadata())
- .setValueCount(getAccessor().getValueCount()) //
- .setBufferLength(getBufferSize()) //
- .build();
- }
-
- @Override
- public void load(SerializedField metadata, DrillBuf buffer) {
- // the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
- final SerializedField offsetField = metadata.getChild(0);
- offsetVector.load(offsetField, buffer);
-
- final int capacity = buffer.capacity();
- final int offsetsLength = offsetField.getBufferLength();
- data = buffer.slice(offsetsLength, capacity - offsetsLength);
- data.retain();
- }
-
- @Override
- public void clear() {
- super.clear();
- offsetVector.clear();
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- final DrillBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(false), super.getBuffers(false), DrillBuf.class);
- if (clear) {
- // does not make much sense but we have to retain buffers even when clear is set. refactor this interface.
- for (final DrillBuf buffer:buffers) {
- buffer.retain(1);
- }
- clear();
- }
- return buffers;
- }
-
- public long getOffsetAddr(){
- return offsetVector.getBuffer().memoryAddress();
- }
-
- public UInt${type.width}Vector getOffsetVector(){
- return offsetVector;
- }
-
- @Override
- public TransferPair getTransferPair(){
- return new TransferImpl(getField());
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref){
- return new TransferImpl(getField().withPath(ref));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new TransferImpl((${minor.class}Vector) to);
- }
-
- public void transferTo(${minor.class}Vector target){
- target.clear();
- this.offsetVector.transferTo(target.offsetVector);
- target.data = data;
- target.data.retain(1);
- clear();
- }
-
- public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
- UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor();
- final int startPoint = offsetVectorAccessor.get(startIndex);
- final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint;
- target.clear();
- target.offsetVector.allocateNew(length + 1);
- offsetVectorAccessor = this.offsetVector.getAccessor();
- final UInt4Vector.Mutator targetOffsetVectorMutator = target.offsetVector.getMutator();
- for (int i = 0; i < length + 1; i++) {
- targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
- }
- target.data = data.slice(startPoint, sliceLength);
- target.data.retain(1);
- target.getMutator().setValueCount(length);
-}
-
- protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
- final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
- final int start = fromOffsetVectorAccessor.get(fromIndex);
- final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
- final int len = end - start;
-
- final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
- from.data.getBytes(start, data, outputStart, len);
- offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
- }
-
- public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
- final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
- final int start = fromOffsetVectorAccessor.get(fromIndex);
- final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
- final int len = end - start;
- final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
-
- while(data.capacity() < outputStart + len) {
- reAlloc();
- }
-
- offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len);
- from.data.getBytes(start, data, outputStart, len);
- return true;
- }
-
- private class TransferImpl implements TransferPair{
- ${minor.class}Vector to;
-
- public TransferImpl(MaterializedField field){
- to = new ${minor.class}Vector(field, allocator);
- }
-
- public TransferImpl(${minor.class}Vector to){
- this.to = to;
- }
-
- @Override
- public ${minor.class}Vector getTo(){
- return to;
- }
-
- @Override
- public void transfer(){
- transferTo(to);
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- splitAndTransferTo(startIndex, length, to);
- }
-
- @Override
- public void copyValueSafe(int fromIndex, int toIndex) {
- to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
- }
- }
-
- @Override
- public void setInitialCapacity(final int valueCount) {
- final long size = 1L * valueCount * ${type.width};
- if (size > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
- }
- allocationSizeInBytes = (int)size;
- offsetVector.setInitialCapacity(valueCount + 1);
- }
-
- @Override
- public void allocateNew() {
- if(!allocateNewSafe()){
- throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
- }
- }
-
- @Override
- public boolean allocateNewSafe() {
- long curAllocationSize = allocationSizeInBytes;
- if (allocationMonitor > 10) {
- curAllocationSize = Math.max(MIN_BYTE_COUNT, curAllocationSize / 2);
- allocationMonitor = 0;
- } else if (allocationMonitor < -2) {
- curAllocationSize = curAllocationSize * 2L;
- allocationMonitor = 0;
- }
-
- if (curAllocationSize > MAX_ALLOCATION_SIZE) {
- return false;
- }
-
- clear();
- /* Boolean to keep track if all the memory allocations were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- try {
- final int requestedSize = (int)curAllocationSize;
- data = allocator.buffer(requestedSize);
- allocationSizeInBytes = requestedSize;
- offsetVector.allocateNew();
- } catch (OutOfMemoryRuntimeException e) {
- clear();
- return false;
- }
- data.readerIndex(0);
- offsetVector.zeroVector();
- return true;
- }
-
- @Override
- public void allocateNew(int totalBytes, int valueCount) {
- clear();
- assert totalBytes >= 0;
- try {
- data = allocator.buffer(totalBytes);
- offsetVector.allocateNew(valueCount + 1);
- } catch (DrillRuntimeException e) {
- clear();
- throw e;
- }
- data.readerIndex(0);
- allocationSizeInBytes = totalBytes;
- offsetVector.zeroVector();
- }
-
- @Override
- public void reset() {
- allocationSizeInBytes = INITIAL_BYTE_COUNT;
- allocationMonitor = 0;
- data.readerIndex(0);
- offsetVector.zeroVector();
- super.reset();
- }
-
- public void reAlloc() {
- final long newAllocationSize = allocationSizeInBytes*2L;
- if (newAllocationSize > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
- }
-
- final DrillBuf newBuf = allocator.buffer((int)newAllocationSize);
- newBuf.setBytes(0, data, 0, data.capacity());
- data.release();
- data = newBuf;
- allocationSizeInBytes = (int)newAllocationSize;
- }
-
- public void decrementAllocationMonitor() {
- if (allocationMonitor > 0) {
- allocationMonitor = 0;
- }
- --allocationMonitor;
- }
-
- private void incrementAllocationMonitor() {
- ++allocationMonitor;
- }
-
- @Override
- public Accessor getAccessor(){
- return accessor;
- }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor {
- final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
- public long getStartEnd(int index){
- return oAccessor.getTwoAsLong(index);
- }
-
- public byte[] get(int index) {
- assert index >= 0;
- final int startIdx = oAccessor.get(index);
- final int length = oAccessor.get(index + 1) - startIdx;
- assert length >= 0;
- final byte[] dst = new byte[length];
- data.getBytes(startIdx, dst, 0, length);
- return dst;
- }
-
- @Override
- public int getValueLength(int index) {
- final UInt${type.width}Vector.Accessor offsetVectorAccessor = offsetVector.getAccessor();
- return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index);
- }
-
- public void get(int index, ${minor.class}Holder holder){
- holder.start = oAccessor.get(index);
- holder.end = oAccessor.get(index + 1);
- holder.buffer = data;
- }
-
- public void get(int index, Nullable${minor.class}Holder holder){
- holder.isSet = 1;
- holder.start = oAccessor.get(index);
- holder.end = oAccessor.get(index + 1);
- holder.buffer = data;
- }
-
-
- <#switch minor.class>
- <#case "VarChar">
- @Override
- public ${friendlyType} getObject(int index) {
- Text text = new Text();
- text.set(get(index));
- return text;
- }
- <#break>
- <#case "Var16Char">
- @Override
- public ${friendlyType} getObject(int index) {
- return new String(get(index), Charsets.UTF_16);
- }
- <#break>
- <#default>
- @Override
- public ${friendlyType} getObject(int index) {
- return get(index);
- }
- </#switch>
-
- @Override
- public int getValueCount() {
- return Math.max(offsetVector.getAccessor().getValueCount()-1, 0);
- }
-
- @Override
- public boolean isNull(int index){
- return false;
- }
-
- public UInt${type.width}Vector getOffsetVector(){
- return offsetVector;
- }
- }
-
- /**
- * Mutable${minor.class} implements a vector of variable width values. Elements in the vector
- * are accessed by position from the logical start of the vector. A fixed width offsetVector
- * is used to convert an element's position to it's offset from the start of the (0-based)
- * DrillBuf. Size is inferred by adjacent elements.
- * The width of each element is ${type.width} byte(s)
- * The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public final class Mutator extends BaseValueVector.BaseMutator implements VariableWidthVector.VariableWidthMutator {
-
- /**
- * Set the variable length element at the specified index to the supplied byte array.
- *
- * @param index position of the bit to set
- * @param bytes array of bytes to write
- */
- protected void set(int index, byte[] bytes) {
- assert index >= 0;
- final int currentOffset = offsetVector.getAccessor().get(index);
- offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
- data.setBytes(currentOffset, bytes, 0, bytes.length);
- }
-
- public void setSafe(int index, byte[] bytes) {
- assert index >= 0;
-
- final int currentOffset = offsetVector.getAccessor().get(index);
- while (data.capacity() < currentOffset + bytes.length) {
- reAlloc();
- }
- offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length);
- data.setBytes(currentOffset, bytes, 0, bytes.length);
- }
-
- /**
- * Set the variable length element at the specified index to the supplied byte array.
- *
- * @param index position of the bit to set
- * @param bytes array of bytes to write
- * @param start start index of bytes to write
- * @param length length of bytes to write
- */
- protected void set(int index, byte[] bytes, int start, int length) {
- assert index >= 0;
- final int currentOffset = offsetVector.getAccessor().get(index);
- offsetVector.getMutator().set(index + 1, currentOffset + length);
- data.setBytes(currentOffset, bytes, start, length);
- }
-
- public void setSafe(int index, ByteBuffer bytes, int start, int length) {
- assert index >= 0;
-
- int currentOffset = offsetVector.getAccessor().get(index);
-
- while (data.capacity() < currentOffset + length) {
- reAlloc();
- }
- offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
- data.setBytes(currentOffset, bytes, start, length);
- }
-
- public void setSafe(int index, byte[] bytes, int start, int length) {
- assert index >= 0;
-
- final int currentOffset = offsetVector.getAccessor().get(index);
-
- while (data.capacity() < currentOffset + length) {
- reAlloc();
- }
- offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
- data.setBytes(currentOffset, bytes, start, length);
- }
-
- @Override
- public void setValueLengthSafe(int index, int length) {
- final int offset = offsetVector.getAccessor().get(index);
- while(data.capacity() < offset + length ) {
- reAlloc();
- }
- offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length);
- }
-
-
- public void setSafe(int index, int start, int end, DrillBuf buffer){
- final int len = end - start;
- final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-
- while(data.capacity() < outputStart + len) {
- reAlloc();
- }
-
- offsetVector.getMutator().setSafe( index+1, outputStart + len);
- buffer.getBytes(start, data, outputStart, len);
- }
-
- public void setSafe(int index, Nullable${minor.class}Holder holder){
- assert holder.isSet == 1;
-
- final int start = holder.start;
- final int end = holder.end;
- final int len = end - start;
-
- int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-
- while(data.capacity() < outputStart + len) {
- reAlloc();
- }
-
- holder.buffer.getBytes(start, data, outputStart, len);
- offsetVector.getMutator().setSafe( index+1, outputStart + len);
- }
-
- public void setSafe(int index, ${minor.class}Holder holder){
- final int start = holder.start;
- final int end = holder.end;
- final int len = end - start;
- final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-
- while(data.capacity() < outputStart + len) {
- reAlloc();
- }
-
- holder.buffer.getBytes(start, data, outputStart, len);
- offsetVector.getMutator().setSafe( index+1, outputStart + len);
- }
-
- protected void set(int index, int start, int length, DrillBuf buffer){
- assert index >= 0;
- final int currentOffset = offsetVector.getAccessor().get(index);
- offsetVector.getMutator().set(index + 1, currentOffset + length);
- final DrillBuf bb = buffer.slice(start, length);
- data.setBytes(currentOffset, bb);
- }
-
- protected void set(int index, Nullable${minor.class}Holder holder){
- final int length = holder.end - holder.start;
- final int currentOffset = offsetVector.getAccessor().get(index);
- offsetVector.getMutator().set(index + 1, currentOffset + length);
- data.setBytes(currentOffset, holder.buffer, holder.start, length);
- }
-
- protected void set(int index, ${minor.class}Holder holder){
- final int length = holder.end - holder.start;
- final int currentOffset = offsetVector.getAccessor().get(index);
- offsetVector.getMutator().set(index + 1, currentOffset + length);
- data.setBytes(currentOffset, holder.buffer, holder.start, length);
- }
-
- @Override
- public void setValueCount(int valueCount) {
- final int currentByteCapacity = getByteCapacity();
- final int idx = offsetVector.getAccessor().get(valueCount);
- data.writerIndex(idx);
- if (valueCount > 0 && currentByteCapacity > idx * 2) {
- incrementAllocationMonitor();
- } else if (allocationMonitor > 0) {
- allocationMonitor = 0;
- }
- VectorTrimmer.trim(data, idx);
- offsetVector.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
- }
-
- @Override
- public void generateTestData(int size){
- boolean even = true;
- <#switch minor.class>
- <#case "Var16Char">
- final java.nio.charset.Charset charset = Charsets.UTF_16;
- <#break>
- <#case "VarChar">
- <#default>
- final java.nio.charset.Charset charset = Charsets.UTF_8;
- </#switch>
- final byte[] evenValue = new String("aaaaa").getBytes(charset);
- final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset);
- for(int i =0; i < size; i++, even = !even){
- set(i, even ? evenValue : oddValue);
- }
- setValueCount(size);
- }
- }
-}
-
-</#if> <#-- type.major -->
-</#list>
-</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java b/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
deleted file mode 100644
index 5e437b2..0000000
--- a/exec/java-exec/src/main/java/io/netty/buffer/DrillBuf.java
+++ /dev/null
@@ -1,763 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import io.netty.util.internal.PlatformDependent;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-import java.nio.charset.Charset;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.drill.exec.memory.Accountor;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.ops.BufferManager;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.ops.OperatorContext;
-import org.apache.drill.exec.util.AssertionUtil;
-
-import com.google.common.base.Preconditions;
-
-public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillBuf.class);
-
- private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
-
- private final ByteBuf b;
- private final long addr;
- private final int offset;
- private final boolean rootBuffer;
- private final AtomicLong rootRefCnt = new AtomicLong(1);
- private volatile BufferAllocator allocator;
- private volatile Accountor acct;
- private volatile int length;
-
- // TODO - cleanup
- // The code is partly shared and partly copy-pasted between
- // these three types. They should be unified under one interface
- // to share code and to remove the hacky code here to use only
- // one of these types at a time and use null checks to find out
- // which.
- private OperatorContext context;
- private FragmentContext fContext;
- private BufferManager bufManager;
-
- public DrillBuf(BufferAllocator allocator, Accountor a, UnsafeDirectLittleEndian b) {
- super(b.maxCapacity());
- this.b = b;
- this.addr = b.memoryAddress();
- this.acct = a;
- this.length = b.capacity();
- this.offset = 0;
- this.rootBuffer = true;
- this.allocator = allocator;
- }
-
- private DrillBuf(BufferAllocator allocator, Accountor a) {
- super(0);
- this.b = new EmptyByteBuf(allocator.getUnderlyingAllocator()).order(ByteOrder.LITTLE_ENDIAN);
- this.allocator = allocator;
- this.acct = a;
- this.length = 0;
- this.addr = 0;
- this.rootBuffer = false;
- this.offset = 0;
- }
-
- /**
- * Special constructor used for RPC ownership transfer. Takes a snapshot slice of the current buf
- * but points directly to the underlying UnsafeLittleEndian buffer. Does this by calling unwrap()
- * twice on the provided DrillBuf and expecting an UnsafeDirectLittleEndian buffer. This operation
- * includes taking a new reference count on the underlying buffer and maintaining returning with a
- * current reference count for itself (masking the underlying reference count).
- * @param allocator
- * @param a Allocator used when users try to receive allocator from buffer.
- * @param b Accountor used for accounting purposes.
- */
- public DrillBuf(BufferAllocator allocator, Accountor a, DrillBuf b) {
- this(allocator, a, getUnderlying(b), b, 0, b.length, true);
- assert b.unwrap().unwrap() instanceof UnsafeDirectLittleEndian;
- b.unwrap().unwrap().retain();
- }
-
-
- private DrillBuf(DrillBuf buffer, int index, int length) {
- this(buffer.allocator, null, buffer, buffer, index, length, false);
- }
-
- private static ByteBuf getUnderlying(DrillBuf b){
- ByteBuf underlying = b.unwrap().unwrap();
- return underlying.slice((int) (b.memoryAddress() - underlying.memoryAddress()), b.length);
- }
- private DrillBuf(BufferAllocator allocator, Accountor a, ByteBuf replacement, DrillBuf buffer, int index, int length, boolean root) {
- super(length);
- if (index < 0 || index > buffer.capacity() - length) {
- throw new IndexOutOfBoundsException(buffer.toString() + ".slice(" + index + ", " + length + ')');
- }
-
- this.length = length;
- writerIndex(length);
-
- this.b = replacement;
- this.addr = buffer.memoryAddress() + index;
- this.offset = index;
- this.acct = a;
- this.length = length;
- this.rootBuffer = root;
- this.allocator = allocator;
- }
-
- public void setOperatorContext(OperatorContext c) {
- this.context = c;
- }
- public void setFragmentContext(FragmentContext c) {
- this.fContext = c;
- }
-
- public void setBufferManager(BufferManager bufManager) {
- this.bufManager = bufManager;
- }
-
- public BufferAllocator getAllocator() {
- return allocator;
- }
-
- public DrillBuf reallocIfNeeded(int size) {
- if (this.capacity() >= size) {
- return this;
- }
- if (context != null) {
- return context.replace(this, size);
- } else if(fContext != null) {
- return fContext.replace(this, size);
- } else if (bufManager != null) {
- return bufManager.replace(this, size);
- } else {
- throw new UnsupportedOperationException("Realloc is only available in the context of an operator's UDFs");
- }
-
- }
-
- @Override
- public int refCnt() {
- if(rootBuffer){
- return (int) this.rootRefCnt.get();
- }else{
- return b.refCnt();
- }
-
- }
-
- private long addr(int index) {
- return addr + index;
- }
-
- private final void checkIndexD(int index) {
- ensureAccessible();
- if (index < 0 || index >= capacity()) {
- throw new IndexOutOfBoundsException(String.format(
- "index: %d (expected: range(0, %d))", index, capacity()));
- }
- }
-
- private final void checkIndexD(int index, int fieldLength) {
- ensureAccessible();
- if (fieldLength < 0) {
- throw new IllegalArgumentException("length: " + fieldLength + " (expected: >= 0)");
- }
- if (index < 0 || index > capacity() - fieldLength) {
- throw new IndexOutOfBoundsException(String.format(
- "index: %d, length: %d (expected: range(0, %d))", index, fieldLength, capacity()));
- }
- }
-
- /**
- * Allows a function to determine whether not reading a particular string of bytes is valid.
- *
- * Will throw an exception if the memory is not readable for some reason. Only doesn't something in the
- * case that AssertionUtil.BOUNDS_CHECKING_ENABLED is true.
- *
- * @param start The starting position of the bytes to be read.
- * @param end The exclusive endpoint of the bytes to be read.
- */
- public void checkBytes(int start, int end){
- if (BOUNDS_CHECKING_ENABLED) {
- checkIndexD(start, end - start);
- }
- }
-
- private void chk(int index, int width) {
- if (BOUNDS_CHECKING_ENABLED) {
- checkIndexD(index, width);
- }
- }
-
- private void chk(int index) {
- if (BOUNDS_CHECKING_ENABLED) {
- checkIndexD(index);
- }
- }
-
- private void ensure(int width) {
- if (BOUNDS_CHECKING_ENABLED) {
- ensureWritable(width);
- }
- }
-
- public boolean transferAccounting(Accountor target) {
- if (rootBuffer) {
- boolean outcome = acct.transferTo(target, this, length);
- acct = target;
- return outcome;
- } else {
- throw new UnsupportedOperationException();
- }
- }
-
- @Override
- public synchronized boolean release() {
- return release(1);
- }
-
- /**
- * Release the provided number of reference counts. If this is a root buffer, will decrease accounting if the local reference count returns to zero.
- */
- @Override
- public synchronized boolean release(int decrement) {
-
- if(rootBuffer){
- final long newRefCnt = this.rootRefCnt.addAndGet(-decrement);
- Preconditions.checkArgument(newRefCnt > -1, "Buffer has negative reference count.");
- if (newRefCnt == 0) {
- b.release(decrement);
- acct.release(this, length);
- return true;
- }else{
- return false;
- }
- }else{
- return b.release(decrement);
- }
- }
-
- @Override
- public int capacity() {
- return length;
- }
-
- @Override
- public synchronized ByteBuf capacity(int newCapacity) {
- if (rootBuffer) {
- if (newCapacity == length) {
- return this;
- } else if (newCapacity < length) {
- b.capacity(newCapacity);
- int diff = length - b.capacity();
- acct.releasePartial(this, diff);
- this.length = length - diff;
- return this;
- } else {
- throw new UnsupportedOperationException("Accounting byte buf doesn't support increasing allocations.");
- }
- } else {
- throw new UnsupportedOperationException("Non root bufs doen't support changing allocations.");
- }
- }
-
- @Override
- public int maxCapacity() {
- return length;
- }
-
- @Override
- public ByteBufAllocator alloc() {
- return b.alloc();
- }
-
- @Override
- public ByteOrder order() {
- return ByteOrder.LITTLE_ENDIAN;
- }
-
- @Override
- public ByteBuf order(ByteOrder endianness) {
- // if(endianness != ByteOrder.LITTLE_ENDIAN) throw new
- // UnsupportedOperationException("Drill buffers only support little endian.");
- return this;
- }
-
- @Override
- public ByteBuf unwrap() {
- return b;
- }
-
- @Override
- public boolean isDirect() {
- return true;
- }
-
- @Override
- public ByteBuf readBytes(int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf readSlice(int length) {
- ByteBuf slice = slice(readerIndex(), length);
- readerIndex(readerIndex() + length);
- return slice;
- }
-
- @Override
- public ByteBuf copy() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf copy(int index, int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBuf slice() {
- return slice(readerIndex(), readableBytes());
- }
-
- @Override
- public DrillBuf slice(int index, int length) {
- DrillBuf buf = new DrillBuf(this, index, length);
- buf.writerIndex = length;
- return buf;
- }
-
- @Override
- public DrillBuf duplicate() {
- return new DrillBuf(this, 0, length);
- }
-
- @Override
- public int nioBufferCount() {
- return 1;
- }
-
- @Override
- public ByteBuffer nioBuffer() {
- return nioBuffer(readerIndex(), readableBytes());
- }
-
- @Override
- public ByteBuffer nioBuffer(int index, int length) {
- return b.nioBuffer(offset + index, length);
- }
-
- @Override
- public ByteBuffer internalNioBuffer(int index, int length) {
- return b.internalNioBuffer(offset + index, length);
- }
-
- @Override
- public ByteBuffer[] nioBuffers() {
- return new ByteBuffer[]{nioBuffer()};
- }
-
- @Override
- public ByteBuffer[] nioBuffers(int index, int length) {
- return new ByteBuffer[]{nioBuffer(index, length)};
- }
-
- @Override
- public boolean hasArray() {
- return b.hasArray();
- }
-
- @Override
- public byte[] array() {
- return b.array();
- }
-
- @Override
- public int arrayOffset() {
- return b.arrayOffset();
- }
-
- @Override
- public boolean hasMemoryAddress() {
- return true;
- }
-
- @Override
- public long memoryAddress() {
- return this.addr;
- }
-
- @Override
- public String toString(Charset charset) {
- return toString(readerIndex, readableBytes(), charset);
- }
-
- @Override
- public String toString(int index, int length, Charset charset) {
- if (length == 0) {
- return "";
- }
-
- ByteBuffer nioBuffer;
- if (nioBufferCount() == 1) {
- nioBuffer = nioBuffer(index, length);
- } else {
- nioBuffer = ByteBuffer.allocate(length);
- getBytes(index, nioBuffer);
- nioBuffer.flip();
- }
-
- return ByteBufUtil.decodeString(nioBuffer, charset);
- }
-
- @Override
- public int hashCode() {
- return System.identityHashCode(this);
- }
-
- @Override
- public boolean equals(Object obj) {
- // identity equals only.
- return this == obj;
- }
-
- @Override
- public synchronized ByteBuf retain(int increment) {
- if(rootBuffer){
- this.rootRefCnt.addAndGet(increment);
- }else{
- b.retain(increment);
- }
- return this;
- }
-
- @Override
- public ByteBuf retain() {
- return retain(1);
- }
-
- @Override
- public long getLong(int index) {
- chk(index, 8);
- long v = PlatformDependent.getLong(addr(index));
- return v;
- }
-
- @Override
- public float getFloat(int index) {
- return Float.intBitsToFloat(getInt(index));
- }
-
- @Override
- public double getDouble(int index) {
- return Double.longBitsToDouble(getLong(index));
- }
-
- @Override
- public char getChar(int index) {
- return (char) getShort(index);
- }
-
- @Override
- public long getUnsignedInt(int index) {
- return getInt(index) & 0xFFFFFFFFL;
- }
-
- @Override
- public int getInt(int index) {
- chk(index, 4);
- int v = PlatformDependent.getInt(addr(index));
- return v;
- }
-
- @Override
- public int getUnsignedShort(int index) {
- return getShort(index) & 0xFFFF;
- }
-
- @Override
- public short getShort(int index) {
- chk(index, 2);
- short v = PlatformDependent.getShort(addr(index));
- return v;
- }
-
- @Override
- public ByteBuf setShort(int index, int value) {
- chk(index, 2);
- PlatformDependent.putShort(addr(index), (short) value);
- return this;
- }
-
- @Override
- public ByteBuf setInt(int index, int value) {
- chk(index, 4);
- PlatformDependent.putInt(addr(index), value);
- return this;
- }
-
- @Override
- public ByteBuf setLong(int index, long value) {
- chk(index, 8);
- PlatformDependent.putLong(addr(index), value);
- return this;
- }
-
- @Override
- public ByteBuf setChar(int index, int value) {
- chk(index, 2);
- PlatformDependent.putShort(addr(index), (short) value);
- return this;
- }
-
- @Override
- public ByteBuf setFloat(int index, float value) {
- chk(index, 4);
- PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value));
- return this;
- }
-
- @Override
- public ByteBuf setDouble(int index, double value) {
- chk(index, 8);
- PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value));
- return this;
- }
-
- @Override
- public ByteBuf writeShort(int value) {
- ensure(2);
- PlatformDependent.putShort(addr(writerIndex), (short) value);
- writerIndex += 2;
- return this;
- }
-
- @Override
- public ByteBuf writeInt(int value) {
- ensure(4);
- PlatformDependent.putInt(addr(writerIndex), value);
- writerIndex += 4;
- return this;
- }
-
- @Override
- public ByteBuf writeLong(long value) {
- ensure(8);
- PlatformDependent.putLong(addr(writerIndex), value);
- writerIndex += 8;
- return this;
- }
-
- @Override
- public ByteBuf writeChar(int value) {
- ensure(2);
- PlatformDependent.putShort(addr(writerIndex), (short) value);
- writerIndex += 2;
- return this;
- }
-
- @Override
- public ByteBuf writeFloat(float value) {
- ensure(4);
- PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value));
- writerIndex += 4;
- return this;
- }
-
- @Override
- public ByteBuf writeDouble(double value) {
- ensure(8);
- PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value));
- writerIndex += 8;
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
- b.getBytes(index + offset, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuffer dst) {
- b.getBytes(index + offset, dst);
- return this;
- }
-
- @Override
- public ByteBuf setByte(int index, int value) {
- chk(index, 1);
- PlatformDependent.putByte(addr(index), (byte) value);
- return this;
- }
-
- public void setByte(int index, byte b){
- chk(index, 1);
- PlatformDependent.putByte(addr(index), b);
- }
-
- public void writeByteUnsafe(byte b){
- PlatformDependent.putByte(addr(readerIndex), b);
- readerIndex++;
- }
-
- @Override
- protected byte _getByte(int index) {
- return getByte(index);
- }
-
- @Override
- protected short _getShort(int index) {
- return getShort(index);
- }
-
- @Override
- protected int _getInt(int index) {
- return getInt(index);
- }
-
- @Override
- protected long _getLong(int index) {
- return getLong(index);
- }
-
- @Override
- protected void _setByte(int index, int value) {
- setByte(index, value);
- }
-
- @Override
- protected void _setShort(int index, int value) {
- setShort(index, value);
- }
-
- @Override
- protected void _setMedium(int index, int value) {
- setMedium(index, value);
- }
-
- @Override
- protected void _setInt(int index, int value) {
- setInt(index, value);
- }
-
- @Override
- protected void _setLong(int index, long value) {
- setLong(index, value);
- }
-
- @Override
- public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
- b.getBytes(index + offset, dst, dstIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
- b.getBytes(index + offset, out, length);
- return this;
- }
-
- @Override
- protected int _getUnsignedMedium(int index) {
- long addr = addr(index);
- return (PlatformDependent.getByte(addr) & 0xff) << 16 |
- (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
- PlatformDependent.getByte(addr + 2) & 0xff;
- }
-
- @Override
- public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
- return b.getBytes(index + offset, out, length);
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
- b.setBytes(index + offset, src, srcIndex, length);
- return this;
- }
-
- public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
- if (src.isDirect()) {
- checkIndex(index, length);
- PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this.memoryAddress() + index,
- length);
- } else {
- if (srcIndex == 0 && src.capacity() == length) {
- b.setBytes(index + offset, src);
- } else {
- ByteBuffer newBuf = src.duplicate();
- newBuf.position(srcIndex);
- newBuf.limit(srcIndex + length);
- b.setBytes(index + offset, src);
- }
- }
-
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
- b.setBytes(index + offset, src, srcIndex, length);
- return this;
- }
-
- @Override
- public ByteBuf setBytes(int index, ByteBuffer src) {
- b.setBytes(index + offset, src);
- return this;
- }
-
- @Override
- public int setBytes(int index, InputStream in, int length) throws IOException {
- return b.setBytes(index + offset, in, length);
- }
-
- @Override
- public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
- return b.setBytes(index + offset, in, length);
- }
-
- @Override
- public byte getByte(int index) {
- chk(index, 1);
- return PlatformDependent.getByte(addr(index));
- }
-
- public static DrillBuf getEmpty(BufferAllocator allocator, Accountor a) {
- return new DrillBuf(allocator, a);
- }
-
- public boolean isRootBuffer() {
- return rootBuffer;
- }
-
- @Override
- public void close() {
- release();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java b/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
deleted file mode 100644
index 721aff9..0000000
--- a/exec/java-exec/src/main/java/io/netty/buffer/FakeAllocator.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.netty.buffer;
-
-import org.apache.drill.exec.memory.Accountor;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.util.Pointer;
-
-class FakeAllocator implements BufferAllocator {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FakeAllocator.class);
-
-
- public static final Accountor FAKE_ACCOUNTOR = new FakeAccountor();
- public static final BufferAllocator FAKE_ALLOCATOR = new FakeAllocator();
-
- @Override
- public DrillBuf buffer(int size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DrillBuf buffer(int minSize, int maxSize) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ByteBufAllocator getUnderlyingAllocator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public BufferAllocator getChildAllocator(FragmentContext context, long initialReservation, long maximumReservation,
- boolean applyFragmentLimit)
- throws OutOfMemoryException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DrillBuf getEmpty() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean takeOwnership(DrillBuf buf) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public PreAllocator getNewPreAllocator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void resetFragmentLimits() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setFragmentLimit(long l) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getFragmentLimit(){
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public long getAllocatedMemory() {
- return 0;
- }
-
- @Override
- public long getPeakMemoryAllocation() {
- return 0;
- }
-
- static class FakeAccountor extends Accountor {
-
- public FakeAccountor() {
- super(null, false, null, null, 0, 0, true);
- }
-
- @Override
- public boolean transferTo(Accountor target, DrillBuf buf, long size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getAvailable() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getCapacity() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getAllocation() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean reserve(long size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean forceAdditionalReservation(long size) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void reserved(long expected, DrillBuf buf) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void releasePartial(DrillBuf buf, long size) {
-
- }
-
- @Override
- public void release(DrillBuf buf, long size) {
-
- }
-
- @Override
- public void close() {
-
- }
-
-
- }
-
- @Override
- public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut) {
- throw new UnsupportedOperationException();
- }
-
-}
[19/45] drill git commit: DRILL-3987: (REFACTOR) Common and Vector
modules building.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
new file mode 100644
index 0000000..4a72719
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
@@ -0,0 +1,427 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.util.AssertionUtil;
+
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+public class AccountorImpl implements Accountor {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AccountorImpl.class);
+
+ private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
+ private final AtomicRemainder remainder;
+ private final long total;
+ private ConcurrentMap<ByteBuf, DebugStackTrace> buffers = Maps.newConcurrentMap();
+ private AccountorImpl parent;
+
+ private final boolean errorOnLeak;
+ // some operators are no subject to the fragment limit. They set the applyFragmentLimit to false
+
+ private final boolean enableFragmentLimit;
+ private final double fragmentMemOvercommitFactor;
+
+ private final boolean DEFAULT_ENABLE_FRAGMENT_LIMIT=false;
+ private final double DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR=1.5;
+
+ private final boolean applyFragmentLimit;
+
+ private final LimitConsumer limitConsumer;
+ long fragmentLimit;
+
+ private long peakMemoryAllocation = 0;
+
+ // The top level Allocator has an accountor that keeps track of all the LimitConsumers currently executing.
+ // This enables the top level accountor to calculate a new fragment limit whenever necessary.
+ private final List<LimitConsumer> limitConsumers;
+
+ public AccountorImpl(DrillConfig config, boolean errorOnLeak, LimitConsumer context, AccountorImpl parent, long max,
+ long preAllocated, boolean applyFragLimit) {
+ // TODO: fix preallocation stuff
+ this.errorOnLeak = errorOnLeak;
+ AtomicRemainder parentRemainder = parent != null ? parent.remainder : null;
+ this.parent = parent;
+
+ boolean enableFragmentLimit;
+ double fragmentMemOvercommitFactor;
+
+ try {
+ enableFragmentLimit = config.getBoolean(ExecConstants.ENABLE_FRAGMENT_MEMORY_LIMIT);
+ fragmentMemOvercommitFactor = config.getDouble(ExecConstants.FRAGMENT_MEM_OVERCOMMIT_FACTOR);
+ }catch(Exception e){
+ enableFragmentLimit = DEFAULT_ENABLE_FRAGMENT_LIMIT;
+ fragmentMemOvercommitFactor = DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR;
+ }
+ this.enableFragmentLimit = enableFragmentLimit;
+ this.fragmentMemOvercommitFactor = fragmentMemOvercommitFactor;
+
+
+ this.applyFragmentLimit=applyFragLimit;
+
+ this.remainder = new AtomicRemainder(errorOnLeak, parentRemainder, max, preAllocated, applyFragmentLimit);
+ this.total = max;
+ this.limitConsumer = context;
+ this.fragmentLimit=this.total; // Allow as much as possible to start with;
+ if (ENABLE_ACCOUNTING) {
+ buffers = Maps.newConcurrentMap();
+ } else {
+ buffers = null;
+ }
+ this.limitConsumers = new ArrayList<LimitConsumer>();
+ if(parent!=null && parent.parent==null){ // Only add the fragment context to the fragment level accountor
+ synchronized(this) {
+ addLimitConsumer(this.limitConsumer);
+ }
+ }
+ }
+
+ public boolean transferTo(Accountor target, DrillBuf buf, long size) {
+ return transfer(target, buf, size, true);
+ }
+
+ public boolean transferIn(DrillBuf buf, long size) {
+ return transfer(this, buf, size, false);
+ }
+
+ private boolean transfer(Accountor target, DrillBuf buf, long size, boolean release) {
+ boolean withinLimit = target.forceAdditionalReservation(size);
+ if(release){
+ release(buf, size);
+ }
+
+ if (ENABLE_ACCOUNTING) {
+ if (target instanceof AccountorImpl) {
+ ((AccountorImpl) target).buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread()
+ .getStackTrace()));
+ }
+ }
+ return withinLimit;
+ }
+
+ public long getAvailable() {
+ if (parent != null) {
+ return Math.min(parent.getAvailable(), getCapacity() - getAllocation());
+ }
+ return getCapacity() - getAllocation();
+ }
+
+ public long getCapacity() {
+ return fragmentLimit;
+ }
+
+ public long getAllocation() {
+ return remainder.getUsed();
+ }
+
+ public long getPeakMemoryAllocation() {
+ return peakMemoryAllocation;
+ }
+
+ public boolean reserve(long size) {
+ boolean status = remainder.get(size, this.applyFragmentLimit);
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+ return status;
+ }
+
+ public boolean forceAdditionalReservation(long size) {
+ if (size > 0) {
+ boolean status = remainder.forceGet(size);
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+ return status;
+ } else {
+ return true;
+ }
+ }
+
+ public void reserved(long expected, DrillBuf buf) {
+ // make sure to take away the additional memory that happened due to rounding.
+
+ long additional = buf.capacity() - expected;
+ if (additional > 0) {
+ remainder.forceGet(additional);
+ }
+
+ if (ENABLE_ACCOUNTING) {
+ buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread().getStackTrace()));
+ }
+
+ peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
+ }
+
+
+ public void releasePartial(DrillBuf buf, long size) {
+ remainder.returnAllocation(size);
+ if (ENABLE_ACCOUNTING) {
+ if (buf != null) {
+ DebugStackTrace dst = buffers.get(buf);
+ if (dst == null) {
+ throw new IllegalStateException("Partially releasing a buffer that has already been released. Buffer: " + buf);
+ }
+ dst.size -= size;
+ if (dst.size < 0) {
+ throw new IllegalStateException("Partially releasing a buffer that has already been released. Buffer: " + buf);
+ }
+ }
+ }
+ }
+
+ void release(long size) {
+ remainder.returnAllocation(size);
+ }
+
+ public void release(DrillBuf buf, long size) {
+ remainder.returnAllocation(size);
+ if (ENABLE_ACCOUNTING) {
+ if (buf != null && buffers.remove(buf) == null) {
+ throw new IllegalStateException("Releasing a buffer that has already been released. Buffer: " + buf);
+ }
+ }
+ }
+
+ private void addLimitConsumer(LimitConsumer c) {
+ if (parent != null){
+ parent.addLimitConsumer(c);
+ }else {
+ if(logger.isTraceEnabled()) {
+ String fragStr = c == null ? "[Null Context]" : c.getIdentifier();
+ fragStr+=" (Object Id: "+System.identityHashCode(c)+")";
+ StackTraceElement[] ste = (new Throwable()).getStackTrace();
+ StringBuffer sb = new StringBuffer();
+ for (StackTraceElement s : ste) {
+ sb.append(s.toString());
+ sb.append("\n");
+ }
+
+ logger.trace("Fragment " + fragStr + " added to root accountor.\n"+sb.toString());
+ }
+ synchronized(this) {
+ limitConsumers.add(c);
+ }
+ }
+ }
+
+ private void removeLimitConsumer(LimitConsumer c) {
+ if (parent != null){
+ if (parent.parent==null){
+ // only fragment level allocators will have the fragment context saved
+ parent.removeLimitConsumer(c);
+ }
+ }else{
+ if(logger.isDebugEnabled()) {
+ String fragStr = c == null ? "[Null Context]" : c.getIdentifier();
+ fragStr += " (Object Id: " + System.identityHashCode(c) + ")";
+ logger.trace("Fragment " + fragStr + " removed from root accountor");
+ }
+ synchronized(this) {
+ limitConsumers.remove(c);
+ }
+ }
+ }
+
+ public long resetFragmentLimits(){
+ // returns the new capacity
+ if(!this.enableFragmentLimit){
+ return getCapacity();
+ }
+
+ if(parent!=null){
+ parent.resetFragmentLimits();
+ }else {
+ //Get remaining memory available per fragment and distribute it EQUALLY among all the fragments.
+ //Fragments get the memory limit added to the amount already allocated.
+ //This favours fragments that are already running which will get a limit greater than newly started fragments.
+ //If the already running fragments end quickly, their limits will be assigned back to the remaining fragments
+ //quickly. If they are long running, then we want to favour them with larger limits anyway.
+ synchronized (this) {
+ int nFragments = limitConsumers.size();
+ long allocatedMemory=0;
+ for (LimitConsumer fragment : limitConsumers) {
+ allocatedMemory += fragment.getAllocated();
+ }
+ if(logger.isTraceEnabled()) {
+ logger.trace("Resetting Fragment Memory Limit: total Available memory== "+total
+ +" Total Allocated Memory :"+allocatedMemory
+ +" Number of fragments: "+nFragments
+ + " fragmentMemOvercommitFactor: "+fragmentMemOvercommitFactor
+ + " Root fragment limit: "+this.fragmentLimit + "(Root obj: "+System.identityHashCode(this)+")"
+ );
+ }
+ if(nFragments>0) {
+ long rem = (total - allocatedMemory) / nFragments;
+ for (LimitConsumer fragment : limitConsumers) {
+ fragment.setLimit((long) (rem * fragmentMemOvercommitFactor));
+ }
+ }
+ if(logger.isTraceEnabled() && false){
+ StringBuffer sb= new StringBuffer();
+ sb.append("[root](0:0)");
+ sb.append("Allocated memory: ");
+ sb.append(this.getAllocation());
+ sb.append(" Fragment Limit: ");
+ sb.append(this.getFragmentLimit());
+ logger.trace(sb.toString());
+ for (LimitConsumer fragment : limitConsumers) {
+ sb= new StringBuffer();
+ sb.append('[');
+ sb.append(fragment.getIdentifier());
+ sb.append(']');
+ sb.append("Allocated memory: ");
+ sb.append(fragment.getAllocated());
+ sb.append(" Fragment Limit: ");
+ sb.append(fragment.getLimit());
+ logger.trace(sb.toString());
+ }
+ logger.trace("Resetting Complete");
+ }
+ }
+ }
+ return getCapacity();
+ }
+
+ public void close() {
+ // remove the fragment context and reset fragment limits whenever an allocator closes
+ if(parent!=null && parent.parent==null) {
+
+ logger.debug("Fragment " + limitConsumer.getIdentifier() + " accountor being closed");
+ removeLimitConsumer(limitConsumer);
+ }
+ resetFragmentLimits();
+
+ if (ENABLE_ACCOUNTING && !buffers.isEmpty()) {
+ StringBuffer sb = new StringBuffer();
+ sb.append("Attempted to close accountor with ");
+ sb.append(buffers.size());
+ sb.append(" buffer(s) still allocated for ");
+ sb.append(limitConsumer.getIdentifier());
+ sb.append(".\n");
+
+ Multimap<DebugStackTrace, DebugStackTrace> multi = LinkedListMultimap.create();
+ for (DebugStackTrace t : buffers.values()) {
+ multi.put(t, t);
+ }
+
+ for (DebugStackTrace entry : multi.keySet()) {
+ Collection<DebugStackTrace> allocs = multi.get(entry);
+
+ sb.append("\n\n\tTotal ");
+ sb.append(allocs.size());
+ sb.append(" allocation(s) of byte size(s): ");
+ for (DebugStackTrace alloc : allocs) {
+ sb.append(alloc.size);
+ sb.append(", ");
+ }
+
+ sb.append("at stack location:\n");
+ entry.addToString(sb);
+ }
+ if (!buffers.isEmpty()) {
+ IllegalStateException e = new IllegalStateException(sb.toString());
+ if (errorOnLeak) {
+ throw e;
+ } else {
+ logger.warn("Memory leaked.", e);
+ }
+ }
+ }
+
+ remainder.close();
+
+ }
+
+ public void setFragmentLimit(long add) {
+ // We ADD the limit to the current allocation. If none has been allocated, this
+ // sets a new limit. If memory has already been allocated, the fragment gets its
+ // limit based on the allocation, though this might still result in reducing the
+ // limit.
+
+ if (parent != null && parent.parent==null) { // This is a fragment level accountor
+ this.fragmentLimit=getAllocation()+add;
+ this.remainder.setLimit(this.fragmentLimit);
+ logger.trace("Fragment " + limitConsumer.getIdentifier() + " memory limit set to " + this.fragmentLimit);
+ }
+ }
+
+ public long getFragmentLimit(){
+ return this.fragmentLimit;
+ }
+
+ public class DebugStackTrace {
+
+ private StackTraceElement[] elements;
+ private long size;
+
+ public DebugStackTrace(long size, StackTraceElement[] elements) {
+ super();
+ this.elements = elements;
+ this.size = size;
+ }
+
+ public void addToString(StringBuffer sb) {
+ for (int i = 3; i < elements.length; i++) {
+ sb.append("\t\t");
+ sb.append(elements[i]);
+ sb.append("\n");
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(elements);
+// result = prime * result + (int) (size ^ (size >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ DebugStackTrace other = (DebugStackTrace) obj;
+ if (!Arrays.equals(elements, other.elements)) {
+ return false;
+ }
+ // weird equal where size doesn't matter for multimap purposes.
+// if (size != other.size)
+// return false;
+ return true;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryException.java
deleted file mode 100644
index 063f1c1..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import org.apache.drill.exec.exception.FragmentSetupException;
-
-public class OutOfMemoryException extends FragmentSetupException{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutOfMemoryException.class);
-
- public OutOfMemoryException() {
- super();
-
- }
-
- public OutOfMemoryException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
-
- }
-
- public OutOfMemoryException(String message, Throwable cause) {
- super(message, cause);
-
- }
-
- public OutOfMemoryException(String message) {
- super(message);
-
- }
-
- public OutOfMemoryException(Throwable cause) {
- super(cause);
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index 54d7d70..40f8e2b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -29,8 +29,7 @@ import java.util.Map.Entry;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.testing.ControlsInjector;
import org.apache.drill.exec.testing.ControlsInjectorFactory;
import org.apache.drill.exec.util.AssertionUtil;
@@ -46,7 +45,7 @@ public class TopLevelAllocator implements BufferAllocator {
private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
private final Map<ChildAllocator, StackTraceElement[]> childrenMap;
private final PooledByteBufAllocatorL innerAllocator = PooledByteBufAllocatorL.DEFAULT;
- private final Accountor acct;
+ private final AccountorImpl acct;
private final boolean errorOnLeak;
private final DrillBuf empty;
private final DrillConfig config;
@@ -55,7 +54,7 @@ public class TopLevelAllocator implements BufferAllocator {
MAXIMUM_DIRECT_MEMORY = maximumAllocation;
this.config=(config!=null) ? config : DrillConfig.create();
this.errorOnLeak = errorOnLeak;
- this.acct = new Accountor(config, errorOnLeak, null, null, maximumAllocation, 0, true);
+ this.acct = new AccountorImpl(config, errorOnLeak, null, null, maximumAllocation, 0, true);
this.empty = DrillBuf.getEmpty(this, acct);
this.childrenMap = ENABLE_ACCOUNTING ? new IdentityHashMap<ChildAllocator, StackTraceElement[]>() : null;
}
@@ -83,7 +82,7 @@ public class TopLevelAllocator implements BufferAllocator {
return empty;
}
if(!acct.reserve(min)) {
- throw new OutOfMemoryRuntimeException(createErrorMsg(this, min));
+ throw new OutOfMemoryException(createErrorMsg(this, min));
}
try {
@@ -94,7 +93,7 @@ public class TopLevelAllocator implements BufferAllocator {
} catch (OutOfMemoryError e) {
if ("Direct buffer memory".equals(e.getMessage())) {
acct.release(min);
- throw new OutOfMemoryRuntimeException(createErrorMsg(this, min), e);
+ throw new OutOfMemoryException(createErrorMsg(this, min), e);
} else {
throw e;
}
@@ -122,18 +121,20 @@ public class TopLevelAllocator implements BufferAllocator {
}
@Override
- public BufferAllocator getChildAllocator(FragmentContext context, long initialReservation, long maximumReservation,
+ public BufferAllocator getChildAllocator(LimitConsumer limitConsumer, long initialReservation,
+ long maximumReservation,
boolean applyFragmentLimit) {
if(!acct.reserve(initialReservation)){
logger.debug(String.format("You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.", initialReservation, acct.getCapacity() - acct.getAllocation()));
- throw new OutOfMemoryRuntimeException(
+ throw new OutOfMemoryException(
String
.format(
"You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.",
initialReservation, acct.getCapacity() - acct.getAllocation()));
}
logger.debug("New child allocator with initial reservation {}", initialReservation);
- ChildAllocator allocator = new ChildAllocator(context, acct, maximumReservation, initialReservation, childrenMap, applyFragmentLimit);
+ ChildAllocator allocator = new ChildAllocator(limitConsumer, acct, maximumReservation, initialReservation,
+ childrenMap, applyFragmentLimit);
if(ENABLE_ACCOUNTING){
childrenMap.put(allocator, Thread.currentThread().getStackTrace());
}
@@ -142,17 +143,17 @@ public class TopLevelAllocator implements BufferAllocator {
}
@Override
- public void resetFragmentLimits() {
+ public void resetLimits() {
acct.resetFragmentLimits();
}
@Override
- public void setFragmentLimit(long limit){
+ public void setLimit(long limit){
acct.setFragmentLimit(limit);
}
@Override
- public long getFragmentLimit(){
+ public long getLimit(){
return acct.getFragmentLimit();
}
@@ -186,29 +187,26 @@ public class TopLevelAllocator implements BufferAllocator {
private class ChildAllocator implements BufferAllocator {
private final DrillBuf empty;
- private Accountor childAcct;
+ private AccountorImpl childAcct;
private Map<ChildAllocator, StackTraceElement[]> children = new HashMap<>();
private boolean closed = false;
- private FragmentHandle handle;
- private FragmentContext fragmentContext;
+ private LimitConsumer limitConsumer;
private Map<ChildAllocator, StackTraceElement[]> thisMap;
private boolean applyFragmentLimit;
- public ChildAllocator(FragmentContext context,
- Accountor parentAccountor,
+ public ChildAllocator(LimitConsumer limitConsumer,
+ AccountorImpl parentAccountor,
long max,
long pre,
Map<ChildAllocator,
StackTraceElement[]> map,
boolean applyFragmentLimit) {
assert max >= pre;
- this.applyFragmentLimit=applyFragmentLimit;
- DrillConfig drillConf = context != null ? context.getConfig() : null;
- childAcct = new Accountor(drillConf, errorOnLeak, context, parentAccountor, max, pre, applyFragmentLimit);
- this.fragmentContext=context;
- this.handle = context != null ? context.getHandle() : null;
+ this.applyFragmentLimit = applyFragmentLimit;
+ this.limitConsumer = limitConsumer;
+ childAcct = new AccountorImpl(config, errorOnLeak, limitConsumer, parentAccountor, max, pre, applyFragmentLimit);
thisMap = map;
- this.empty = DrillBuf.getEmpty(this, childAcct);
+ empty = DrillBuf.getEmpty(this, childAcct);
}
@Override
@@ -226,15 +224,11 @@ public class TopLevelAllocator implements BufferAllocator {
@Override
public DrillBuf buffer(int size, int max) {
- if (ENABLE_ACCOUNTING) {
- injector.injectUnchecked(fragmentContext, CHILD_BUFFER_INJECTION_SITE);
- }
-
if (size == 0) {
return empty;
}
if(!childAcct.reserve(size)) {
- throw new OutOfMemoryRuntimeException(createErrorMsg(this, size));
+ throw new OutOfMemoryException(createErrorMsg(this, size));
}
try {
@@ -245,7 +239,7 @@ public class TopLevelAllocator implements BufferAllocator {
} catch (OutOfMemoryError e) {
if ("Direct buffer memory".equals(e.getMessage())) {
childAcct.release(size);
- throw new OutOfMemoryRuntimeException(createErrorMsg(this, size), e);
+ throw new OutOfMemoryException(createErrorMsg(this, size), e);
} else {
throw e;
}
@@ -262,17 +256,19 @@ public class TopLevelAllocator implements BufferAllocator {
}
@Override
- public BufferAllocator getChildAllocator(FragmentContext context, long initialReservation, long maximumReservation,
+ public BufferAllocator getChildAllocator(LimitConsumer limitConsumer, long initialReservation,
+ long maximumReservation,
boolean applyFragmentLimit) {
if (!childAcct.reserve(initialReservation)) {
- throw new OutOfMemoryRuntimeException(
+ throw new OutOfMemoryException(
String
.format(
"You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.",
initialReservation, childAcct.getAvailable()));
}
logger.debug("New child allocator with initial reservation {}", initialReservation);
- ChildAllocator newChildAllocator = new ChildAllocator(context, childAcct, maximumReservation, initialReservation, null, applyFragmentLimit);
+ ChildAllocator newChildAllocator = new ChildAllocator(limitConsumer, childAcct, maximumReservation,
+ initialReservation, null, applyFragmentLimit);
this.children.put(newChildAllocator, Thread.currentThread().getStackTrace());
return newChildAllocator;
}
@@ -282,17 +278,17 @@ public class TopLevelAllocator implements BufferAllocator {
}
@Override
- public void resetFragmentLimits(){
+ public void resetLimits(){
childAcct.resetFragmentLimits();
}
@Override
- public void setFragmentLimit(long limit){
+ public void setLimit(long limit){
childAcct.setFragmentLimit(limit);
}
@Override
- public long getFragmentLimit(){
+ public long getLimit(){
return childAcct.getFragmentLimit();
}
@@ -314,8 +310,8 @@ public class TopLevelAllocator implements BufferAllocator {
IllegalStateException e = new IllegalStateException(String.format(
- "Failure while trying to close child allocator: Child level allocators not closed. Fragment %d:%d. Stack trace: \n %s",
- handle.getMajorFragmentId(), handle.getMinorFragmentId(), sb.toString()));
+ "Failure while trying to close child allocator: Child level allocators not closed. Identifier: %s. Stack trace: \n %s",
+ limitConsumer.getIdentifier(), sb.toString()));
if (errorOnLeak) {
throw e;
} else {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManager.java
deleted file mode 100644
index e0f1eab..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManager.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.ops;
-
-import com.carrotsearch.hppc.LongObjectOpenHashMap;
-import io.netty.buffer.DrillBuf;
-import org.apache.drill.exec.memory.BufferAllocator;
-
-/**
- * Manages a list of {@link DrillBuf}s that can be reallocated as needed. Upon
- * re-allocation the old buffer will be freed. Managing a list of these buffers
- * prevents some parts of the system from needing to define a correct location
- * to place the final call to free them.
- *
- * The current uses of these types of buffers are within the pluggable components of Drill.
- * In UDFs, memory management should not be a concern. We provide access to re-allocatable
- * DrillBufs to give UDF writers general purpose buffers we can account for. To prevent the need
- * for UDFs to contain boilerplate to close all of the buffers they request, this list
- * is tracked at a higher level and all of the buffers are freed once we are sure that
- * the code depending on them is done executing (currently {@link FragmentContext}
- * and {@link QueryContext}.
- */
-public class BufferManager implements AutoCloseable {
- private LongObjectOpenHashMap<DrillBuf> managedBuffers = new LongObjectOpenHashMap<>();
- private final BufferAllocator allocator;
-
- // fragment context associated with this buffer manager, if the buffer
- // manager is owned by another type, this can be null
- private final FragmentContext fragmentContext;
-
- public BufferManager(BufferAllocator allocator, FragmentContext fragmentContext) {
- this.allocator = allocator;
- this.fragmentContext = fragmentContext;
- }
-
- @Override
- public void close() throws Exception {
- final Object[] mbuffers = ((LongObjectOpenHashMap<Object>) (Object) managedBuffers).values;
- for (int i = 0; i < mbuffers.length; i++) {
- if (managedBuffers.allocated[i]) {
- ((DrillBuf) mbuffers[i]).release(1);
- }
- }
- managedBuffers.clear();
- }
-
- public DrillBuf replace(DrillBuf old, int newSize) {
- if (managedBuffers.remove(old.memoryAddress()) == null) {
- throw new IllegalStateException("Tried to remove unmanaged buffer.");
- }
- old.release(1);
- return getManagedBuffer(newSize);
- }
-
- public DrillBuf getManagedBuffer() {
- return getManagedBuffer(256);
- }
-
- public DrillBuf getManagedBuffer(int size) {
- DrillBuf newBuf = allocator.buffer(size);
- managedBuffers.put(newBuf.memoryAddress(), newBuf);
- newBuf.setFragmentContext(fragmentContext);
- newBuf.setBufferManager(this);
- return newBuf;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
new file mode 100644
index 0000000..38f2736
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
@@ -0,0 +1,46 @@
+package org.apache.drill.exec.ops;
+
+import io.netty.buffer.DrillBuf;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+
+import com.carrotsearch.hppc.LongObjectOpenHashMap;
+
+public class BufferManagerImpl implements BufferManager {
+ private LongObjectOpenHashMap<DrillBuf> managedBuffers = new LongObjectOpenHashMap<>();
+ private final BufferAllocator allocator;
+
+ public BufferManagerImpl(BufferAllocator allocator) {
+ this.allocator = allocator;
+ }
+
+ @Override
+ public void close() {
+ final Object[] mbuffers = ((LongObjectOpenHashMap<Object>) (Object) managedBuffers).values;
+ for (int i = 0; i < mbuffers.length; i++) {
+ if (managedBuffers.allocated[i]) {
+ ((DrillBuf) mbuffers[i]).release(1);
+ }
+ }
+ managedBuffers.clear();
+ }
+
+ public DrillBuf replace(DrillBuf old, int newSize) {
+ if (managedBuffers.remove(old.memoryAddress()) == null) {
+ throw new IllegalStateException("Tried to remove unmanaged buffer.");
+ }
+ old.release(1);
+ return getManagedBuffer(newSize);
+ }
+
+ public DrillBuf getManagedBuffer() {
+ return getManagedBuffer(256);
+ }
+
+ public DrillBuf getManagedBuffer(int size) {
+ DrillBuf newBuf = allocator.buffer(size);
+ managedBuffers.put(newBuf.memoryAddress(), newBuf);
+ newBuf.setBufferManager(this);
+ return newBuf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index c974652..a773c22 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -30,18 +30,19 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.memory.LimitConsumer;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
+import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.RpcOutcomeListener;
import org.apache.drill.exec.rpc.control.ControlTunnel;
@@ -158,9 +159,10 @@ public class FragmentContext implements AutoCloseable, UdfUtilities {
// Add the fragment context to the root allocator.
// The QueryManager will call the root allocator to recalculate all the memory limits for all the fragments
try {
- allocator = context.getAllocator().getChildAllocator(this, fragment.getMemInitial(), fragment.getMemMax(), true);
+ allocator = context.getAllocator().getChildAllocator(new AsLimitConsumer(), fragment.getMemInitial(),
+ fragment.getMemMax(), true);
Preconditions.checkNotNull(allocator, "Unable to acuqire allocator");
- } catch(final OutOfMemoryException | OutOfMemoryRuntimeException e) {
+ } catch (final OutOfMemoryException e) {
throw UserException.memoryError(e)
.addContext("Fragment", getHandle().getMajorFragmentId() + ":" + getHandle().getMinorFragmentId())
.build(logger);
@@ -169,7 +171,32 @@ public class FragmentContext implements AutoCloseable, UdfUtilities {
}
stats = new FragmentStats(allocator, dbContext.getMetrics(), fragment.getAssignment());
- bufferManager = new BufferManager(this.allocator, this);
+ bufferManager = new BufferManagerImpl(this.allocator);
+ }
+
+ private class AsLimitConsumer implements LimitConsumer {
+ final String identifier = QueryIdHelper.getFragmentId(fragment.getHandle());
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public long getAllocated() {
+ return allocator.getAllocatedMemory();
+ }
+
+ @Override
+ public long getLimit() {
+ return allocator.getLimit();
+ }
+
+ @Override
+ public void setLimit(long limit) {
+ allocator.setLimit(limit);
+ }
+
}
/**
@@ -270,6 +297,9 @@ public class FragmentContext implements AutoCloseable, UdfUtilities {
return frag;
}
+ public LimitConsumer asLimitConsumer() {
+ return new AsLimitConsumer();
+ }
/**
@@ -287,7 +317,8 @@ public class FragmentContext implements AutoCloseable, UdfUtilities {
public BufferAllocator getNewChildAllocator(final long initialReservation,
final long maximumReservation,
final boolean applyFragmentLimit) throws OutOfMemoryException {
- return allocator.getChildAllocator(this, initialReservation, maximumReservation, applyFragmentLimit);
+ return allocator.getChildAllocator(new AsLimitConsumer(), initialReservation, maximumReservation,
+ applyFragmentLimit);
}
public <T> T getImplementationClass(final ClassGenerator<T> cg)
@@ -365,7 +396,7 @@ public class FragmentContext implements AutoCloseable, UdfUtilities {
}
public void setFragmentLimit(final long limit) {
- allocator.setFragmentLimit(limit);
+ allocator.setLimit(limit);
}
public ExecutionControls getExecutionControls() {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContextImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContextImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContextImpl.java
index a78ee23..a3ec6bd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContextImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorContextImpl.java
@@ -17,28 +17,27 @@
*/
package org.apache.drill.exec.ops;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
import io.netty.buffer.DrillBuf;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+
import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.physical.base.PhysicalOperator;
-
-import com.carrotsearch.hppc.LongObjectOpenHashMap;
-import org.apache.drill.exec.testing.ExecutionControls;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.exec.testing.ExecutionControls;
import org.apache.drill.exec.work.WorkManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
class OperatorContextImpl extends OperatorContext implements AutoCloseable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorContextImpl.class);
@@ -46,9 +45,9 @@ class OperatorContextImpl extends OperatorContext implements AutoCloseable {
private final BufferAllocator allocator;
private final ExecutionControls executionControls;
private boolean closed = false;
- private PhysicalOperator popConfig;
- private OperatorStats stats;
- private LongObjectOpenHashMap<DrillBuf> managedBuffers = new LongObjectOpenHashMap<>();
+ private final PhysicalOperator popConfig;
+ private final OperatorStats stats;
+ private final BufferManager manager;
private final boolean applyFragmentLimit;
private DrillFileSystem fs;
private final ExecutorService executor;
@@ -65,6 +64,7 @@ class OperatorContextImpl extends OperatorContext implements AutoCloseable {
this.applyFragmentLimit=applyFragmentLimit;
this.allocator = context.getNewChildAllocator(popConfig.getInitialAllocation(), popConfig.getMaxAllocation(), applyFragmentLimit);
this.popConfig = popConfig;
+ this.manager = new BufferManagerImpl(allocator);
OpProfileDef def = new OpProfileDef(popConfig.getOperatorId(), popConfig.getOperatorType(), getChildCount(popConfig));
stats = context.getStats().newOperatorStats(def, allocator);
@@ -76,28 +76,22 @@ class OperatorContextImpl extends OperatorContext implements AutoCloseable {
this.applyFragmentLimit=applyFragmentLimit;
this.allocator = context.getNewChildAllocator(popConfig.getInitialAllocation(), popConfig.getMaxAllocation(), applyFragmentLimit);
this.popConfig = popConfig;
+ this.manager = new BufferManagerImpl(allocator);
this.stats = stats;
executionControls = context.getExecutionControls();
executor = context.getDrillbitContext().getExecutor();
}
public DrillBuf replace(DrillBuf old, int newSize) {
- if (managedBuffers.remove(old.memoryAddress()) == null) {
- throw new IllegalStateException("Tried to remove unmanaged buffer.");
- }
- old.release();
- return getManagedBuffer(newSize);
+ return manager.replace(old, newSize);
}
public DrillBuf getManagedBuffer() {
- return getManagedBuffer(256);
+ return manager.getManagedBuffer();
}
public DrillBuf getManagedBuffer(int size) {
- DrillBuf newBuf = allocator.buffer(size);
- managedBuffers.put(newBuf.memoryAddress(), newBuf);
- newBuf.setOperatorContext(this);
- return newBuf;
+ return manager.getManagedBuffer(size);
}
public ExecutionControls getExecutionControls() {
@@ -123,13 +117,7 @@ class OperatorContextImpl extends OperatorContext implements AutoCloseable {
}
logger.debug("Closing context for {}", popConfig != null ? popConfig.getClass().getName() : null);
- // release managed buffers.
- Object[] buffers = ((LongObjectOpenHashMap<Object>)(Object)managedBuffers).values;
- for (int i =0; i < buffers.length; i++) {
- if (managedBuffers.allocated[i]) {
- ((DrillBuf)buffers[i]).release();
- }
- }
+ manager.close();
if (allocator != null) {
allocator.close();
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index b56c16c..3bcd111 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -17,23 +17,22 @@
*/
package org.apache.drill.exec.ops;
+import io.netty.buffer.DrillBuf;
+
import java.io.IOException;
import java.util.Collection;
import java.util.List;
-import com.google.common.collect.Lists;
-import io.netty.buffer.DrillBuf;
-import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.jdbc.SimpleCalciteSchema;
-
+import org.apache.calcite.schema.SchemaPlus;
import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.LogicalPlanPersistence;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.planner.sql.DrillOperatorTable;
import org.apache.drill.exec.proto.BitControl.QueryContextInformation;
@@ -51,6 +50,8 @@ import org.apache.drill.exec.testing.ExecutionControls;
import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.drill.exec.util.Utilities;
+import com.google.common.collect.Lists;
+
// TODO except for a couple of tests, this is only created by Foreman
// TODO the many methods that just return drillbitContext.getXxx() should be replaced with getDrillbitContext()
// TODO - consider re-name to PlanningContext, as the query execution context actually appears
@@ -98,7 +99,7 @@ public class QueryContext implements AutoCloseable, OptimizerRulesContext {
throw new DrillRuntimeException("Error creating off-heap allocator for planning context.",e);
}
// TODO(DRILL-1942) the new allocator has this capability built-in, so this can be removed once that is available
- bufferManager = new BufferManager(this.allocator, null);
+ bufferManager = new BufferManagerImpl(this.allocator);
viewExpansionContext = new ViewExpansionContext(this);
schemaTreesToClose = Lists.newArrayList();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
index 9301650..ee6475c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.physical.impl;
import java.util.List;
import org.apache.drill.common.DeferredException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OpProfileDef;
import org.apache.drill.exec.ops.OperatorContext;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java
index 1bf6c01..59e7448 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.physical.impl;
import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergingReceiverPOP;
import org.apache.drill.exec.physical.impl.mergereceiver.MergingRecordBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index dbb5e00..6c763d6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -30,10 +30,9 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.base.PhysicalOperator;
@@ -184,7 +183,7 @@ public class ScanBatch implements CloseableRecordBatch {
injector.injectChecked(context.getExecutionControls(), "next-allocate", OutOfMemoryException.class);
currentReader.allocate(fieldVectorMap);
- } catch (OutOfMemoryException | OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
logger.debug("Caught Out of Memory Exception", e);
clearFieldVectorMap();
return IterOutcome.OUT_OF_MEMORY;
@@ -260,7 +259,7 @@ public class ScanBatch implements CloseableRecordBatch {
} else {
return IterOutcome.OK;
}
- } catch (OutOfMemoryRuntimeException ex) {
+ } catch (OutOfMemoryException ex) {
context.fail(UserException.memoryError(ex).build(logger));
return IterOutcome.STOP;
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
index a46ea35..3b90979 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
@@ -20,8 +20,8 @@ package org.apache.drill.exec.physical.impl;
import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.AccountingUserConnection;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
@@ -35,6 +35,7 @@ import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.IterOutcome;
import com.google.common.base.Preconditions;
+
import org.apache.drill.exec.testing.ControlsInjector;
import org.apache.drill.exec.testing.ControlsInjectorFactory;
@@ -81,7 +82,7 @@ public class ScreenCreator implements RootCreator<Screen> {
logger.trace("Screen Outcome {}", outcome);
switch (outcome) {
case OUT_OF_MEMORY:
- throw new OutOfMemoryRuntimeException();
+ throw new OutOfMemoryException();
case STOP:
return false;
case NONE:
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
index 248c8da..73f3f95 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
@@ -20,8 +20,8 @@ package org.apache.drill.exec.physical.impl;
import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.AccountingDataTunnel;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
@@ -98,7 +98,7 @@ public class SingleSenderCreator implements RootCreator<SingleSender>{
// logger.debug("Outcome of sender next {}", out);
switch (out) {
case OUT_OF_MEMORY:
- throw new OutOfMemoryRuntimeException();
+ throw new OutOfMemoryException();
case STOP:
case NONE:
// if we didn't do anything yet, send an empty schema.
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index aca7549..54d2839 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -30,6 +30,7 @@ import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
@@ -37,7 +38,6 @@ import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.TopN;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
index 34c514e..7da8a16 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.Writer;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
index 89122c8..ee9a0ab 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
@@ -30,6 +30,7 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
@@ -39,7 +40,6 @@ import org.apache.drill.exec.expr.HoldingContainerExpression;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.StreamingAggregate;
import org.apache.drill.exec.physical.impl.aggregate.StreamingAggregator.AggOutcome;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
index 98db0cd..c731e38 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
@@ -19,8 +19,8 @@ package org.apache.drill.exec.physical.impl.broadcastsender;
import java.util.List;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.AccountingDataTunnel;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
@@ -97,7 +97,7 @@ public class BroadcastSenderRootExec extends BaseRootExec {
logger.debug("Outcome of sender next {}", out);
switch(out){
case OUT_OF_MEMORY:
- throw new OutOfMemoryRuntimeException();
+ throw new OutOfMemoryException();
case STOP:
case NONE:
for (int i = 0; i < tunnels.length; ++i) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
index 17ac7b1..e435e79 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
@@ -26,11 +26,11 @@ import org.apache.drill.common.expression.ExpressionStringBuilder;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Filter;
import org.apache.drill.exec.record.AbstractSingleRecordBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate2.java
index 11d01d7..d3f9eda 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate2.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.physical.impl.filter;
import javax.inject.Named;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import org.apache.drill.exec.record.RecordBatch;
@@ -65,7 +65,7 @@ public abstract class FilterTemplate2 implements Filterer{
return;
}
if (! outgoingSelectionVector.allocateNewSafe(recordCount)) {
- throw new OutOfMemoryRuntimeException("Unable to allocate filter batch");
+ throw new OutOfMemoryException("Unable to allocate filter batch");
}
switch(svMode){
case NONE:
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 3a8b735..c3b3f45 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -29,6 +29,7 @@ import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.logical.data.NamedExpression;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
@@ -38,7 +39,6 @@ import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.FlattenPOP;
import org.apache.drill.exec.record.AbstractSingleRecordBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 21ebd9a..2d0bd43 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -31,10 +31,10 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
import org.apache.drill.exec.physical.config.HashJoinPOP;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 96113e9..edafbfc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -34,13 +34,13 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.impl.join.JoinUtils.JoinComparator;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
index 8f2dad3..b390b8f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
@@ -25,10 +25,10 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.NestedLoopJoinPOP;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
@@ -40,6 +40,7 @@ import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.AllocationHelper;
+
import com.google.common.base.Preconditions;
import com.sun.codemodel.JExpr;
import com.sun.codemodel.JExpression;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
index 06dd699..176ee17 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
@@ -19,8 +19,8 @@ package org.apache.drill.exec.physical.impl.limit;
import java.util.List;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Limit;
import org.apache.drill.exec.record.AbstractSingleRecordBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index cd94274..1cc598a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -35,13 +35,13 @@ import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
import org.apache.drill.exec.physical.MinorFragmentEndpoint;
@@ -79,7 +79,6 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.calcite.rel.RelFieldCollation.Direction;
import com.google.common.base.Preconditions;
-
import com.google.common.collect.Lists;
import com.sun.codemodel.JConditional;
import com.sun.codemodel.JExpr;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index 629a3e2..c9483ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -40,6 +40,7 @@ import org.apache.drill.exec.cache.DistributedMap;
import org.apache.drill.exec.cache.DistributedMultiMap;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
@@ -49,7 +50,6 @@ import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.OrderedPartitionSender;
import org.apache.drill.exec.physical.impl.sort.SortBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
index 6e49e78..5e9e84c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
@@ -27,12 +27,12 @@ import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.AccountingDataTunnel;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
@@ -169,7 +169,7 @@ public class PartitionSenderRootExec extends BaseRootExec {
return false;
case OUT_OF_MEMORY:
- throw new OutOfMemoryRuntimeException();
+ throw new OutOfMemoryException();
case STOP:
if (partitioner != null) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
index 98ee320..e3033b4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
@@ -22,9 +22,9 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingDeque;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.ProducerConsumer;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
@@ -81,7 +81,7 @@ public class ProducerConsumerBatch extends AbstractRecordBatch {
} else if (wrapper.failed) {
return IterOutcome.STOP;
} else if (wrapper.outOfMemory) {
- throw new OutOfMemoryRuntimeException();
+ throw new OutOfMemoryException();
}
recordCount = wrapper.batch.getRecordCount();
@@ -149,7 +149,7 @@ public class ProducerConsumerBatch extends AbstractRecordBatch {
throw new UnsupportedOperationException();
}
}
- } catch (final OutOfMemoryRuntimeException e) {
+ } catch (final OutOfMemoryException e) {
try {
queue.putFirst(RecordBatchDataWrapper.outOfMemory());
} catch (final InterruptedException ex) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index ce7c5ec..1ca5d1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -40,6 +40,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
@@ -51,7 +52,6 @@ import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Project;
import org.apache.drill.exec.planner.StarColumnHelper;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
index cebefa5..fb17669 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
@@ -26,13 +26,13 @@ import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Sort;
import org.apache.drill.exec.record.AbstractRecordBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
index cb04244..fa6001b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
@@ -21,10 +21,10 @@ import java.io.IOException;
import java.util.List;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.SelectionVectorRemover;
import org.apache.drill.exec.record.AbstractSingleRecordBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
index 357269d..51d74cc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
@@ -30,14 +30,14 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.UnionAll;
import org.apache.drill.exec.record.AbstractRecordBatch;
@@ -146,7 +146,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
for (ValueVector v : allocationVectors) {
try {
AllocationHelper.allocateNew(v, current.getRecordCount());
- } catch (OutOfMemoryRuntimeException ex) {
+ } catch (OutOfMemoryException ex) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java
index fafa14e..00cf295 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unorderedreceiver/UnorderedReceiverBatch.java
@@ -23,8 +23,8 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
import org.apache.drill.exec.ops.OperatorContext;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
index da13669..2261734 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.List;
import com.google.common.collect.Iterables;
+
import org.apache.drill.common.exceptions.DrillException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.FunctionCall;
@@ -30,12 +31,12 @@ import org.apache.drill.common.logical.data.Order;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.WindowPOP;
import org.apache.drill.exec.record.AbstractRecordBatch;
[08/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
deleted file mode 100644
index 5575b5e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex.impl;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.VectorContainer;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.MapVector;
-import org.apache.drill.exec.vector.complex.StateTool;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
-
-import com.google.common.base.Preconditions;
-
-public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
-
- private SingleMapWriter mapRoot;
- private SingleListWriter listRoot;
- private final MapVector container;
-
- Mode mode = Mode.INIT;
- private final String name;
- private final boolean unionEnabled;
-
- private enum Mode { INIT, MAP, LIST };
-
- public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){
- super(null);
- this.name = name;
- this.container = container;
- this.unionEnabled = unionEnabled;
- }
-
- public ComplexWriterImpl(String name, MapVector container){
- this(name, container, false);
- }
-
- @Override
- public MaterializedField getField() {
- return container.getField();
- }
-
- @Override
- public int getValueCapacity() {
- return container.getValueCapacity();
- }
-
- private void check(Mode... modes){
- StateTool.check(mode, modes);
- }
-
- @Override
- public void reset(){
- setPosition(0);
- }
-
- @Override
- public void close() throws Exception {
- clear();
- mapRoot.close();
- if (listRoot != null) {
- listRoot.close();
- }
- }
-
- @Override
- public void clear(){
- switch(mode){
- case MAP:
- mapRoot.clear();
- break;
- case LIST:
- listRoot.clear();
- break;
- }
- }
-
- @Override
- public void setValueCount(int count){
- switch(mode){
- case MAP:
- mapRoot.setValueCount(count);
- break;
- case LIST:
- listRoot.setValueCount(count);
- break;
- }
- }
-
- @Override
- public void setPosition(int index){
- super.setPosition(index);
- switch(mode){
- case MAP:
- mapRoot.setPosition(index);
- break;
- case LIST:
- listRoot.setPosition(index);
- break;
- }
- }
-
-
- public MapWriter directMap(){
- Preconditions.checkArgument(name == null);
-
- switch(mode){
-
- case INIT:
- MapVector map = (MapVector) container;
- mapRoot = new SingleMapWriter(map, this, unionEnabled);
- mapRoot.setPosition(idx());
- mode = Mode.MAP;
- break;
-
- case MAP:
- break;
-
- default:
- check(Mode.INIT, Mode.MAP);
- }
-
- return mapRoot;
- }
-
- @Override
- public MapWriter rootAsMap() {
- switch(mode){
-
- case INIT:
- MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
- mapRoot = new SingleMapWriter(map, this, unionEnabled);
- mapRoot.setPosition(idx());
- mode = Mode.MAP;
- break;
-
- case MAP:
- break;
-
- default:
- check(Mode.INIT, Mode.MAP);
- }
-
- return mapRoot;
- }
-
-
- @Override
- public void allocate() {
- if(mapRoot != null) {
- mapRoot.allocate();
- } else if(listRoot != null) {
- listRoot.allocate();
- }
- }
-
- @Override
- public ListWriter rootAsList() {
- switch(mode){
-
- case INIT:
- listRoot = new SingleListWriter(name, container, this);
- listRoot.setPosition(idx());
- mode = Mode.LIST;
- break;
-
- case LIST:
- break;
-
- default:
- check(Mode.INIT, Mode.MAP);
- }
-
- return listRoot;
- }
-
- private static class VectorAccessibleFacade extends MapVector {
-
- private final VectorContainer vc;
-
- public VectorAccessibleFacade(VectorContainer vc) {
- super("", null, null);
- this.vc = vc;
- }
-
- @Override
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
- final ValueVector v = vc.addOrGet(name, type, clazz);
- putChild(name, v);
- return this.typeify(v, clazz);
-
- }
-
- }
-
- public static ComplexWriter getWriter(String name, VectorContainer container){
- VectorAccessibleFacade vc = new VectorAccessibleFacade(container);
- return new ComplexWriterImpl(name, vc);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
deleted file mode 100644
index 5f57c01..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex.impl;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VectorDescriptor;
-import org.apache.drill.exec.vector.ZeroVector;
-import org.apache.drill.exec.vector.complex.AbstractMapVector;
-import org.apache.drill.exec.vector.complex.ListVector;
-import org.apache.drill.exec.vector.complex.UnionVector;
-import org.apache.drill.exec.vector.complex.writer.FieldWriter;
-
-import java.lang.reflect.Constructor;
-
-/**
- * This FieldWriter implementation delegates all FieldWriter API calls to an inner FieldWriter. This inner field writer
- * can start as a specific type, and this class will promote the writer to a UnionWriter if a call is made that the specifically
- * typed writer cannot handle. A new UnionVector is created, wrapping the original vector, and replaces the original vector
- * in the parent vector, which can be either an AbstractMapVector or a ListVector.
- */
-public class PromotableWriter extends AbstractPromotableFieldWriter {
-
- private final AbstractMapVector parentContainer;
- private final ListVector listVector;
- private int position;
-
- private enum State {
- UNTYPED, SINGLE, UNION
- }
-
- private MinorType type;
- private ValueVector vector;
- private UnionVector unionVector;
- private State state;
- private FieldWriter writer;
-
- public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
- super(null);
- this.parentContainer = parentContainer;
- this.listVector = null;
- init(v);
- }
-
- public PromotableWriter(ValueVector v, ListVector listVector) {
- super(null);
- this.listVector = listVector;
- this.parentContainer = null;
- init(v);
- }
-
- private void init(ValueVector v) {
- if (v instanceof UnionVector) {
- state = State.UNION;
- unionVector = (UnionVector) v;
- writer = new UnionWriter(unionVector);
- } else if (v instanceof ZeroVector) {
- state = State.UNTYPED;
- } else {
- setWriter(v);
- }
- }
-
- private void setWriter(ValueVector v) {
- state = State.SINGLE;
- vector = v;
- type = v.getField().getType().getMinorType();
- Class writerClass = TypeHelper.getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
- if (writerClass.equals(SingleListWriter.class)) {
- writerClass = UnionListWriter.class;
- }
- Class vectorClass = TypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField().getDataMode());
- try {
- Constructor constructor = null;
- for (Constructor c : writerClass.getConstructors()) {
- if (c.getParameterTypes().length == 3) {
- constructor = c;
- }
- }
- if (constructor == null) {
- constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class);
- writer = (FieldWriter) constructor.newInstance(vector, null);
- } else {
- writer = (FieldWriter) constructor.newInstance(vector, null, true);
- }
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void setPosition(int index) {
- super.setPosition(index);
- FieldWriter w = getWriter();
- if (w == null) {
- position = index;
- } else {
- w.setPosition(index);
- }
- }
-
- protected FieldWriter getWriter(MinorType type) {
- if (state == State.UNION) {
- return writer;
- }
- if (state == State.UNTYPED) {
- if (type == null) {
- return null;
- }
- ValueVector v = listVector.addOrGetVector(new VectorDescriptor(Types.optional(type))).getVector();
- v.allocateNew();
- setWriter(v);
- writer.setPosition(position);
- }
- if (type != this.type) {
- return promoteToUnion();
- }
- return writer;
- }
-
- @Override
- public boolean isEmptyMap() {
- return writer.isEmptyMap();
- }
-
- protected FieldWriter getWriter() {
- return getWriter(type);
- }
-
- private FieldWriter promoteToUnion() {
- String name = vector.getField().getLastName();
- TransferPair tp = vector.getTransferPair(new FieldReference(vector.getField().getType().getMinorType().name().toLowerCase()));
- tp.transfer();
- if (parentContainer != null) {
- unionVector = parentContainer.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class);
- } else if (listVector != null) {
- unionVector = listVector.promoteToUnion();
- }
- unionVector.addVector(tp.getTo());
- writer = new UnionWriter(unionVector);
- writer.setPosition(idx());
- for (int i = 0; i < idx(); i++) {
- unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
- }
- vector = null;
- state = State.UNION;
- return writer;
- }
-
- @Override
- public void allocate() {
- getWriter().allocate();
- }
-
- @Override
- public void clear() {
- getWriter().clear();
- }
-
- @Override
- public MaterializedField getField() {
- return getWriter().getField();
- }
-
- @Override
- public int getValueCapacity() {
- return getWriter().getValueCapacity();
- }
-
- @Override
- public void close() throws Exception {
- getWriter().close();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java
deleted file mode 100644
index 36e9bee..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.vector.complex.impl;
-
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.holders.RepeatedListHolder;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.RepeatedListVector;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-
-public class RepeatedListReaderImpl extends AbstractFieldReader{
- private static final int NO_VALUES = Integer.MAX_VALUE - 1;
- private static final MajorType TYPE = Types.repeated(MinorType.LIST);
- private final String name;
- private final RepeatedListVector container;
- private FieldReader reader;
-
- public RepeatedListReaderImpl(String name, RepeatedListVector container) {
- super();
- this.name = name;
- this.container = container;
- }
-
- @Override
- public MajorType getType() {
- return TYPE;
- }
-
- @Override
- public void copyAsValue(ListWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedListWriter impl = (RepeatedListWriter) writer;
- impl.container.copyFromSafe(idx(), impl.idx(), container);
- }
-
- @Override
- public void copyAsField(String name, MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedListWriter impl = (RepeatedListWriter) writer.list(name);
- impl.container.copyFromSafe(idx(), impl.idx(), container);
- }
-
- private int currentOffset;
- private int maxOffset;
-
- @Override
- public void reset() {
- super.reset();
- currentOffset = 0;
- maxOffset = 0;
- if (reader != null) {
- reader.reset();
- }
- reader = null;
- }
-
- @Override
- public int size() {
- return maxOffset - currentOffset;
- }
-
- @Override
- public void setPosition(int index) {
- if (index < 0 || index == NO_VALUES) {
- currentOffset = NO_VALUES;
- return;
- }
-
- super.setPosition(index);
- RepeatedListHolder h = new RepeatedListHolder();
- container.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- currentOffset = h.start-1;
- maxOffset = h.end;
- if(reader != null) {
- reader.setPosition(currentOffset);
- }
- }
- }
-
- @Override
- public boolean next() {
- if (currentOffset +1 < maxOffset) {
- currentOffset++;
- if (reader != null) {
- reader.setPosition(currentOffset);
- }
- return true;
- } else {
- currentOffset = NO_VALUES;
- return false;
- }
- }
-
- @Override
- public Object readObject() {
- return container.getAccessor().getObject(idx());
- }
-
- @Override
- public FieldReader reader() {
- if (reader == null) {
- ValueVector child = container.getChild(name);
- if (child == null) {
- reader = NullReader.INSTANCE;
- } else {
- reader = child.getReader();
- }
- reader.setPosition(currentOffset);
- }
- return reader;
- }
-
- public boolean isSet() {
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
deleted file mode 100644
index b2fe7b7..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.vector.complex.impl;
-
-import java.util.Map;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.RepeatedMapVector;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-
-import com.google.common.collect.Maps;
-
-@SuppressWarnings("unused")
-public class RepeatedMapReaderImpl extends AbstractFieldReader{
- private static final int NO_VALUES = Integer.MAX_VALUE - 1;
-
- private final RepeatedMapVector vector;
- private final Map<String, FieldReader> fields = Maps.newHashMap();
-
- public RepeatedMapReaderImpl(RepeatedMapVector vector) {
- this.vector = vector;
- }
-
- private void setChildrenPosition(int index) {
- for (FieldReader r : fields.values()) {
- r.setPosition(index);
- }
- }
-
- @Override
- public FieldReader reader(String name) {
- FieldReader reader = fields.get(name);
- if (reader == null) {
- ValueVector child = vector.getChild(name);
- if (child == null) {
- reader = NullReader.INSTANCE;
- } else {
- reader = child.getReader();
- }
- fields.put(name, reader);
- reader.setPosition(currentOffset);
- }
- return reader;
- }
-
- @Override
- public FieldReader reader() {
- if (currentOffset == NO_VALUES) {
- return NullReader.INSTANCE;
- }
-
- setChildrenPosition(currentOffset);
- return new SingleLikeRepeatedMapReaderImpl(vector, this);
- }
-
- private int currentOffset;
- private int maxOffset;
-
- @Override
- public void reset() {
- super.reset();
- currentOffset = 0;
- maxOffset = 0;
- for (FieldReader reader:fields.values()) {
- reader.reset();
- }
- fields.clear();
- }
-
- @Override
- public int size() {
- if (isNull()) {
- return 0;
- }
- return maxOffset - (currentOffset < 0 ? 0 : currentOffset);
- }
-
- @Override
- public void setPosition(int index) {
- if (index < 0 || index == NO_VALUES) {
- currentOffset = NO_VALUES;
- return;
- }
-
- super.setPosition(index);
- RepeatedMapHolder h = new RepeatedMapHolder();
- vector.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- currentOffset = h.start-1;
- maxOffset = h.end;
- setChildrenPosition(currentOffset);
- }
- }
-
- public void setSinglePosition(int index, int childIndex) {
- super.setPosition(index);
- RepeatedMapHolder h = new RepeatedMapHolder();
- vector.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- int singleOffset = h.start + childIndex;
- assert singleOffset < h.end;
- currentOffset = singleOffset;
- maxOffset = singleOffset + 1;
- setChildrenPosition(singleOffset);
- }
- }
-
- @Override
- public boolean next() {
- if (currentOffset +1 < maxOffset) {
- setChildrenPosition(++currentOffset);
- return true;
- } else {
- currentOffset = NO_VALUES;
- return false;
- }
- }
-
- public boolean isNull() {
- return currentOffset == NO_VALUES;
- }
-
- @Override
- public Object readObject() {
- return vector.getAccessor().getObject(idx());
- }
-
- @Override
- public MajorType getType() {
- return vector.getField().getType();
- }
-
- @Override
- public java.util.Iterator<String> iterator() {
- return vector.fieldNameIterator();
- }
-
- @Override
- public boolean isSet() {
- return true;
- }
-
- @Override
- public void copyAsValue(MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedMapWriter impl = (RepeatedMapWriter) writer;
- impl.container.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public void copyAsValueSingle(MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- SingleMapWriter impl = (SingleMapWriter) writer;
- impl.container.copyFromSafe(currentOffset, impl.idx(), vector);
- }
-
- @Override
- public void copyAsField(String name, MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedMapWriter impl = (RepeatedMapWriter) writer.map(name);
- impl.container.copyFromSafe(idx(), impl.idx(), vector);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
deleted file mode 100644
index 3f89a9f..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.vector.complex.impl;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.vector.complex.RepeatedMapVector;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-
-public class SingleLikeRepeatedMapReaderImpl extends AbstractFieldReader{
-
- private RepeatedMapReaderImpl delegate;
-
- public SingleLikeRepeatedMapReaderImpl(RepeatedMapVector vector, FieldReader delegate) {
- this.delegate = (RepeatedMapReaderImpl) delegate;
- }
-
- @Override
- public int size() {
- throw new UnsupportedOperationException("You can't call size on a single map reader.");
- }
-
- @Override
- public boolean next() {
- throw new UnsupportedOperationException("You can't call next on a single map reader.");
- }
-
- @Override
- public MajorType getType() {
- return Types.required(MinorType.MAP);
- }
-
-
- @Override
- public void copyAsValue(MapWriter writer) {
- delegate.copyAsValueSingle(writer);
- }
-
- public void copyAsValueSingle(MapWriter writer){
- delegate.copyAsValueSingle(writer);
- }
-
- @Override
- public FieldReader reader(String name) {
- return delegate.reader(name);
- }
-
- @Override
- public void setPosition(int index) {
- delegate.setPosition(index);
- }
-
- @Override
- public Object readObject() {
- return delegate.readObject();
- }
-
- @Override
- public Iterator<String> iterator() {
- return delegate.iterator();
- }
-
- @Override
- public boolean isSet() {
- return ! delegate.isNull();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleListReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleListReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleListReaderImpl.java
deleted file mode 100644
index 60d8c4e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleListReaderImpl.java
+++ /dev/null
@@ -1,88 +0,0 @@
-
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.vector.complex.impl;
-
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.vector.complex.AbstractContainerVector;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-
-@SuppressWarnings("unused")
-public class SingleListReaderImpl extends AbstractFieldReader{
-
- private static final MajorType TYPE = Types.optional(MinorType.LIST);
- private final String name;
- private final AbstractContainerVector container;
- private FieldReader reader;
-
- public SingleListReaderImpl(String name, AbstractContainerVector container) {
- super();
- this.name = name;
- this.container = container;
- }
-
- @Override
- public MajorType getType() {
- return TYPE;
- }
-
-
- @Override
- public void setPosition(int index) {
- super.setPosition(index);
- if (reader != null) {
- reader.setPosition(index);
- }
- }
-
- @Override
- public Object readObject() {
- return reader.readObject();
- }
-
- @Override
- public FieldReader reader() {
- if (reader == null) {
- reader = container.getChild(name).getReader();
- setPosition(idx());
- }
- return reader;
- }
-
- @Override
- public boolean isSet() {
- return false;
- }
-
- @Override
- public void copyAsValue(ListWriter writer) {
- throw new UnsupportedOperationException("Generic list copying not yet supported. Please resolve to typed list.");
- }
-
- @Override
- public void copyAsField(String name, MapWriter writer) {
- throw new UnsupportedOperationException("Generic list copying not yet supported. Please resolve to typed list.");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
deleted file mode 100644
index 1b39775..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
+++ /dev/null
@@ -1,108 +0,0 @@
-
-
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.vector.complex.impl;
-
-
-import java.util.Map;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.MapVector;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-
-import com.google.common.collect.Maps;
-
-@SuppressWarnings("unused")
-public class SingleMapReaderImpl extends AbstractFieldReader{
-
- private final MapVector vector;
- private final Map<String, FieldReader> fields = Maps.newHashMap();
-
- public SingleMapReaderImpl(MapVector vector) {
- this.vector = vector;
- }
-
- private void setChildrenPosition(int index){
- for(FieldReader r : fields.values()){
- r.setPosition(index);
- }
- }
-
- @Override
- public FieldReader reader(String name){
- FieldReader reader = fields.get(name);
- if(reader == null){
- ValueVector child = vector.getChild(name);
- if(child == null){
- reader = NullReader.INSTANCE;
- }else{
- reader = child.getReader();
- }
- fields.put(name, reader);
- reader.setPosition(idx());
- }
- return reader;
- }
-
- @Override
- public void setPosition(int index){
- super.setPosition(index);
- for(FieldReader r : fields.values()){
- r.setPosition(index);
- }
- }
-
- @Override
- public Object readObject() {
- return vector.getAccessor().getObject(idx());
- }
-
- @Override
- public boolean isSet() {
- return true;
- }
-
- @Override
- public MajorType getType(){
- return vector.getField().getType();
- }
-
- @Override
- public java.util.Iterator<String> iterator(){
- return vector.fieldNameIterator();
- }
-
- @Override
- public void copyAsValue(MapWriter writer){
- SingleMapWriter impl = (SingleMapWriter) writer;
- impl.container.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- @Override
- public void copyAsField(String name, MapWriter writer){
- SingleMapWriter impl = (SingleMapWriter) writer.map(name);
- impl.container.copyFromSafe(idx(), impl.idx(), vector);
- }
-
-
-}
-
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
deleted file mode 100644
index 2d351f2..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/UnionListReader.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.vector.complex.impl;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.holders.UnionHolder;
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.complex.ListVector;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.drill.exec.vector.complex.writer.FieldWriter;
-
-public class UnionListReader extends AbstractFieldReader {
-
- private ListVector vector;
- private ValueVector data;
- private UInt4Vector offsets;
-
- public UnionListReader(ListVector vector) {
- this.vector = vector;
- this.data = vector.getDataVector();
- this.offsets = vector.getOffsetVector();
- }
-
- @Override
- public boolean isSet() {
- return true;
- }
-
- MajorType type = Types.optional(MinorType.LIST);
-
- public MajorType getType() {
- return type;
- }
-
- private int currentOffset;
- private int maxOffset;
-
- @Override
- public void setPosition(int index) {
- super.setPosition(index);
- currentOffset = offsets.getAccessor().get(index) - 1;
- maxOffset = offsets.getAccessor().get(index + 1);
- }
-
- @Override
- public FieldReader reader() {
- return data.getReader();
- }
-
- @Override
- public Object readObject() {
- return vector.getAccessor().getObject(idx());
- }
-
- @Override
- public void read(int index, UnionHolder holder) {
- setPosition(idx());
- for (int i = -1; i < index; i++) {
- next();
- }
- holder.reader = data.getReader();
- holder.isSet = data.getReader().isSet() ? 1 : 0;
- }
-
- @Override
- public boolean next() {
- if (currentOffset + 1 < maxOffset) {
- data.getReader().setPosition(++currentOffset);
- return true;
- } else {
- return false;
- }
- }
-
- public void copyAsValue(ListWriter writer) {
- ComplexCopier.copy(this, (FieldWriter) writer);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java
deleted file mode 100644
index caa3aa6..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/reader/FieldReader.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex.reader;
-
-import org.apache.drill.exec.vector.complex.reader.BaseReader.ListReader;
-import org.apache.drill.exec.vector.complex.reader.BaseReader.MapReader;
-import org.apache.drill.exec.vector.complex.reader.BaseReader.RepeatedListReader;
-import org.apache.drill.exec.vector.complex.reader.BaseReader.RepeatedMapReader;
-import org.apache.drill.exec.vector.complex.reader.BaseReader.ScalarReader;
-
-
-
-public interface FieldReader extends MapReader, ListReader, ScalarReader, RepeatedMapReader, RepeatedListReader {
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/writer/FieldWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/writer/FieldWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/writer/FieldWriter.java
deleted file mode 100644
index 1a64978..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/writer/FieldWriter.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex.writer;
-
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ScalarWriter;
-
-public interface FieldWriter extends MapWriter, ListWriter, ScalarWriter {
- void allocate();
- void clear();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/pom.xml b/exec/pom.xml
index 98a0398..ebad5fa 100644
--- a/exec/pom.xml
+++ b/exec/pom.xml
@@ -45,6 +45,7 @@
<modules>
+ <module>vector</module>
<module>java-exec</module>
<module>jdbc</module>
</modules>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/pom.xml
----------------------------------------------------------------------
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
new file mode 100644
index 0000000..4bd968e
--- /dev/null
+++ b/exec/vector/pom.xml
@@ -0,0 +1,146 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>exec-parent</artifactId>
+ <groupId>org.apache.drill.exec</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>vector</artifactId>
+ <name>exec/Vectors</name>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-protocol</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>2.7.1</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ <version>0.4.2</version>
+ </dependency>
+
+ </dependencies>
+
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution> <!-- copy all templates in the same location to compile them at once -->
+ <id>copy-fmpp-resources</id>
+ <phase>initialize</phase>
+ <goals><goal>copy-resources</goal></goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/codegen</outputDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/codegen</directory>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin> <!-- generate sources from fmpp -->
+ <groupId>org.apache.drill.tools</groupId>
+ <artifactId>drill-fmpp-maven-plugin</artifactId>
+ <version>${project.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.freemarker</groupId>
+ <artifactId>freemarker</artifactId>
+ <version>2.3.19</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>generate-fmpp</id>
+ <phase>generate-sources</phase>
+ <goals><goal>generate</goal></goals>
+ <configuration>
+ <config>src/main/codegen/config.fmpp</config>
+ <output>${project.build.directory}/generated-sources</output>
+ <templates>${project.build.directory}/codegen/templates</templates>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings
+ only. It has no influence on the Maven build itself. -->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.drill.tools</groupId>
+ <artifactId>drill-fmpp-maven-plugin</artifactId>
+ <versionRange>[1.0,)</versionRange>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute>
+ <runOnIncremental>false</runOnIncremental>
+ <runOnConfiguration>true</runOnConfiguration>
+ </execute>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+
+
+</project>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/config.fmpp b/exec/vector/src/main/codegen/config.fmpp
new file mode 100644
index 0000000..663677c
--- /dev/null
+++ b/exec/vector/src/main/codegen/config.fmpp
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data: {
+ # TODO: Rename to ~valueVectorModesAndTypes for clarity.
+ vv: tdd(../data/ValueVectorTypes.tdd),
+
+}
+freemarkerLinks: {
+ includes: includes/
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/data/ValueVectorTypes.tdd b/exec/vector/src/main/codegen/data/ValueVectorTypes.tdd
new file mode 100644
index 0000000..26bf02d
--- /dev/null
+++ b/exec/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -0,0 +1,168 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ modes: [
+ {name: "Optional", prefix: "Nullable"},
+ {name: "Required", prefix: ""},
+ {name: "Repeated", prefix: "Repeated"}
+ ],
+ types: [
+ {
+ major: "Fixed",
+ width: 1,
+ javaType: "byte",
+ boxedType: "Byte",
+ fields: [{name: "value", type: "byte"}],
+ minor: [
+ { class: "TinyInt", valueHolder: "IntHolder" },
+ { class: "UInt1", valueHolder: "UInt1Holder" }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 2,
+ javaType: "char",
+ boxedType: "Character",
+ fields: [{name: "value", type: "char"}],
+ minor: [
+ { class: "UInt2", valueHolder: "UInt2Holder"}
+ ]
+ }, {
+ major: "Fixed",
+ width: 2,
+ javaType: "short",
+ boxedType: "Short",
+ fields: [{name: "value", type: "short"}],
+ minor: [
+ { class: "SmallInt", valueHolder: "Int2Holder"},
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 4,
+ javaType: "int",
+ boxedType: "Integer",
+ fields: [{name: "value", type: "int"}],
+ minor: [
+ { class: "Int", valueHolder: "IntHolder"},
+ { class: "UInt4", valueHolder: "UInt4Holder" },
+ { class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]},
+ { class: "Time", javaType: "int", friendlyType: "DateTime" },
+ { class: "IntervalYear", javaType: "int", friendlyType: "Period" }
+ { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal", fields: [{name:"value", type:"int"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 8,
+ javaType: "long",
+ boxedType: "Long",
+ fields: [{name: "value", type: "long"}],
+ minor: [
+ { class: "BigInt"},
+ { class: "UInt8" },
+ { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
+ { class: "Date", javaType: "long", friendlyType: "DateTime" },
+ { class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
+ { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal", fields: [{name:"value", type:"long"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
+ <#--
+ { class: "Money", maxPrecisionDigits: 2, scale: 1, },
+ -->
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 12,
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
+ minor: [
+ { class: "IntervalDay", millisecondsOffset: 4, friendlyType: "Period", fields: [ {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 16,
+ javaType: "DrillBuf"
+ boxedType: "DrillBuf",
+ minor: [
+ { class: "Interval", daysOffset: 4, millisecondsOffset: 8, friendlyType: "Period", fields: [ {name: "months", type: "int"}, {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 12,
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
+ minor: [
+ <#--
+ { class: "TimeTZ" },
+ { class: "Interval" }
+ -->
+ { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 16,
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
+
+ minor: [
+ { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 24,
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
+ minor: [
+ { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 20,
+ javaType: "DrillBuf",
+ boxedType: "DrillBuf",
+ minor: [
+ { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+ ]
+ },
+ {
+ major: "VarLen",
+ width: 4,
+ javaType: "int",
+ boxedType: "DrillBuf",
+ fields: [{name: "start", type: "int"}, {name: "end", type: "int"}, {name: "buffer", type: "DrillBuf"}],
+ minor: [
+ { class: "VarBinary" , friendlyType: "byte[]" },
+ { class: "VarChar" , friendlyType: "Text" },
+ { class: "Var16Char" , friendlyType: "String" }
+ ]
+ },
+ {
+ major: "Bit",
+ width: 1,
+ javaType: "int",
+ boxedType: "Integer",
+ minor: [
+ { class: "Bit" , friendlyType: "Boolean", fields: [{name: "value", type: "int"}] }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/includes/license.ftl
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/includes/license.ftl b/exec/vector/src/main/codegen/includes/license.ftl
new file mode 100644
index 0000000..0455fd8
--- /dev/null
+++ b/exec/vector/src/main/codegen/includes/license.ftl
@@ -0,0 +1,18 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/includes/vv_imports.ftl b/exec/vector/src/main/codegen/includes/vv_imports.ftl
new file mode 100644
index 0000000..0954dcf
--- /dev/null
+++ b/exec/vector/src/main/codegen/includes/vv_imports.ftl
@@ -0,0 +1,74 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.ObjectArrays;
+import com.google.common.base.Charsets;
+import com.google.common.collect.ObjectArrays;
+
+import com.google.common.base.Preconditions;
+import io.netty.buffer.*;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.memory.*;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.vector.*;
+import org.apache.drill.common.exceptions.*;
+import org.apache.drill.exec.exception.*;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.types.TypeProtos.*;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.util.DrillStringUtils;
+import org.apache.drill.exec.vector.complex.*;
+import org.apache.drill.exec.vector.complex.reader.*;
+import org.apache.drill.exec.vector.complex.impl.*;
+import org.apache.drill.exec.vector.complex.writer.*;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.util.JsonStringArrayList;
+
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.List;
+
+import java.io.Closeable;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.joda.time.DateTime;
+import org.joda.time.Period;
+
+import org.apache.hadoop.io.Text;
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/AbstractFieldReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/AbstractFieldReader.java b/exec/vector/src/main/codegen/templates/AbstractFieldReader.java
new file mode 100644
index 0000000..2b7b305
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/AbstractFieldReader.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+@SuppressWarnings("unused")
+abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
+
+ AbstractFieldReader(){
+ super();
+ }
+
+ /**
+ * Returns true if the current value of the reader is not null
+ * @return
+ */
+ public boolean isSet() {
+ return true;
+ }
+
+ <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
+ "Character", "DateTime", "Period", "Double", "Float",
+ "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
+ <#assign safeType=friendlyType />
+ <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+
+ public ${friendlyType} read${safeType}(int arrayIndex){
+ fail("read${safeType}(int arrayIndex)");
+ return null;
+ }
+
+ public ${friendlyType} read${safeType}(){
+ fail("read${safeType}()");
+ return null;
+ }
+
+ </#list>
+
+ public void copyAsValue(MapWriter writer){
+ fail("CopyAsValue MapWriter");
+ }
+ public void copyAsField(String name, MapWriter writer){
+ fail("CopyAsField MapWriter");
+ }
+
+ public void copyAsField(String name, ListWriter writer){
+ fail("CopyAsFieldList");
+ }
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign boxedType = (minor.boxedType!type.boxedType) />
+
+ public void read(${name}Holder holder){
+ fail("${name}");
+ }
+
+ public void read(Nullable${name}Holder holder){
+ fail("${name}");
+ }
+
+ public void read(int arrayIndex, ${name}Holder holder){
+ fail("Repeated${name}");
+ }
+
+ public void read(int arrayIndex, Nullable${name}Holder holder){
+ fail("Repeated${name}");
+ }
+
+ public void copyAsValue(${name}Writer writer){
+ fail("CopyAsValue${name}");
+ }
+ public void copyAsField(String name, ${name}Writer writer){
+ fail("CopyAsField${name}");
+ }
+ </#list></#list>
+
+ public FieldReader reader(String name){
+ fail("reader(String name)");
+ return null;
+ }
+
+ public FieldReader reader(){
+ fail("reader()");
+ return null;
+
+ }
+
+ public int size(){
+ fail("size()");
+ return -1;
+ }
+
+ private void fail(String name){
+ throw new IllegalArgumentException(String.format("You tried to read a [%s] type when you are using a field reader of type [%s].", name, this.getClass().getSimpleName()));
+ }
+
+
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/AbstractFieldWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/AbstractFieldWriter.java b/exec/vector/src/main/codegen/templates/AbstractFieldWriter.java
new file mode 100644
index 0000000..7ab5dce
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/AbstractFieldWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter {
+ AbstractFieldWriter(FieldWriter parent) {
+ super(parent);
+ }
+
+ @Override
+ public void start() {
+ throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
+ }
+
+ @Override
+ public void end() {
+ throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
+ }
+
+ @Override
+ public void startList() {
+ throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
+ }
+
+ @Override
+ public void endList() {
+ throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
+ }
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ @Override
+ public void write(${name}Holder holder) {
+ fail("${name}");
+ }
+
+ public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ fail("${name}");
+ }
+
+ </#list></#list>
+
+ public void writeNull() {
+ fail("${name}");
+ }
+
+ /**
+ * This implementation returns {@code false}.
+ * <p>
+ * Must be overridden by map writers.
+ * </p>
+ */
+ @Override
+ public boolean isEmptyMap() {
+ return false;
+ }
+
+ @Override
+ public MapWriter map() {
+ fail("Map");
+ return null;
+ }
+
+ @Override
+ public ListWriter list() {
+ fail("List");
+ return null;
+ }
+
+ @Override
+ public MapWriter map(String name) {
+ fail("Map");
+ return null;
+ }
+
+ @Override
+ public ListWriter list(String name) {
+ fail("List");
+ return null;
+ }
+
+ <#list vv.types as type><#list type.minor as minor>
+ <#assign lowerName = minor.class?uncap_first />
+ <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+ <#assign upperName = minor.class?upper_case />
+ <#assign capName = minor.class?cap_first />
+ <#if minor.class?starts_with("Decimal") >
+ public ${capName}Writer ${lowerName}(String name, int scale, int precision) {
+ fail("${capName}");
+ return null;
+ }
+ </#if>
+
+ @Override
+ public ${capName}Writer ${lowerName}(String name) {
+ fail("${capName}");
+ return null;
+ }
+
+ @Override
+ public ${capName}Writer ${lowerName}() {
+ fail("${capName}");
+ return null;
+ }
+
+ </#list></#list>
+
+ public void copyReader(FieldReader reader) {
+ fail("Copy FieldReader");
+ }
+
+ public void copyReaderToField(String name, FieldReader reader) {
+ fail("Copy FieldReader to STring");
+ }
+
+ private void fail(String name) {
+ throw new IllegalArgumentException(String.format("You tried to write a %s type when you are using a ValueWriter of type %s.", name, this.getClass().getSimpleName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/exec/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
new file mode 100644
index 0000000..f2f8193
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/AbstractPromotableFieldWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * A FieldWriter which delegates calls to another FieldWriter. The delegate FieldWriter can be promoted to a new type
+ * when necessary. Classes that extend this class are responsible for handling promotion.
+ *
+ * This class is generated using freemarker and the ${.template_name} template.
+ *
+ */
+@SuppressWarnings("unused")
+abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
+ AbstractPromotableFieldWriter(FieldWriter parent) {
+ super(parent);
+ }
+
+ /**
+ * Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type
+ * @param type
+ * @return
+ */
+ abstract protected FieldWriter getWriter(MinorType type);
+
+ /**
+ * Return the current FieldWriter
+ * @return
+ */
+ abstract protected FieldWriter getWriter();
+
+ @Override
+ public void start() {
+ getWriter(MinorType.MAP).start();
+ }
+
+ @Override
+ public void end() {
+ getWriter(MinorType.MAP).end();
+ }
+
+ @Override
+ public void startList() {
+ getWriter(MinorType.LIST).startList();
+ }
+
+ @Override
+ public void endList() {
+ getWriter(MinorType.LIST).endList();
+ }
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#if !minor.class?starts_with("Decimal") >
+ @Override
+ public void write(${name}Holder holder) {
+ getWriter(MinorType.${name?upper_case}).write(holder);
+ }
+
+ public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ }
+
+ </#if>
+ </#list></#list>
+
+ public void writeNull() {
+ }
+
+ @Override
+ public MapWriter map() {
+ return getWriter(MinorType.LIST).map();
+ }
+
+ @Override
+ public ListWriter list() {
+ return getWriter(MinorType.LIST).list();
+ }
+
+ @Override
+ public MapWriter map(String name) {
+ return getWriter(MinorType.MAP).map(name);
+ }
+
+ @Override
+ public ListWriter list(String name) {
+ return getWriter(MinorType.MAP).list(name);
+ }
+
+ <#list vv.types as type><#list type.minor as minor>
+ <#assign lowerName = minor.class?uncap_first />
+ <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+ <#assign upperName = minor.class?upper_case />
+ <#assign capName = minor.class?cap_first />
+ <#if !minor.class?starts_with("Decimal") >
+
+ @Override
+ public ${capName}Writer ${lowerName}(String name) {
+ return getWriter(MinorType.MAP).${lowerName}(name);
+ }
+
+ @Override
+ public ${capName}Writer ${lowerName}() {
+ return getWriter(MinorType.LIST).${lowerName}();
+ }
+
+ </#if>
+ </#list></#list>
+
+ public void copyReader(FieldReader reader) {
+ getWriter().copyReader(reader);
+ }
+
+ public void copyReaderToField(String name, FieldReader reader) {
+ getWriter().copyReaderToField(name, reader);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/BaseReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/BaseReader.java b/exec/vector/src/main/codegen/templates/BaseReader.java
new file mode 100644
index 0000000..78f32f4
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/BaseReader.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/reader/BaseReader.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.reader;
+
+<#include "/@includes/vv_imports.ftl" />
+
+
+
+@SuppressWarnings("unused")
+public interface BaseReader extends Positionable{
+ MajorType getType();
+ MaterializedField getField();
+ void reset();
+ void read(UnionHolder holder);
+ void read(int index, UnionHolder holder);
+ void copyAsValue(UnionWriter writer);
+ boolean isSet();
+
+ public interface MapReader extends BaseReader, Iterable<String>{
+ FieldReader reader(String name);
+ }
+
+ public interface RepeatedMapReader extends MapReader{
+ boolean next();
+ int size();
+ void copyAsValue(MapWriter writer);
+ }
+
+ public interface ListReader extends BaseReader{
+ FieldReader reader();
+ }
+
+ public interface RepeatedListReader extends ListReader{
+ boolean next();
+ int size();
+ void copyAsValue(ListWriter writer);
+ }
+
+ public interface ScalarReader extends
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, </#list></#list>
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> Repeated${name}Reader, </#list></#list>
+ BaseReader {}
+
+ interface ComplexReader{
+ MapReader rootAsMap();
+ ListReader rootAsList();
+ boolean rootIsMap();
+ boolean ok();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/BaseWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/BaseWriter.java b/exec/vector/src/main/codegen/templates/BaseWriter.java
new file mode 100644
index 0000000..8a9ea56
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/BaseWriter.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/writer/BaseWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.writer;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * File generated from ${.template_name} using FreeMarker.
+ */
+@SuppressWarnings("unused")
+ public interface BaseWriter extends AutoCloseable, Positionable {
+ FieldWriter getParent();
+ int getValueCapacity();
+
+ public interface MapWriter extends BaseWriter {
+
+ MaterializedField getField();
+
+ /**
+ * Whether this writer is a map writer and is empty (has no children).
+ *
+ * <p>
+ * Intended only for use in determining whether to add dummy vector to
+ * avoid empty (zero-column) schema, as in JsonReader.
+ * </p>
+ *
+ */
+ boolean isEmptyMap();
+
+ <#list vv.types as type><#list type.minor as minor>
+ <#assign lowerName = minor.class?uncap_first />
+ <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+ <#assign upperName = minor.class?upper_case />
+ <#assign capName = minor.class?cap_first />
+ <#if minor.class?starts_with("Decimal") >
+ ${capName}Writer ${lowerName}(String name, int scale, int precision);
+ </#if>
+ ${capName}Writer ${lowerName}(String name);
+ </#list></#list>
+
+ void copyReaderToField(String name, FieldReader reader);
+ MapWriter map(String name);
+ ListWriter list(String name);
+ void start();
+ void end();
+ }
+
+ public interface ListWriter extends BaseWriter {
+ void startList();
+ void endList();
+ MapWriter map();
+ ListWriter list();
+ void copyReader(FieldReader reader);
+
+ <#list vv.types as type><#list type.minor as minor>
+ <#assign lowerName = minor.class?uncap_first />
+ <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+ <#assign upperName = minor.class?upper_case />
+ <#assign capName = minor.class?cap_first />
+ ${capName}Writer ${lowerName}();
+ </#list></#list>
+ }
+
+ public interface ScalarWriter extends
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Writer, </#list></#list> BaseWriter {}
+
+ public interface ComplexWriter {
+ void allocate();
+ void clear();
+ void copyReader(FieldReader reader);
+ MapWriter rootAsMap();
+ ListWriter rootAsList();
+
+ void setPosition(int index);
+ void setValueCount(int count);
+ void reset();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/ComplexCopier.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/ComplexCopier.java b/exec/vector/src/main/codegen/templates/ComplexCopier.java
new file mode 100644
index 0000000..b8b7616
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/ComplexCopier.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/ComplexCopier.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ComplexCopier {
+
+ /**
+ * Do a deep copy of the value in input into output
+ * @param in
+ * @param out
+ */
+ public static void copy(FieldReader input, FieldWriter output) {
+ writeValue(input, output);
+ }
+
+ private static void writeValue(FieldReader reader, FieldWriter writer) {
+ final DataMode m = reader.getType().getMode();
+ final MinorType mt = reader.getType().getMinorType();
+
+ switch(m){
+ case OPTIONAL:
+ case REQUIRED:
+
+
+ switch (mt) {
+
+ case LIST:
+ writer.startList();
+ while (reader.next()) {
+ writeValue(reader.reader(), getListWriterForReader(reader.reader(), writer));
+ }
+ writer.endList();
+ break;
+ case MAP:
+ writer.start();
+ if (reader.isSet()) {
+ for(String name : reader){
+ FieldReader childReader = reader.reader(name);
+ if(childReader.isSet()){
+ writeValue(childReader, getMapWriterForReader(childReader, writer, name));
+ }
+ }
+ }
+ writer.end();
+ break;
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+
+ case ${name?upper_case}:
+ if (reader.isSet()) {
+ Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
+ reader.read(${uncappedName}Holder);
+ writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list>);
+ }
+ break;
+
+ </#if>
+ </#list></#list>
+ }
+ break;
+ }
+ }
+
+ private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
+ switch (reader.getType().getMinorType()) {
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+ case ${name?upper_case}:
+ return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name);
+ </#if>
+ </#list></#list>
+ case MAP:
+ return (FieldWriter) writer.map(name);
+ case LIST:
+ return (FieldWriter) writer.list(name);
+ default:
+ throw new UnsupportedOperationException(reader.getType().toString());
+ }
+ }
+
+ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) {
+ switch (reader.getType().getMinorType()) {
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+ case ${name?upper_case}:
+ return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>();
+ </#if>
+ </#list></#list>
+ case MAP:
+ return (FieldWriter) writer.map();
+ case LIST:
+ return (FieldWriter) writer.list();
+ default:
+ throw new UnsupportedOperationException(reader.getType().toString());
+ }
+ }
+}
[20/45] drill git commit: DRILL-3987: (REFACTOR) Common and Vector
modules building.
Posted by ja...@apache.org.
DRILL-3987: (REFACTOR) Common and Vector modules building.
- Extract Accountor interface from Implementation
- Separate FMPP modules to separate out Vector Needs versus external needs
- Separate out Vector classes from those that are VectorAccessible.
- Cleanup Memory Exception hiearchy
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4524fdbe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4524fdbe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4524fdbe
Branch: refs/heads/master
Commit: 4524fdbe938324af9eebeea2f8863ecb62c66a25
Parents: 9969d8b
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Nov 7 18:16:05 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 17:00:46 2015 -0800
----------------------------------------------------------------------
.../src/main/codegen/includes/vv_imports.ftl | 2 +-
.../src/main/codegen/templates/TypeHelper.java | 95 +++
.../apache/drill/exec/client/DrillClient.java | 2 +-
.../exec/expr/fn/impl/ByteFunctionHelpers.java | 236 ------
.../drill/exec/expr/fn/impl/DateUtility.java | 682 -----------------
.../org/apache/drill/exec/memory/Accountor.java | 462 ------------
.../apache/drill/exec/memory/AccountorImpl.java | 427 +++++++++++
.../drill/exec/memory/OutOfMemoryException.java | 51 --
.../drill/exec/memory/TopLevelAllocator.java | 70 +-
.../apache/drill/exec/ops/BufferManager.java | 81 --
.../drill/exec/ops/BufferManagerImpl.java | 46 ++
.../apache/drill/exec/ops/FragmentContext.java | 45 +-
.../drill/exec/ops/OperatorContextImpl.java | 52 +-
.../org/apache/drill/exec/ops/QueryContext.java | 15 +-
.../drill/exec/physical/impl/BaseRootExec.java | 2 +-
.../physical/impl/MergingReceiverCreator.java | 2 +-
.../drill/exec/physical/impl/ScanBatch.java | 7 +-
.../drill/exec/physical/impl/ScreenCreator.java | 7 +-
.../exec/physical/impl/SingleSenderCreator.java | 6 +-
.../exec/physical/impl/TopN/TopNBatch.java | 2 +-
.../exec/physical/impl/WriterRecordBatch.java | 2 +-
.../impl/aggregate/StreamingAggBatch.java | 2 +-
.../BroadcastSenderRootExec.java | 6 +-
.../physical/impl/filter/FilterRecordBatch.java | 2 +-
.../physical/impl/filter/FilterTemplate2.java | 4 +-
.../impl/flatten/FlattenRecordBatch.java | 2 +-
.../exec/physical/impl/join/HashJoinBatch.java | 2 +-
.../exec/physical/impl/join/MergeJoinBatch.java | 2 +-
.../physical/impl/join/NestedLoopJoinBatch.java | 3 +-
.../physical/impl/limit/LimitRecordBatch.java | 2 +-
.../impl/mergereceiver/MergingRecordBatch.java | 3 +-
.../OrderedPartitionRecordBatch.java | 2 +-
.../PartitionSenderRootExec.java | 6 +-
.../impl/producer/ProducerConsumerBatch.java | 8 +-
.../impl/project/ProjectRecordBatch.java | 2 +-
.../exec/physical/impl/sort/SortBatch.java | 2 +-
.../impl/svremover/RemovingRecordBatch.java | 2 +-
.../impl/union/UnionAllRecordBatch.java | 6 +-
.../UnorderedReceiverBatch.java | 2 +-
.../impl/window/WindowFrameRecordBatch.java | 3 +-
.../physical/impl/xsort/ExternalSortBatch.java | 11 +-
.../drill/exec/record/AbstractRecordBatch.java | 2 +-
.../exec/record/AbstractSingleRecordBatch.java | 2 +-
.../exec/record/selection/SelectionVector2.java | 4 +-
.../drill/exec/rpc/ProtobufLengthDecoder.java | 4 +-
.../drill/exec/store/AbstractRecordReader.java | 2 +-
.../apache/drill/exec/store/RecordReader.java | 2 +-
.../exec/store/easy/json/JSONRecordReader.java | 2 +-
.../drill/exec/store/mock/MockRecordReader.java | 2 +-
.../exec/store/parquet/ParquetFormatPlugin.java | 3 +-
.../exec/store/parquet/ParquetRecordWriter.java | 3 +-
.../columnreaders/ParquetRecordReader.java | 3 +-
.../exec/store/parquet2/DrillParquetReader.java | 3 +-
.../drill/exec/store/pojo/PojoRecordReader.java | 2 +-
.../apache/drill/exec/util/AssertionUtil.java | 39 -
.../apache/drill/exec/util/DecimalUtility.java | 730 -------------------
.../exec/work/batch/AbstractDataCollector.java | 2 +-
.../exec/work/batch/SpoolingRawBatchBuffer.java | 2 +-
.../apache/drill/exec/work/foreman/Foreman.java | 7 +-
.../exec/work/fragment/FragmentExecutor.java | 4 +-
.../parquet/hadoop/ColumnChunkIncReadStore.java | 4 +-
.../java/org/apache/drill/QueryTestUtil.java | 2 +-
.../apache/drill/TestAllocationException.java | 4 +-
.../drill/exec/memory/TestAllocators.java | 4 +-
.../partitionsender/TestPartitionSender.java | 2 +-
.../drill/exec/server/DrillClientFactory.java | 2 +-
exec/vector/pom.xml | 6 +
.../src/main/codegen/includes/vv_imports.ftl | 2 +-
.../main/codegen/templates/BasicTypeHelper.java | 544 ++++++++++++++
.../codegen/templates/FixedValueVectors.java | 6 +-
.../codegen/templates/NullableValueVectors.java | 4 +-
.../codegen/templates/RepeatedValueVectors.java | 6 +-
.../src/main/codegen/templates/TypeHelper.java | 600 ---------------
.../src/main/codegen/templates/UnionVector.java | 2 +-
.../templates/VariableLengthVectors.java | 6 +-
.../src/main/java/io/netty/buffer/DrillBuf.java | 18 +-
.../java/io/netty/buffer/FakeAllocator.java | 45 +-
.../netty/buffer/PooledByteBufAllocatorL.java | 9 +-
.../exec/exception/OutOfMemoryException.java | 51 ++
.../exception/OversizedAllocationException.java | 1 -
.../exec/expr/fn/impl/ByteFunctionHelpers.java | 236 ++++++
.../drill/exec/expr/fn/impl/DateUtility.java | 682 +++++++++++++++++
.../org/apache/drill/exec/memory/Accountor.java | 43 ++
.../drill/exec/memory/BufferAllocator.java | 30 +-
.../apache/drill/exec/memory/LimitConsumer.java | 28 +
.../memory/OutOfMemoryRuntimeException.java | 49 --
.../apache/drill/exec/ops/BufferManager.java | 66 ++
.../drill/exec/record/MaterializedField.java | 4 +-
.../apache/drill/exec/record/TypedFieldId.java | 5 +-
.../apache/drill/exec/util/AssertionUtil.java | 39 +
.../apache/drill/exec/util/DecimalUtility.java | 730 +++++++++++++++++++
.../drill/exec/vector/AllocationHelper.java | 2 +-
.../org/apache/drill/exec/vector/BitVector.java | 6 +-
.../apache/drill/exec/vector/ObjectVector.java | 6 +-
.../drill/exec/vector/ValueHolderHelper.java | 10 +-
.../apache/drill/exec/vector/ValueVector.java | 6 +-
.../apache/drill/exec/vector/ZeroVector.java | 4 +-
.../vector/complex/AbstractContainerVector.java | 6 +-
.../exec/vector/complex/AbstractMapVector.java | 18 +-
.../vector/complex/BaseRepeatedValueVector.java | 12 +-
.../drill/exec/vector/complex/ListVector.java | 4 +-
.../drill/exec/vector/complex/MapVector.java | 9 +-
.../exec/vector/complex/RepeatedListVector.java | 8 +-
.../exec/vector/complex/RepeatedMapVector.java | 8 +-
.../vector/complex/impl/ComplexWriterImpl.java | 25 -
.../vector/complex/impl/PromotableWriter.java | 12 +-
106 files changed, 3309 insertions(+), 3241 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 2fedfd0..5865ece 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -46,7 +46,7 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import com.sun.codemodel.JType;
import com.sun.codemodel.JCodeModel;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
new file mode 100644
index 0000000..133ed3b
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.exec.vector.complex.UnionVector;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/TypeHelper.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr;
+
+<#include "/@includes/vv_imports.ftl" />
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.accessor.*;
+import org.apache.drill.exec.vector.complex.RepeatedMapVector;
+import org.apache.drill.exec.util.CallBack;
+
+public class TypeHelper extends BasicTypeHelper{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
+
+ public static SqlAccessor getSqlAccessor(ValueVector vector){
+ final MajorType type = vector.getField().getType();
+ switch(type.getMinorType()){
+ case UNION:
+ return new UnionSqlAccessor((UnionVector) vector);
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Accessor((${minor.class}Vector) vector);
+ case OPTIONAL:
+ return new Nullable${minor.class}Accessor((Nullable${minor.class}Vector) vector);
+ case REPEATED:
+ return new GenericAccessor(vector);
+ }
+ </#list>
+ </#list>
+ case MAP:
+ case LIST:
+ return new GenericAccessor(vector);
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("find sql accessor", type));
+ }
+
+ public static JType getHolderType(JCodeModel model, MinorType type, DataMode mode){
+ switch (type) {
+ case UNION:
+ return model._ref(UnionHolder.class);
+ case MAP:
+ case LIST:
+ return model._ref(ComplexHolder.class);
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return model._ref(${minor.class}Holder.class);
+ case OPTIONAL:
+ return model._ref(Nullable${minor.class}Holder.class);
+ case REPEATED:
+ return model._ref(Repeated${minor.class}Holder.class);
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ return model._ref(ObjectHolder.class);
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get holder type", type, mode));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index 08663eb..e476fff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -41,8 +41,8 @@ import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.zk.ZKClusterCoordinator;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.memory.RootAllocatorFactory;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
deleted file mode 100644
index 191b459..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteFunctionHelpers.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.expr.fn.impl;
-
-import io.netty.buffer.DrillBuf;
-import io.netty.util.internal.PlatformDependent;
-
-import org.apache.drill.exec.util.AssertionUtil;
-import org.apache.drill.exec.util.DecimalUtility;
-
-import com.google.common.primitives.UnsignedLongs;
-
-public class ByteFunctionHelpers {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteFunctionHelpers.class);
-
- private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
-
- /**
- * Helper function to check for equality of bytes in two DrillBuffers
- *
- * @param left Left DrillBuf for comparison
- * @param lStart start offset in the buffer
- * @param lEnd end offset in the buffer
- * @param right Right DrillBuf for comparison
- * @param rStart start offset in the buffer
- * @param rEnd end offset in the buffer
- * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
- */
- public static final int equal(final DrillBuf left, int lStart, int lEnd, final DrillBuf right, int rStart, int rEnd){
- if(BOUNDS_CHECKING_ENABLED){
- left.checkBytes(lStart, lEnd);
- right.checkBytes(rStart, rEnd);
- }
- return memEqual(left.memoryAddress(), lStart, lEnd, right.memoryAddress(), rStart, rEnd);
- }
-
- private static final int memEqual(final long laddr, int lStart, int lEnd, final long raddr, int rStart,
- final int rEnd) {
-
- int n = lEnd - lStart;
- if (n == rEnd - rStart) {
- long lPos = laddr + lStart;
- long rPos = raddr + rStart;
-
- while (n > 7) {
- long leftLong = PlatformDependent.getLong(lPos);
- long rightLong = PlatformDependent.getLong(rPos);
- if (leftLong != rightLong) {
- return 0;
- }
- lPos += 8;
- rPos += 8;
- n -= 8;
- }
- while (n-- != 0) {
- byte leftByte = PlatformDependent.getByte(lPos);
- byte rightByte = PlatformDependent.getByte(rPos);
- if (leftByte != rightByte) {
- return 0;
- }
- lPos++;
- rPos++;
- }
- return 1;
- } else {
- return 0;
- }
- }
-
- /**
- * Helper function to compare a set of bytes in two DrillBuffers.
- *
- * Function will check data before completing in the case that
- *
- * @param left Left DrillBuf to compare
- * @param lStart start offset in the buffer
- * @param lEnd end offset in the buffer
- * @param right Right DrillBuf to compare
- * @param rStart start offset in the buffer
- * @param rEnd end offset in the buffer
- * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
- */
- public static final int compare(final DrillBuf left, int lStart, int lEnd, final DrillBuf right, int rStart, int rEnd){
- if(BOUNDS_CHECKING_ENABLED){
- left.checkBytes(lStart, lEnd);
- right.checkBytes(rStart, rEnd);
- }
- return memcmp(left.memoryAddress(), lStart, lEnd, right.memoryAddress(), rStart, rEnd);
- }
-
- private static final int memcmp(final long laddr, int lStart, int lEnd, final long raddr, int rStart, final int rEnd) {
- int lLen = lEnd - lStart;
- int rLen = rEnd - rStart;
- int n = Math.min(rLen, lLen);
- long lPos = laddr + lStart;
- long rPos = raddr + rStart;
-
- while (n > 7) {
- long leftLong = PlatformDependent.getLong(lPos);
- long rightLong = PlatformDependent.getLong(rPos);
- if (leftLong != rightLong) {
- return UnsignedLongs.compare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
- }
- lPos += 8;
- rPos += 8;
- n -= 8;
- }
-
- while (n-- != 0) {
- byte leftByte = PlatformDependent.getByte(lPos);
- byte rightByte = PlatformDependent.getByte(rPos);
- if (leftByte != rightByte) {
- return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
- }
- lPos++;
- rPos++;
- }
-
- if (lLen == rLen) {
- return 0;
- }
-
- return lLen > rLen ? 1 : -1;
-
- }
-
- /**
- * Helper function to compare a set of bytes in DrillBuf to a ByteArray.
- *
- * @param left Left DrillBuf for comparison purposes
- * @param lStart start offset in the buffer
- * @param lEnd end offset in the buffer
- * @param right second input to be compared
- * @param rStart start offset in the byte array
- * @param rEnd end offset in the byte array
- * @return 1 if left input is greater, -1 if left input is smaller, 0 otherwise
- */
- public static final int compare(final DrillBuf left, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
- if(BOUNDS_CHECKING_ENABLED){
- left.checkBytes(lStart, lEnd);
- }
- return memcmp(left.memoryAddress(), lStart, lEnd, right, rStart, rEnd);
- }
-
-
- private static final int memcmp(final long laddr, int lStart, int lEnd, final byte[] right, int rStart, final int rEnd) {
- int lLen = lEnd - lStart;
- int rLen = rEnd - rStart;
- int n = Math.min(rLen, lLen);
- long lPos = laddr + lStart;
- int rPos = rStart;
-
- while (n-- != 0) {
- byte leftByte = PlatformDependent.getByte(lPos);
- byte rightByte = right[rPos];
- if (leftByte != rightByte) {
- return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
- }
- lPos++;
- rPos++;
- }
-
- if (lLen == rLen) {
- return 0;
- }
-
- return lLen > rLen ? 1 : -1;
- }
-
- /*
- * Following are helper functions to interact with sparse decimal represented in a byte array.
- */
-
- // Get the integer ignore the sign
- public static int getInteger(byte[] b, int index) {
- return getInteger(b, index, true);
- }
- // Get the integer, ignore the sign
- public static int getInteger(byte[] b, int index, boolean ignoreSign) {
- int startIndex = index * DecimalUtility.integerSize;
-
- if (index == 0 && ignoreSign == true) {
- return (b[startIndex + 3] & 0xFF) |
- (b[startIndex + 2] & 0xFF) << 8 |
- (b[startIndex + 1] & 0xFF) << 16 |
- (b[startIndex] & 0x7F) << 24;
- }
-
- return ((b[startIndex + 3] & 0xFF) |
- (b[startIndex + 2] & 0xFF) << 8 |
- (b[startIndex + 1] & 0xFF) << 16 |
- (b[startIndex] & 0xFF) << 24);
-
- }
-
- // Set integer in the byte array
- public static void setInteger(byte[] b, int index, int value) {
- int startIndex = index * DecimalUtility.integerSize;
- b[startIndex] = (byte) ((value >> 24) & 0xFF);
- b[startIndex + 1] = (byte) ((value >> 16) & 0xFF);
- b[startIndex + 2] = (byte) ((value >> 8) & 0xFF);
- b[startIndex + 3] = (byte) ((value) & 0xFF);
- }
-
- // Set the sign in a sparse decimal representation
- public static void setSign(byte[] b, boolean sign) {
- int value = getInteger(b, 0);
- if (sign == true) {
- setInteger(b, 0, value | 0x80000000);
- } else {
- setInteger(b, 0, value & 0x7FFFFFFF);
- }
- }
-
- // Get the sign
- public static boolean getSign(byte[] b) {
- return ((getInteger(b, 0, false) & 0x80000000) != 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java
deleted file mode 100644
index 98a0b61..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DateUtility.java
+++ /dev/null
@@ -1,682 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.expr.fn.impl;
-
-import org.joda.time.Period;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.DateTimeFormatterBuilder;
-import org.joda.time.format.DateTimeParser;
-
-import com.carrotsearch.hppc.ObjectIntOpenHashMap;
-
-// Utility class for Date, DateTime, TimeStamp, Interval data types
-public class DateUtility {
-
-
- /* We have a hashmap that stores the timezone as the key and an index as the value
- * While storing the timezone in value vectors, holders we only use this index. As we
- * reconstruct the timestamp, we use this index to index through the array timezoneList
- * and get the corresponding timezone and pass it to joda-time
- */
- public static ObjectIntOpenHashMap<String> timezoneMap = new ObjectIntOpenHashMap<String>();
-
- public static String[] timezoneList = {"Africa/Abidjan",
- "Africa/Accra",
- "Africa/Addis_Ababa",
- "Africa/Algiers",
- "Africa/Asmara",
- "Africa/Asmera",
- "Africa/Bamako",
- "Africa/Bangui",
- "Africa/Banjul",
- "Africa/Bissau",
- "Africa/Blantyre",
- "Africa/Brazzaville",
- "Africa/Bujumbura",
- "Africa/Cairo",
- "Africa/Casablanca",
- "Africa/Ceuta",
- "Africa/Conakry",
- "Africa/Dakar",
- "Africa/Dar_es_Salaam",
- "Africa/Djibouti",
- "Africa/Douala",
- "Africa/El_Aaiun",
- "Africa/Freetown",
- "Africa/Gaborone",
- "Africa/Harare",
- "Africa/Johannesburg",
- "Africa/Juba",
- "Africa/Kampala",
- "Africa/Khartoum",
- "Africa/Kigali",
- "Africa/Kinshasa",
- "Africa/Lagos",
- "Africa/Libreville",
- "Africa/Lome",
- "Africa/Luanda",
- "Africa/Lubumbashi",
- "Africa/Lusaka",
- "Africa/Malabo",
- "Africa/Maputo",
- "Africa/Maseru",
- "Africa/Mbabane",
- "Africa/Mogadishu",
- "Africa/Monrovia",
- "Africa/Nairobi",
- "Africa/Ndjamena",
- "Africa/Niamey",
- "Africa/Nouakchott",
- "Africa/Ouagadougou",
- "Africa/Porto-Novo",
- "Africa/Sao_Tome",
- "Africa/Timbuktu",
- "Africa/Tripoli",
- "Africa/Tunis",
- "Africa/Windhoek",
- "America/Adak",
- "America/Anchorage",
- "America/Anguilla",
- "America/Antigua",
- "America/Araguaina",
- "America/Argentina/Buenos_Aires",
- "America/Argentina/Catamarca",
- "America/Argentina/ComodRivadavia",
- "America/Argentina/Cordoba",
- "America/Argentina/Jujuy",
- "America/Argentina/La_Rioja",
- "America/Argentina/Mendoza",
- "America/Argentina/Rio_Gallegos",
- "America/Argentina/Salta",
- "America/Argentina/San_Juan",
- "America/Argentina/San_Luis",
- "America/Argentina/Tucuman",
- "America/Argentina/Ushuaia",
- "America/Aruba",
- "America/Asuncion",
- "America/Atikokan",
- "America/Atka",
- "America/Bahia",
- "America/Bahia_Banderas",
- "America/Barbados",
- "America/Belem",
- "America/Belize",
- "America/Blanc-Sablon",
- "America/Boa_Vista",
- "America/Bogota",
- "America/Boise",
- "America/Buenos_Aires",
- "America/Cambridge_Bay",
- "America/Campo_Grande",
- "America/Cancun",
- "America/Caracas",
- "America/Catamarca",
- "America/Cayenne",
- "America/Cayman",
- "America/Chicago",
- "America/Chihuahua",
- "America/Coral_Harbour",
- "America/Cordoba",
- "America/Costa_Rica",
- "America/Cuiaba",
- "America/Curacao",
- "America/Danmarkshavn",
- "America/Dawson",
- "America/Dawson_Creek",
- "America/Denver",
- "America/Detroit",
- "America/Dominica",
- "America/Edmonton",
- "America/Eirunepe",
- "America/El_Salvador",
- "America/Ensenada",
- "America/Fort_Wayne",
- "America/Fortaleza",
- "America/Glace_Bay",
- "America/Godthab",
- "America/Goose_Bay",
- "America/Grand_Turk",
- "America/Grenada",
- "America/Guadeloupe",
- "America/Guatemala",
- "America/Guayaquil",
- "America/Guyana",
- "America/Halifax",
- "America/Havana",
- "America/Hermosillo",
- "America/Indiana/Indianapolis",
- "America/Indiana/Knox",
- "America/Indiana/Marengo",
- "America/Indiana/Petersburg",
- "America/Indiana/Tell_City",
- "America/Indiana/Vevay",
- "America/Indiana/Vincennes",
- "America/Indiana/Winamac",
- "America/Indianapolis",
- "America/Inuvik",
- "America/Iqaluit",
- "America/Jamaica",
- "America/Jujuy",
- "America/Juneau",
- "America/Kentucky/Louisville",
- "America/Kentucky/Monticello",
- "America/Knox_IN",
- "America/Kralendijk",
- "America/La_Paz",
- "America/Lima",
- "America/Los_Angeles",
- "America/Louisville",
- "America/Lower_Princes",
- "America/Maceio",
- "America/Managua",
- "America/Manaus",
- "America/Marigot",
- "America/Martinique",
- "America/Matamoros",
- "America/Mazatlan",
- "America/Mendoza",
- "America/Menominee",
- "America/Merida",
- "America/Metlakatla",
- "America/Mexico_City",
- "America/Miquelon",
- "America/Moncton",
- "America/Monterrey",
- "America/Montevideo",
- "America/Montreal",
- "America/Montserrat",
- "America/Nassau",
- "America/New_York",
- "America/Nipigon",
- "America/Nome",
- "America/Noronha",
- "America/North_Dakota/Beulah",
- "America/North_Dakota/Center",
- "America/North_Dakota/New_Salem",
- "America/Ojinaga",
- "America/Panama",
- "America/Pangnirtung",
- "America/Paramaribo",
- "America/Phoenix",
- "America/Port-au-Prince",
- "America/Port_of_Spain",
- "America/Porto_Acre",
- "America/Porto_Velho",
- "America/Puerto_Rico",
- "America/Rainy_River",
- "America/Rankin_Inlet",
- "America/Recife",
- "America/Regina",
- "America/Resolute",
- "America/Rio_Branco",
- "America/Rosario",
- "America/Santa_Isabel",
- "America/Santarem",
- "America/Santiago",
- "America/Santo_Domingo",
- "America/Sao_Paulo",
- "America/Scoresbysund",
- "America/Shiprock",
- "America/Sitka",
- "America/St_Barthelemy",
- "America/St_Johns",
- "America/St_Kitts",
- "America/St_Lucia",
- "America/St_Thomas",
- "America/St_Vincent",
- "America/Swift_Current",
- "America/Tegucigalpa",
- "America/Thule",
- "America/Thunder_Bay",
- "America/Tijuana",
- "America/Toronto",
- "America/Tortola",
- "America/Vancouver",
- "America/Virgin",
- "America/Whitehorse",
- "America/Winnipeg",
- "America/Yakutat",
- "America/Yellowknife",
- "Antarctica/Casey",
- "Antarctica/Davis",
- "Antarctica/DumontDUrville",
- "Antarctica/Macquarie",
- "Antarctica/Mawson",
- "Antarctica/McMurdo",
- "Antarctica/Palmer",
- "Antarctica/Rothera",
- "Antarctica/South_Pole",
- "Antarctica/Syowa",
- "Antarctica/Vostok",
- "Arctic/Longyearbyen",
- "Asia/Aden",
- "Asia/Almaty",
- "Asia/Amman",
- "Asia/Anadyr",
- "Asia/Aqtau",
- "Asia/Aqtobe",
- "Asia/Ashgabat",
- "Asia/Ashkhabad",
- "Asia/Baghdad",
- "Asia/Bahrain",
- "Asia/Baku",
- "Asia/Bangkok",
- "Asia/Beirut",
- "Asia/Bishkek",
- "Asia/Brunei",
- "Asia/Calcutta",
- "Asia/Choibalsan",
- "Asia/Chongqing",
- "Asia/Chungking",
- "Asia/Colombo",
- "Asia/Dacca",
- "Asia/Damascus",
- "Asia/Dhaka",
- "Asia/Dili",
- "Asia/Dubai",
- "Asia/Dushanbe",
- "Asia/Gaza",
- "Asia/Harbin",
- "Asia/Hebron",
- "Asia/Ho_Chi_Minh",
- "Asia/Hong_Kong",
- "Asia/Hovd",
- "Asia/Irkutsk",
- "Asia/Istanbul",
- "Asia/Jakarta",
- "Asia/Jayapura",
- "Asia/Jerusalem",
- "Asia/Kabul",
- "Asia/Kamchatka",
- "Asia/Karachi",
- "Asia/Kashgar",
- "Asia/Kathmandu",
- "Asia/Katmandu",
- "Asia/Kolkata",
- "Asia/Krasnoyarsk",
- "Asia/Kuala_Lumpur",
- "Asia/Kuching",
- "Asia/Kuwait",
- "Asia/Macao",
- "Asia/Macau",
- "Asia/Magadan",
- "Asia/Makassar",
- "Asia/Manila",
- "Asia/Muscat",
- "Asia/Nicosia",
- "Asia/Novokuznetsk",
- "Asia/Novosibirsk",
- "Asia/Omsk",
- "Asia/Oral",
- "Asia/Phnom_Penh",
- "Asia/Pontianak",
- "Asia/Pyongyang",
- "Asia/Qatar",
- "Asia/Qyzylorda",
- "Asia/Rangoon",
- "Asia/Riyadh",
- "Asia/Saigon",
- "Asia/Sakhalin",
- "Asia/Samarkand",
- "Asia/Seoul",
- "Asia/Shanghai",
- "Asia/Singapore",
- "Asia/Taipei",
- "Asia/Tashkent",
- "Asia/Tbilisi",
- "Asia/Tehran",
- "Asia/Tel_Aviv",
- "Asia/Thimbu",
- "Asia/Thimphu",
- "Asia/Tokyo",
- "Asia/Ujung_Pandang",
- "Asia/Ulaanbaatar",
- "Asia/Ulan_Bator",
- "Asia/Urumqi",
- "Asia/Vientiane",
- "Asia/Vladivostok",
- "Asia/Yakutsk",
- "Asia/Yekaterinburg",
- "Asia/Yerevan",
- "Atlantic/Azores",
- "Atlantic/Bermuda",
- "Atlantic/Canary",
- "Atlantic/Cape_Verde",
- "Atlantic/Faeroe",
- "Atlantic/Faroe",
- "Atlantic/Jan_Mayen",
- "Atlantic/Madeira",
- "Atlantic/Reykjavik",
- "Atlantic/South_Georgia",
- "Atlantic/St_Helena",
- "Atlantic/Stanley",
- "Australia/ACT",
- "Australia/Adelaide",
- "Australia/Brisbane",
- "Australia/Broken_Hill",
- "Australia/Canberra",
- "Australia/Currie",
- "Australia/Darwin",
- "Australia/Eucla",
- "Australia/Hobart",
- "Australia/LHI",
- "Australia/Lindeman",
- "Australia/Lord_Howe",
- "Australia/Melbourne",
- "Australia/NSW",
- "Australia/North",
- "Australia/Perth",
- "Australia/Queensland",
- "Australia/South",
- "Australia/Sydney",
- "Australia/Tasmania",
- "Australia/Victoria",
- "Australia/West",
- "Australia/Yancowinna",
- "Brazil/Acre",
- "Brazil/DeNoronha",
- "Brazil/East",
- "Brazil/West",
- "CET",
- "CST6CDT",
- "Canada/Atlantic",
- "Canada/Central",
- "Canada/East-Saskatchewan",
- "Canada/Eastern",
- "Canada/Mountain",
- "Canada/Newfoundland",
- "Canada/Pacific",
- "Canada/Saskatchewan",
- "Canada/Yukon",
- "Chile/Continental",
- "Chile/EasterIsland",
- "Cuba",
- "EET",
- "EST",
- "EST5EDT",
- "Egypt",
- "Eire",
- "Etc/GMT",
- "Etc/GMT+0",
- "Etc/GMT+1",
- "Etc/GMT+10",
- "Etc/GMT+11",
- "Etc/GMT+12",
- "Etc/GMT+2",
- "Etc/GMT+3",
- "Etc/GMT+4",
- "Etc/GMT+5",
- "Etc/GMT+6",
- "Etc/GMT+7",
- "Etc/GMT+8",
- "Etc/GMT+9",
- "Etc/GMT-0",
- "Etc/GMT-1",
- "Etc/GMT-10",
- "Etc/GMT-11",
- "Etc/GMT-12",
- "Etc/GMT-13",
- "Etc/GMT-14",
- "Etc/GMT-2",
- "Etc/GMT-3",
- "Etc/GMT-4",
- "Etc/GMT-5",
- "Etc/GMT-6",
- "Etc/GMT-7",
- "Etc/GMT-8",
- "Etc/GMT-9",
- "Etc/GMT0",
- "Etc/Greenwich",
- "Etc/UCT",
- "Etc/UTC",
- "Etc/Universal",
- "Etc/Zulu",
- "Europe/Amsterdam",
- "Europe/Andorra",
- "Europe/Athens",
- "Europe/Belfast",
- "Europe/Belgrade",
- "Europe/Berlin",
- "Europe/Bratislava",
- "Europe/Brussels",
- "Europe/Bucharest",
- "Europe/Budapest",
- "Europe/Chisinau",
- "Europe/Copenhagen",
- "Europe/Dublin",
- "Europe/Gibraltar",
- "Europe/Guernsey",
- "Europe/Helsinki",
- "Europe/Isle_of_Man",
- "Europe/Istanbul",
- "Europe/Jersey",
- "Europe/Kaliningrad",
- "Europe/Kiev",
- "Europe/Lisbon",
- "Europe/Ljubljana",
- "Europe/London",
- "Europe/Luxembourg",
- "Europe/Madrid",
- "Europe/Malta",
- "Europe/Mariehamn",
- "Europe/Minsk",
- "Europe/Monaco",
- "Europe/Moscow",
- "Europe/Nicosia",
- "Europe/Oslo",
- "Europe/Paris",
- "Europe/Podgorica",
- "Europe/Prague",
- "Europe/Riga",
- "Europe/Rome",
- "Europe/Samara",
- "Europe/San_Marino",
- "Europe/Sarajevo",
- "Europe/Simferopol",
- "Europe/Skopje",
- "Europe/Sofia",
- "Europe/Stockholm",
- "Europe/Tallinn",
- "Europe/Tirane",
- "Europe/Tiraspol",
- "Europe/Uzhgorod",
- "Europe/Vaduz",
- "Europe/Vatican",
- "Europe/Vienna",
- "Europe/Vilnius",
- "Europe/Volgograd",
- "Europe/Warsaw",
- "Europe/Zagreb",
- "Europe/Zaporozhye",
- "Europe/Zurich",
- "GB",
- "GB-Eire",
- "GMT",
- "GMT+0",
- "GMT-0",
- "GMT0",
- "Greenwich",
- "HST",
- "Hongkong",
- "Iceland",
- "Indian/Antananarivo",
- "Indian/Chagos",
- "Indian/Christmas",
- "Indian/Cocos",
- "Indian/Comoro",
- "Indian/Kerguelen",
- "Indian/Mahe",
- "Indian/Maldives",
- "Indian/Mauritius",
- "Indian/Mayotte",
- "Indian/Reunion",
- "Iran",
- "Israel",
- "Jamaica",
- "Japan",
- "Kwajalein",
- "Libya",
- "MET",
- "MST",
- "MST7MDT",
- "Mexico/BajaNorte",
- "Mexico/BajaSur",
- "Mexico/General",
- "NZ",
- "NZ-CHAT",
- "Navajo",
- "PRC",
- "PST8PDT",
- "Pacific/Apia",
- "Pacific/Auckland",
- "Pacific/Chatham",
- "Pacific/Chuuk",
- "Pacific/Easter",
- "Pacific/Efate",
- "Pacific/Enderbury",
- "Pacific/Fakaofo",
- "Pacific/Fiji",
- "Pacific/Funafuti",
- "Pacific/Galapagos",
- "Pacific/Gambier",
- "Pacific/Guadalcanal",
- "Pacific/Guam",
- "Pacific/Honolulu",
- "Pacific/Johnston",
- "Pacific/Kiritimati",
- "Pacific/Kosrae",
- "Pacific/Kwajalein",
- "Pacific/Majuro",
- "Pacific/Marquesas",
- "Pacific/Midway",
- "Pacific/Nauru",
- "Pacific/Niue",
- "Pacific/Norfolk",
- "Pacific/Noumea",
- "Pacific/Pago_Pago",
- "Pacific/Palau",
- "Pacific/Pitcairn",
- "Pacific/Pohnpei",
- "Pacific/Ponape",
- "Pacific/Port_Moresby",
- "Pacific/Rarotonga",
- "Pacific/Saipan",
- "Pacific/Samoa",
- "Pacific/Tahiti",
- "Pacific/Tarawa",
- "Pacific/Tongatapu",
- "Pacific/Truk",
- "Pacific/Wake",
- "Pacific/Wallis",
- "Pacific/Yap",
- "Poland",
- "Portugal",
- "ROC",
- "ROK",
- "Singapore",
- "Turkey",
- "UCT",
- "US/Alaska",
- "US/Aleutian",
- "US/Arizona",
- "US/Central",
- "US/East-Indiana",
- "US/Eastern",
- "US/Hawaii",
- "US/Indiana-Starke",
- "US/Michigan",
- "US/Mountain",
- "US/Pacific",
- "US/Pacific-New",
- "US/Samoa",
- "UTC",
- "Universal",
- "W-SU",
- "WET",
- "Zulu"};
-
- static {
- for (int i = 0; i < timezoneList.length; i++) {
- timezoneMap.put(timezoneList[i], i);
- }
- }
-
- public static final DateTimeFormatter formatDate = DateTimeFormat.forPattern("yyyy-MM-dd");
- public static final DateTimeFormatter formatTimeStamp = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
- public static final DateTimeFormatter formatTimeStampTZ = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ZZZ");
- public static final DateTimeFormatter formatTime = DateTimeFormat.forPattern("HH:mm:ss.SSS");
-
- public static DateTimeFormatter dateTimeTZFormat = null;
- public static DateTimeFormatter timeFormat = null;
-
- public static final int yearsToMonths = 12;
- public static final int hoursToMillis = 60 * 60 * 1000;
- public static final int minutesToMillis = 60 * 1000;
- public static final int secondsToMillis = 1000;
- public static final int monthToStandardDays = 30;
- public static final long monthsToMillis = 2592000000L; // 30 * 24 * 60 * 60 * 1000
- public static final int daysToStandardMillis = 24 * 60 * 60 * 1000;
-
-
- public static int getIndex(String timezone) {
- return timezoneMap.get(timezone);
- }
-
- public static String getTimeZone(int index) {
- return timezoneList[index];
- }
-
- // Function returns the date time formatter used to parse date strings
- public static DateTimeFormatter getDateTimeFormatter() {
-
- if (dateTimeTZFormat == null) {
- DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");
- DateTimeParser optionalTime = DateTimeFormat.forPattern(" HH:mm:ss").getParser();
- DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
- DateTimeParser optionalZone = DateTimeFormat.forPattern(" ZZZ").getParser();
-
- dateTimeTZFormat = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).appendOptional(optionalZone).toFormatter();
- }
-
- return dateTimeTZFormat;
- }
-
- // Function returns time formatter used to parse time strings
- public static DateTimeFormatter getTimeFormatter() {
- if (timeFormat == null) {
- DateTimeFormatter timeFormatter = DateTimeFormat.forPattern("HH:mm:ss");
- DateTimeParser optionalSec = DateTimeFormat.forPattern(".SSS").getParser();
- timeFormat = new DateTimeFormatterBuilder().append(timeFormatter).appendOptional(optionalSec).toFormatter();
- }
- return timeFormat;
- }
-
- public static int monthsFromPeriod(Period period){
- return (period.getYears() * yearsToMonths) + period.getMonths();
- }
-
- public static int millisFromPeriod(final Period period){
- return (period.getHours() * hoursToMillis) +
- (period.getMinutes() * minutesToMillis) +
- (period.getSeconds() * secondsToMillis) +
- (period.getMillis());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
deleted file mode 100644
index ad6a787..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
+++ /dev/null
@@ -1,462 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.DrillBuf;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.proto.helper.QueryIdHelper;
-import org.apache.drill.exec.util.AssertionUtil;
-
-import com.google.common.collect.LinkedListMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-
-public class Accountor {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Accountor.class);
-
- private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
- private final AtomicRemainder remainder;
- private final long total;
- private ConcurrentMap<ByteBuf, DebugStackTrace> buffers = Maps.newConcurrentMap();
- private final FragmentHandle handle;
- private String fragmentStr;
- private Accountor parent;
-
- private final boolean errorOnLeak;
- // some operators are no subject to the fragment limit. They set the applyFragmentLimit to false
-
- private final boolean enableFragmentLimit;
- private final double fragmentMemOvercommitFactor;
-
- private final boolean DEFAULT_ENABLE_FRAGMENT_LIMIT=false;
- private final double DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR=1.5;
-
- private final boolean applyFragmentLimit;
-
- private final FragmentContext fragmentContext;
- long fragmentLimit;
-
- private long peakMemoryAllocation = 0;
-
- // The top level Allocator has an accountor that keeps track of all the FragmentContexts currently executing.
- // This enables the top level accountor to calculate a new fragment limit whenever necessary.
- private final List<FragmentContext> fragmentContexts;
-
- public Accountor(DrillConfig config, boolean errorOnLeak, FragmentContext context, Accountor parent, long max, long preAllocated, boolean applyFragLimit) {
- // TODO: fix preallocation stuff
- this.errorOnLeak = errorOnLeak;
- AtomicRemainder parentRemainder = parent != null ? parent.remainder : null;
- this.parent = parent;
-
- boolean enableFragmentLimit;
- double fragmentMemOvercommitFactor;
-
- try {
- enableFragmentLimit = config.getBoolean(ExecConstants.ENABLE_FRAGMENT_MEMORY_LIMIT);
- fragmentMemOvercommitFactor = config.getDouble(ExecConstants.FRAGMENT_MEM_OVERCOMMIT_FACTOR);
- }catch(Exception e){
- enableFragmentLimit = DEFAULT_ENABLE_FRAGMENT_LIMIT;
- fragmentMemOvercommitFactor = DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR;
- }
- this.enableFragmentLimit = enableFragmentLimit;
- this.fragmentMemOvercommitFactor = fragmentMemOvercommitFactor;
-
-
- this.applyFragmentLimit=applyFragLimit;
-
- this.remainder = new AtomicRemainder(errorOnLeak, parentRemainder, max, preAllocated, applyFragmentLimit);
- this.total = max;
- this.fragmentContext=context;
- this.handle = (context!=null) ? context.getHandle() : null;
- this.fragmentStr= (handle!=null) ? ( handle.getMajorFragmentId()+":"+handle.getMinorFragmentId() ) : "0:0";
- this.fragmentLimit=this.total; // Allow as much as possible to start with;
- if (ENABLE_ACCOUNTING) {
- buffers = Maps.newConcurrentMap();
- } else {
- buffers = null;
- }
- this.fragmentContexts = new ArrayList<FragmentContext>();
- if(parent!=null && parent.parent==null){ // Only add the fragment context to the fragment level accountor
- synchronized(this) {
- addFragmentContext(this.fragmentContext);
- }
- }
- }
-
- public boolean transferTo(Accountor target, DrillBuf buf, long size) {
- return transfer(target, buf, size, true);
- }
-
- public boolean transferIn(DrillBuf buf, long size) {
- return transfer(this, buf, size, false);
- }
-
- private boolean transfer(Accountor target, DrillBuf buf, long size, boolean release) {
- boolean withinLimit = target.forceAdditionalReservation(size);
- if(release){
- release(buf, size);
- }
-
- if (ENABLE_ACCOUNTING) {
- target.buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread().getStackTrace()));
- }
- return withinLimit;
- }
-
- public long getAvailable() {
- if (parent != null) {
- return Math.min(parent.getAvailable(), getCapacity() - getAllocation());
- }
- return getCapacity() - getAllocation();
- }
-
- public long getCapacity() {
- return fragmentLimit;
- }
-
- public long getAllocation() {
- return remainder.getUsed();
- }
-
- public long getPeakMemoryAllocation() {
- return peakMemoryAllocation;
- }
-
- public boolean reserve(long size) {
- logger.trace("Fragment:"+fragmentStr+" Reserved "+size+" bytes. Total Allocated: "+getAllocation());
- boolean status = remainder.get(size, this.applyFragmentLimit);
- peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
- return status;
- }
-
- public boolean forceAdditionalReservation(long size) {
- if (size > 0) {
- boolean status = remainder.forceGet(size);
- peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
- return status;
- } else {
- return true;
- }
- }
-
- public void reserved(long expected, DrillBuf buf) {
- // make sure to take away the additional memory that happened due to rounding.
-
- long additional = buf.capacity() - expected;
- if (additional > 0) {
- remainder.forceGet(additional);
- }
-
- if (ENABLE_ACCOUNTING) {
- buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread().getStackTrace()));
- }
-
- peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
- }
-
-
- public void releasePartial(DrillBuf buf, long size) {
- remainder.returnAllocation(size);
- if (ENABLE_ACCOUNTING) {
- if (buf != null) {
- DebugStackTrace dst = buffers.get(buf);
- if (dst == null) {
- throw new IllegalStateException("Partially releasing a buffer that has already been released. Buffer: " + buf);
- }
- dst.size -= size;
- if (dst.size < 0) {
- throw new IllegalStateException("Partially releasing a buffer that has already been released. Buffer: " + buf);
- }
- }
- }
- }
-
- void release(long size) {
- remainder.returnAllocation(size);
- }
-
- public void release(DrillBuf buf, long size) {
- remainder.returnAllocation(size);
- if (ENABLE_ACCOUNTING) {
- if (buf != null && buffers.remove(buf) == null) {
- throw new IllegalStateException("Releasing a buffer that has already been released. Buffer: " + buf);
- }
- }
- }
-
- private void addFragmentContext(FragmentContext c) {
- if (parent != null){
- parent.addFragmentContext(c);
- }else {
- if(logger.isTraceEnabled()) {
- FragmentHandle hndle;
- String fragStr;
- if(c!=null) {
- hndle = c.getHandle();
- fragStr = (hndle != null) ? (hndle.getMajorFragmentId() + ":" + hndle.getMinorFragmentId()) : "[Null Fragment Handle]";
- }else{
- fragStr = "[Null Context]";
- }
- fragStr+=" (Object Id: "+System.identityHashCode(c)+")";
- StackTraceElement[] ste = (new Throwable()).getStackTrace();
- StringBuffer sb = new StringBuffer();
- for (StackTraceElement s : ste) {
- sb.append(s.toString());
- sb.append("\n");
- }
-
- logger.trace("Fragment " + fragStr + " added to root accountor.\n"+sb.toString());
- }
- synchronized(this) {
- fragmentContexts.add(c);
- }
- }
- }
-
- private void removeFragmentContext(FragmentContext c) {
- if (parent != null){
- if (parent.parent==null){
- // only fragment level allocators will have the fragment context saved
- parent.removeFragmentContext(c);
- }
- }else{
- if(logger.isDebugEnabled()) {
- FragmentHandle hndle;
- String fragStr;
- if (c != null) {
- hndle = c.getHandle();
- fragStr = (hndle != null) ? (hndle.getMajorFragmentId() + ":" + hndle.getMinorFragmentId()) : "[Null Fragment Handle]";
- } else {
- fragStr = "[Null Context]";
- }
- fragStr += " (Object Id: " + System.identityHashCode(c) + ")";
- logger.trace("Fragment " + fragStr + " removed from root accountor");
- }
- synchronized(this) {
- fragmentContexts.remove(c);
- }
- }
- }
-
- public long resetFragmentLimits(){
- // returns the new capacity
- if(!this.enableFragmentLimit){
- return getCapacity();
- }
-
- if(parent!=null){
- parent.resetFragmentLimits();
- }else {
- //Get remaining memory available per fragment and distribute it EQUALLY among all the fragments.
- //Fragments get the memory limit added to the amount already allocated.
- //This favours fragments that are already running which will get a limit greater than newly started fragments.
- //If the already running fragments end quickly, their limits will be assigned back to the remaining fragments
- //quickly. If they are long running, then we want to favour them with larger limits anyway.
- synchronized (this) {
- int nFragments=fragmentContexts.size();
- long allocatedMemory=0;
- for(FragmentContext fragment: fragmentContexts){
- BufferAllocator a = fragment.getAllocator();
- if(a!=null) {
- allocatedMemory += fragment.getAllocator().getAllocatedMemory();
- }
- }
- if(logger.isTraceEnabled()) {
- logger.trace("Resetting Fragment Memory Limit: total Available memory== "+total
- +" Total Allocated Memory :"+allocatedMemory
- +" Number of fragments: "+nFragments
- + " fragmentMemOvercommitFactor: "+fragmentMemOvercommitFactor
- + " Root fragment limit: "+this.fragmentLimit + "(Root obj: "+System.identityHashCode(this)+")"
- );
- }
- if(nFragments>0) {
- long rem = (total - allocatedMemory) / nFragments;
- for (FragmentContext fragment : fragmentContexts) {
- fragment.setFragmentLimit((long) (rem * fragmentMemOvercommitFactor));
- }
- }
- if(logger.isTraceEnabled() && false){
- StringBuffer sb= new StringBuffer();
- sb.append("[root](0:0)");
- sb.append("Allocated memory: ");
- sb.append(this.getAllocation());
- sb.append(" Fragment Limit: ");
- sb.append(this.getFragmentLimit());
- logger.trace(sb.toString());
- for(FragmentContext fragment: fragmentContexts){
- sb= new StringBuffer();
- if (handle != null) {
- sb.append("[");
- sb.append(QueryIdHelper.getQueryId(handle.getQueryId()));
- sb.append("](");
- sb.append(handle.getMajorFragmentId());
- sb.append(":");
- sb.append(handle.getMinorFragmentId());
- sb.append(")");
- }else{
- sb.append("[fragment](0:0)");
- }
- sb.append("Allocated memory: ");
- sb.append(fragment.getAllocator().getAllocatedMemory());
- sb.append(" Fragment Limit: ");
- sb.append(fragment.getAllocator().getFragmentLimit());
- logger.trace(sb.toString());
- }
- logger.trace("Resetting Complete");
- }
- }
- }
- return getCapacity();
- }
-
- public void close() {
- // remove the fragment context and reset fragment limits whenever an allocator closes
- if(parent!=null && parent.parent==null) {
- logger.debug("Fragment " + fragmentStr + " accountor being closed");
- removeFragmentContext(fragmentContext);
- }
- resetFragmentLimits();
-
- if (ENABLE_ACCOUNTING && !buffers.isEmpty()) {
- StringBuffer sb = new StringBuffer();
- sb.append("Attempted to close accountor with ");
- sb.append(buffers.size());
- sb.append(" buffer(s) still allocated");
- if (handle != null) {
- sb.append("for QueryId: ");
- sb.append(QueryIdHelper.getQueryId(handle.getQueryId()));
- sb.append(", MajorFragmentId: ");
- sb.append(handle.getMajorFragmentId());
- sb.append(", MinorFragmentId: ");
- sb.append(handle.getMinorFragmentId());
- }
- sb.append(".\n");
-
- Multimap<DebugStackTrace, DebugStackTrace> multi = LinkedListMultimap.create();
- for (DebugStackTrace t : buffers.values()) {
- multi.put(t, t);
- }
-
- for (DebugStackTrace entry : multi.keySet()) {
- Collection<DebugStackTrace> allocs = multi.get(entry);
-
- sb.append("\n\n\tTotal ");
- sb.append(allocs.size());
- sb.append(" allocation(s) of byte size(s): ");
- for (DebugStackTrace alloc : allocs) {
- sb.append(alloc.size);
- sb.append(", ");
- }
-
- sb.append("at stack location:\n");
- entry.addToString(sb);
- }
- if (!buffers.isEmpty()) {
- IllegalStateException e = new IllegalStateException(sb.toString());
- if (errorOnLeak) {
- throw e;
- } else {
- logger.warn("Memory leaked.", e);
- }
- }
- }
-
- remainder.close();
-
- }
-
- public void setFragmentLimit(long add) {
- // We ADD the limit to the current allocation. If none has been allocated, this
- // sets a new limit. If memory has already been allocated, the fragment gets its
- // limit based on the allocation, though this might still result in reducing the
- // limit.
-
- if (parent != null && parent.parent==null) { // This is a fragment level accountor
- this.fragmentLimit=getAllocation()+add;
- this.remainder.setLimit(this.fragmentLimit);
- logger.trace("Fragment "+fragmentStr+" memory limit set to "+this.fragmentLimit);
- }
- }
-
- public long getFragmentLimit(){
- return this.fragmentLimit;
- }
-
- public class DebugStackTrace {
-
- private StackTraceElement[] elements;
- private long size;
-
- public DebugStackTrace(long size, StackTraceElement[] elements) {
- super();
- this.elements = elements;
- this.size = size;
- }
-
- public void addToString(StringBuffer sb) {
- for (int i = 3; i < elements.length; i++) {
- sb.append("\t\t");
- sb.append(elements[i]);
- sb.append("\n");
- }
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(elements);
-// result = prime * result + (int) (size ^ (size >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- DebugStackTrace other = (DebugStackTrace) obj;
- if (!Arrays.equals(elements, other.elements)) {
- return false;
- }
- // weird equal where size doesn't matter for multimap purposes.
-// if (size != other.size)
-// return false;
- return true;
- }
-
- }
-
-}
[29/45] drill git commit: DRILL-3987: (CLEANUP) Delete unused files
Posted by ja...@apache.org.
DRILL-3987: (CLEANUP) Delete unused files
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/363bf33d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/363bf33d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/363bf33d
Branch: refs/heads/master
Commit: 363bf33d1323157818582ad66a3d3b37eadeaa77
Parents: 62353a6
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Nov 8 08:20:19 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 17:00:48 2015 -0800
----------------------------------------------------------------------
.../drill/common/storage/proto/ProtoLexer.g.bk | 287 ---------
.../drill/common/storage/proto/ProtoParser.g.bk | 605 -------------------
2 files changed, 892 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/363bf33d/common/src/main/antlr3/org/apache/drill/common/storage/proto/ProtoLexer.g.bk
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/storage/proto/ProtoLexer.g.bk b/common/src/main/antlr3/org/apache/drill/common/storage/proto/ProtoLexer.g.bk
deleted file mode 100644
index 81b3e03..0000000
--- a/common/src/main/antlr3/org/apache/drill/common/storage/proto/ProtoLexer.g.bk
+++ /dev/null
@@ -1,287 +0,0 @@
-//========================================================================
-//Copyright 2007-2009 David Yu dyuproject@gmail.com
-//------------------------------------------------------------------------
-//Licensed under the Apache License, Version 2.0 (the "License");
-//you may not use this file except in compliance with the License.
-//You may obtain a copy of the License at
-//http://www.apache.org/licenses/LICENSE-2.0
-//Unless required by applicable law or agreed to in writing, software
-//distributed under the License is distributed on an "AS IS" BASIS,
-//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//See the License for the specific language governing permissions and
-//limitations under the License.
-//========================================================================
-
-/** Apache Drill notes: Based off r1586 of protostuff **/
-
-lexer grammar ProtoLexer;
-
-options {
- language=Java; // Default
-}
-
-@header {
- package org.apache.drill.common.storage.proto;
-}
-
-ASSIGN
- : '='
- ;
-
-AT
- : '@'
- ;
-
-LEFTCURLY
- : '{'
- ;
-
-RIGHTCURLY
- : '}'
- ;
-
-LEFTPAREN
- : '('
- ;
-
-RIGHTPAREN
- : ')'
- ;
-
-LEFTSQUARE
- : '['
- ;
-
-RIGHTSQUARE
- : ']'
- ;
-
-SEMICOLON
- : ';'
- ;
-
-COMMA
- : ','
- ;
-
-PLUS
- : '+'
- ;
-
-MINUS
- : '-'
- ;
-
-TO
- : 'to'
- ;
-
-TRUE
- : 'true'
- ;
-
-FALSE
- : 'false'
- ;
-
-PKG
- : 'package'
- ;
-
-SYNTAX
- : 'syntax'
- ;
-
-IMPORT
- : 'import'
- ;
-
-OPTION
- : 'option'
- ;
-
-MESSAGE
- : 'message'
- ;
-
-SERVICE
- : 'service'
- ;
-
-ENUM
- : 'enum'
- ;
-
-REQUIRED
- : 'required'
- ;
-
-OPTIONAL
- : 'optional'
- ;
-
-REPEATED
- : 'repeated'
- ;
-
-EXTENSIONS
- : 'extensions'
- ;
-
-EXTEND
- : 'extend'
- ;
-
-GROUP
- : 'group'
- ;
-
-RPC
- : 'rpc'
- ;
-
-RETURNS
- : 'returns'
- ;
-
-INT32
- : 'int32'
- ;
-
-INT64
- : 'int64'
- ;
-
-UINT32
- : 'uint32'
- ;
-
-UINT64
- : 'uint64'
- ;
-
-SINT32
- : 'sint32'
- ;
-
-SINT64
- : 'sint64'
- ;
-
-FIXED32
- : 'fixed32'
- ;
-
-FIXED64
- : 'fixed64'
- ;
-
-SFIXED32
- : 'sfixed32'
- ;
-
-SFIXED64
- : 'sfixed64'
- ;
-
-FLOAT
- : 'float'
- ;
-
-DOUBLE
- : 'double'
- ;
-
-BOOL
- : 'bool'
- ;
-
-STRING
- : 'string'
- ;
-
-BYTES
- : 'bytes'
- ;
-
-DEFAULT
- : 'default'
- ;
-
-MAX
- : 'max'
- ;
-
-VOID
- : 'void'
- ;
-
-FULL_ID
- : ID ('.' ID)+
- ;
-
-ID
- : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
- ;
-
-EXP
- : NUMINT ('e'|'E') NUMINT
- ;
-
-NUMDOUBLE
- : (NUMFLOAT|NUMINT)
- 'e' '0'..'9'+
- ;
-
-NUMFLOAT
- : NUMINT '.' '0'..'9'+ 'f'?
- ;
-
-NUMINT
- : '0' | MINUS? '1'..'9' '0'..'9'*
- ;
-
-HEX
- : MINUS? '0' ('x'|'X') HEX_DIGIT+
- ;
-
-OCTAL
- : '0' ('0'..'7')+
- ;
-
-COMMENT
- : '//' ~('\n'|'\r')* '\r'? '\n' {skip();}
- | '/*' ( options {greedy=false;} : . )* '*/' {skip();}
- ;
-
-WS : ( ' '
- | '\t'
- | '\r'
- | '\n'
- ) {skip();}
- ;
-
-STRING_LITERAL
- : '"' ( ESC_SEQ | ~('\\'|'"') )* '"'
- ;
-
-fragment HEX_DIGIT
- : ('0'..'9'|'a'..'f'|'A'..'F')
- ;
-
-fragment ESC_SEQ
- : '\\' ('a'|'v'|'b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\')
- | '\\' ('x'|'X') HEX_DIGIT HEX_DIGIT
- | UNICODE_ESC
- | OCTAL_ESC
- ;
-
-fragment OCTAL_ESC
- : '\\' ('0'..'3') ('0'..'7') ('0'..'7')
- | '\\' ('0'..'7') ('0'..'7')
- | '\\' ('0'..'7')
- ;
-
-fragment UNICODE_ESC
- : '\\' 'u' HEX_DIGIT HEX_DIGIT HEX_DIGIT HEX_DIGIT
- ;
http://git-wip-us.apache.org/repos/asf/drill/blob/363bf33d/common/src/main/antlr3/org/apache/drill/common/storage/proto/ProtoParser.g.bk
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/storage/proto/ProtoParser.g.bk b/common/src/main/antlr3/org/apache/drill/common/storage/proto/ProtoParser.g.bk
deleted file mode 100644
index 81b6eb6..0000000
--- a/common/src/main/antlr3/org/apache/drill/common/storage/proto/ProtoParser.g.bk
+++ /dev/null
@@ -1,605 +0,0 @@
-//========================================================================
-//Copyright 2007-2009 David Yu dyuproject@gmail.com
-//------------------------------------------------------------------------
-//Licensed under the Apache License, Version 2.0 (the "License");
-//you may not use this file except in compliance with the License.
-//You may obtain a copy of the License at
-//http://www.apache.org/licenses/LICENSE-2.0
-//Unless required by applicable law or agreed to in writing, software
-//distributed under the License is distributed on an "AS IS" BASIS,
-//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//See the License for the specific language governing permissions and
-//limitations under the License.
-//========================================================================
-
-/** Apache Drill notes: Based off r1586 of protostuff **/
-
-parser grammar ProtoParser;
-
-options {
- language = Java;
- output = AST;
- tokenVocab = ProtoLexer;
-}
-
-@header {
- package org.apache.drill.common.storage.proto;
-}
-
-parse [Proto proto]
- : (statement[proto])+ EOF! {
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
-
- proto.postParse();
- }
- ;
-
-statement [Proto proto]
- : header_syntax[proto]
- | header_package[proto]
- | header_import[proto]
- | message_block[proto, null]
- | enum_block[proto, null]
- | extend_block[proto, null]
- | service_block[proto]
- | annotation_entry[proto]
- | option_entry[proto, proto]
- ;
-
-// some keywords that might possibly be used as a variable
-var
- : ID | TO | PKG | SYNTAX | IMPORT | OPTION | MESSAGE | SERVICE | ENUM |
- REQUIRED | OPTIONAL | REPEATED | EXTENSIONS | EXTEND | GROUP | RPC |
- RETURNS | INT32 | INT64 | UINT32 | UINT64 | SINT32 | SINT64 |
- FIXED32 | FIXED64 | SFIXED32 | SFIXED64 | FLOAT | DOUBLE | BOOL |
- STRING | BYTES | DEFAULT | MAX | VOID
- ;
-
-annotation_entry [Proto proto]
-@init {
- Annotation annotation = null;
-}
- : AT var { annotation = new Annotation($var.text); }
- (LEFTPAREN
- annotation_keyval[proto, annotation] (COMMA annotation_keyval[proto, annotation])*
- RIGHTPAREN)? {
- proto.add(annotation);
- }
- ;
-
-annotation_keyval [Proto proto, Annotation annotation]
- : k=var ASSIGN (
- ID { annotation.putRef($k.text, $ID.text); }
- | fid=FULL_ID { annotation.putRef($k.text, $fid.text); }
- | NUMFLOAT { annotation.put($k.text, Float.valueOf($NUMFLOAT.text)); }
- | NUMINT { annotation.put($k.text, Integer.valueOf($NUMINT.text)); }
- | NUMDOUBLE { annotation.put($k.text, Double.valueOf($NUMDOUBLE.text)); }
- | TRUE { annotation.put($k.text, Boolean.TRUE); }
- | FALSE { annotation.put($k.text, Boolean.FALSE); }
- | STRING_LITERAL { annotation.put($k.text, getStringFromStringLiteral($STRING_LITERAL.text)); }
- | v=var { annotation.put($k.text, $v.text); }
- )
- ;
-
-header_syntax [Proto proto]
- : SYNTAX ASSIGN STRING_LITERAL SEMICOLON! {
- if(!"proto2".equals(getStringFromStringLiteral($STRING_LITERAL.text))) {
- throw new IllegalStateException("Syntax isn't proto2: '" +
- getStringFromStringLiteral($STRING_LITERAL.text)+"'");
- }
-
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
- }
- ;
-
-header_package [Proto proto] // ignore
- : PKG (FULL_ID | var ) SEMICOLON!
- ;
-
-header_import [Proto proto]
- : IMPORT STRING_LITERAL SEMICOLON! {
- proto.importProto(getStringFromStringLiteral($STRING_LITERAL.text));
-
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
- }
- ;
-
-option_entry [Proto proto, HasOptions ho]
- : OPTION LEFTPAREN? k=(ID|FULL_ID) RIGHTPAREN? ASSIGN (
- id=ID { ho.putStandardOption($k.text, $id.text); }
- | fid=FULL_ID { ho.putStandardOption($k.text, $fid.text); }
- | NUMFLOAT { ho.putExtraOption($k.text, Float.valueOf($NUMFLOAT.text)); }
- | NUMINT { ho.putExtraOption($k.text, Integer.valueOf($NUMINT.text)); }
- | NUMDOUBLE { ho.putExtraOption($k.text, Double.valueOf($NUMDOUBLE.text)); }
- | TRUE { ho.putExtraOption($k.text, Boolean.TRUE); }
- | FALSE { ho.putExtraOption($k.text, Boolean.FALSE); }
- | STRING_LITERAL { ho.putExtraOption($k.text, getStringFromStringLiteral($STRING_LITERAL.text)); }
- ) SEMICOLON! {
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
- }
- ;
-
-message_block [Proto proto, Message parent]
-@init {
- Message message = null;
-}
- : MESSAGE ID {
- message = new Message($ID.text);
- if(parent==null)
- proto.addMessage(message);
- else
- parent.addNestedMessage(message);
-
- proto.addAnnotationsTo(message);
- }
- LEFTCURLY (message_body[proto, message])* RIGHTCURLY {
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
- }
- ;
-
-message_body [Proto proto, Message message]
- : message_block[proto, message]
- | message_field[proto, message]
- | enum_block[proto, message]
- | extend_block[proto, message]
- | extensions_range[proto, message]
- | annotation_entry[proto]
- | option_entry[proto, message]
- ;
-
-extensions_range [Proto proto, Message message]
-@init {
- int first = -1;
- int last = -1;
-}
- : EXTENSIONS f=NUMINT { first = Integer.parseInt($f.text); last = first;}
- ( TO ( l=NUMINT { last = Integer.parseInt($l.text); } | MAX {last = 536870911; } ) )?
- SEMICOLON! {
- message.defineExtensionRange(first, last);
- }
- ;
-
-message_field [Proto proto, HasFields message]
-@init {
- Field.Modifier modifier = null;
- FieldHolder fieldHolder = null;
-}
- : (OPTIONAL { modifier = Field.Modifier.OPTIONAL; }
- | REQUIRED { modifier = Field.Modifier.REQUIRED; }
- | REPEATED { modifier = Field.Modifier.REPEATED; }) {
- fieldHolder = new FieldHolder();
- }
- field_type[proto, message, fieldHolder]
- var ASSIGN NUMINT {
- if(fieldHolder.field != null) {
- fieldHolder.field.modifier = modifier;
- fieldHolder.field.name = $var.text;
- fieldHolder.field.number = Integer.parseInt($NUMINT.text);
- }
- }
- (field_options[proto, message, fieldHolder.field])? {
- if(fieldHolder.field != null) {
- proto.addAnnotationsTo(fieldHolder.field, message.getEnclosingNamespace());
- message.addField(fieldHolder.field);
- }
- }
- (SEMICOLON! | ignore_block)
- ;
-
-field_type [Proto proto, HasFields message, FieldHolder fieldHolder]
- : INT32 { fieldHolder.setField(new Field.Int32()); }
- | UINT32 { fieldHolder.setField(new Field.UInt32()); }
- | SINT32 { fieldHolder.setField(new Field.SInt32()); }
- | FIXED32 { fieldHolder.setField(new Field.Fixed32()); }
- | SFIXED32 { fieldHolder.setField(new Field.SFixed32()); }
- | INT64 { fieldHolder.setField(new Field.Int64()); }
- | UINT64 { fieldHolder.setField(new Field.UInt64()); }
- | SINT64 { fieldHolder.setField(new Field.SInt64()); }
- | FIXED64 { fieldHolder.setField(new Field.Fixed64()); }
- | SFIXED64 { fieldHolder.setField(new Field.SFixed64()); }
- | FLOAT { fieldHolder.setField(new Field.Float()); }
- | DOUBLE { fieldHolder.setField(new Field.Double()); }
- | BOOL { fieldHolder.setField(new Field.Bool()); }
- | STRING { fieldHolder.setField(new Field.String()); }
- | BYTES { fieldHolder.setField(new Field.Bytes()); }
- | GROUP {
- String suffix = proto.getFile()==null ? "" : " of " + proto.getFile().getName();
- warn("'group' not supported @ line " + $GROUP.line + suffix);
- }
- | FULL_ID {
- String fullType = $FULL_ID.text;
- int lastDot = fullType.lastIndexOf('.');
- String packageName = fullType.substring(0, lastDot);
- String type = fullType.substring(lastDot+1);
- fieldHolder.setField(new Field.Reference(packageName, type, message));
- }
- | ID {
- String type = $ID.text;
- fieldHolder.setField(new Field.Reference(null, type, message));
- }
- ;
-
-field_options [Proto proto, HasFields message, Field field]
- : LEFTSQUARE field_options_keyval[proto, message, field]
- (COMMA field_options_keyval[proto, message, field])* RIGHTSQUARE
- ;
-
-field_options_keyval [Proto proto, HasFields message, Field field]
- : key=(DEFAULT|ID) ASSIGN (STRING_LITERAL {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.String)
- field.defaultValue = getStringFromStringLiteral($STRING_LITERAL.text);
- else if(field instanceof Field.Bytes)
- field.defaultValue = getBytesFromStringLiteral($STRING_LITERAL.text);
- else
- throw new IllegalStateException("Invalid string default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
-
- field.putExtraOption($key.text, getStringFromStringLiteral($STRING_LITERAL.text));
- }
- | NUMFLOAT {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.Float)
- field.defaultValue = Float.valueOf($NUMFLOAT.text);
- else if(field instanceof Field.Double)
- field.defaultValue = Double.valueOf($NUMFLOAT.text);
- else
- throw new IllegalStateException("Invalid float default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
-
- field.putExtraOption($key.text, Float.valueOf($NUMFLOAT.text));
- }
- | NUMINT {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.Number) {
- if(field.getClass().getSimpleName().endsWith("32"))
- field.defaultValue = Integer.valueOf($NUMINT.text);
- else if(field.getClass().getSimpleName().endsWith("64"))
- field.defaultValue = Long.valueOf($NUMINT.text);
- else if(field instanceof Field.Float)
- field.defaultValue = Float.valueOf($NUMINT.text);
- else if(field instanceof Field.Double)
- field.defaultValue = Double.valueOf($NUMINT.text);
- else
- throw new IllegalStateException("Invalid numeric default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
- else
- throw new IllegalStateException("Invalid numeric default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
-
- field.putExtraOption($key.text, Integer.valueOf($NUMINT.text));
- }
- | NUMDOUBLE {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.Float)
- field.defaultValue = Float.valueOf($NUMDOUBLE.text);
- else if(field instanceof Field.Double)
- field.defaultValue = Double.valueOf($NUMDOUBLE.text);
- else
- throw new IllegalStateException("Invalid numeric default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
-
- field.putExtraOption($key.text, Double.valueOf($NUMDOUBLE.text));
- }
- | HEX {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.Number) {
- if(field instanceof Field.Int32)
- field.defaultValue = new Integer(TextFormat.parseInt32($HEX.text));
- else if(field instanceof Field.UInt32)
- field.defaultValue = new Integer(TextFormat.parseUInt32($HEX.text));
- else if(field instanceof Field.Int64)
- field.defaultValue = new Long(TextFormat.parseInt64($HEX.text));
- else if(field instanceof Field.UInt64)
- field.defaultValue = new Long(TextFormat.parseUInt64($HEX.text));
- else if(field instanceof Field.Float)
- field.defaultValue = new Float(Long.decode($HEX.text).floatValue());
- else if(field instanceof Field.Double)
- field.defaultValue = new Double(Long.decode($HEX.text).doubleValue());
- }
- else if(field instanceof Field.Bytes) {
- field.defaultValue = getBytesFromHexString($HEX.text);
- }
- else
- throw new IllegalStateException("Invalid numeric default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
-
- }
-
- field.putExtraOption($key.text, $HEX.text);
- }
- | OCTAL {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.Number) {
- if(field instanceof Field.Int32)
- field.defaultValue = new Integer(TextFormat.parseInt32($OCTAL.text));
- else if(field instanceof Field.UInt32)
- field.defaultValue = new Integer(TextFormat.parseUInt32($OCTAL.text));
- else if(field instanceof Field.Int64)
- field.defaultValue = new Long(TextFormat.parseInt64($OCTAL.text));
- else if(field instanceof Field.UInt64)
- field.defaultValue = new Long(TextFormat.parseUInt64($OCTAL.text));
- else if(field instanceof Field.Float)
- field.defaultValue = new Float(Long.decode($OCTAL.text).floatValue());
- else if(field instanceof Field.Double)
- field.defaultValue = new Double(Long.decode($OCTAL.text).doubleValue());
- }
- else
- throw new IllegalStateException("Invalid numeric default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
-
- field.putExtraOption($key.text, $OCTAL.text);
- }
- | TRUE {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.Bool)
- field.defaultValue = Boolean.TRUE;
- else
- throw new IllegalStateException("invalid boolean default value for the non-boolean field: " + field.getClass().getSimpleName() + " " + field.name);
- }
-
- field.putExtraOption($key.text, Boolean.TRUE);
- }
- | FALSE {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.Bool)
- field.defaultValue = Boolean.FALSE;
- else
- throw new IllegalStateException("invalid boolean default value for the non-boolean field: " + field.getClass().getSimpleName() + " " + field.name);
- }
-
- field.putExtraOption($key.text, Boolean.FALSE);
- }
- | val=ID {
- boolean refOption = false;
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- String refName = $val.text;
- if(field instanceof Field.Reference)
- field.defaultValue = refName;
- else if(field instanceof Field.Float) {
- if("inf".equals(refName)) {
- field.defaultValue = Float.POSITIVE_INFINITY;
- field.defaultValueConstant = "Float.POSITIVE_INFINITY";
- }
- else if("nan".equals(refName)) {
- field.defaultValue = Float.NaN;
- field.defaultValueConstant = "Float.NaN";
- }
- else
- throw new IllegalStateException("Invalid float default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
- else if(field instanceof Field.Double) {
- if("inf".equals(refName)) {
- field.defaultValue = Double.POSITIVE_INFINITY;
- field.defaultValueConstant = "Double.POSITIVE_INFINITY";
- }
- else if("nan".equals(refName)) {
- field.defaultValue = Double.NaN;
- field.defaultValueConstant = "Double.NaN";
- }
- else
- throw new IllegalStateException("Invalid double default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
- else {
- refOption = true;
- //throw new IllegalStateException("invalid field value '" + refName + "' for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
- }
- else {
- refOption = true;
- }
-
- if(refOption)
- field.putStandardOption($key.text, $val.text);
- else
- field.putExtraOption($key.text, $val.text);
- }
- | FULL_ID {
- field.putStandardOption($key.text, $FULL_ID.text);
- }
- | EXP {
- if("default".equals($key.text)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- if(field instanceof Field.Float)
- field.defaultValue = Float.valueOf($EXP.text);
- else if(field instanceof Field.Double)
- field.defaultValue = Double.valueOf($EXP.text);
- else
- throw new IllegalStateException("Invalid float default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
-
- field.putExtraOption($key.text, $EXP.text);
- }
- | signed_constant[proto, message, field, $key.text] {
- field.putExtraOption($key.text, $signed_constant.text);
- }
- )
- ;
-
-signed_constant [Proto proto, HasFields message, Field field, String key]
- : MINUS ID {
- if("default".equals(key)) {
- if(field.defaultValue!=null || field.modifier == Field.Modifier.REPEATED)
- throw new IllegalStateException("a field can only have a single default value");
-
- String refName = $ID.text;
- if(field instanceof Field.Float) {
- if("inf".equals(refName)) {
- field.defaultValue = Float.NEGATIVE_INFINITY;
- field.defaultValueConstant = "Float.NEGATIVE_INFINITY";
- }
- else
- throw new IllegalStateException("Invalid float default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
- else if(field instanceof Field.Double) {
- if("inf".equals(refName)) {
- field.defaultValue = Double.NEGATIVE_INFINITY;
- field.defaultValueConstant = "Double.NEGATIVE_INFINITY";
- }
- else
- throw new IllegalStateException("Invalid double default value for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
- else
- throw new IllegalStateException("invalid field value '" + refName + "' for the field: " + field.getClass().getSimpleName() + " " + field.name);
- }
- }
- ;
-
-enum_block [Proto proto, Message message]
-@init {
- EnumGroup enumGroup = null;
-}
- : ENUM ID {
- enumGroup = new EnumGroup($ID.text);
- if(message==null)
- proto.addEnumGroup(enumGroup);
- else
- message.addNestedEnumGroup(enumGroup);
-
- proto.addAnnotationsTo(enumGroup);
- }
- LEFTCURLY (enum_body[proto, message, enumGroup])* RIGHTCURLY {
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
- } (SEMICOLON?)!
- ;
-
-enum_body [Proto proto, Message message, EnumGroup enumGroup]
- : enum_field[proto, message, enumGroup]
- | annotation_entry[proto]
- | option_entry[proto, enumGroup]
- ;
-
-enum_field [Proto proto, Message message, EnumGroup enumGroup]
- : ID ASSIGN NUMINT SEMICOLON! {
- EnumGroup.Value v = new EnumGroup.Value($ID.text, Integer.parseInt($NUMINT.text));
- enumGroup.add(v);
- proto.addAnnotationsTo(v);
- }
- ;
-
-service_block [Proto proto]
-@init {
- Service service = null;
-}
- : SERVICE ID {
- service = new Service($ID.text, proto);
- proto.addAnnotationsTo(service);
- } LEFTCURLY
- (service_body[proto, service])+ RIGHTCURLY (SEMICOLON?)! {
- if(service.rpcMethods.isEmpty())
- throw new IllegalStateException("Empty Service block: " + service.getName());
-
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
- }
- ;
-
-service_body [Proto proto, Service service]
- : rpc_block[proto, service]
- | annotation_entry[proto]
- | option_entry[proto, service]
- ;
-
-rpc_block [Proto proto, Service service]
-@init {
- String argName = null, argPackage = null, retName = null, retPackage = null;
- Service.RpcMethod rm = null;
-}
- : RPC n=ID LEFTPAREN (ap=FULL_ID {
- String argFull = $ap.text;
- int lastDot = argFull.lastIndexOf('.');
- argPackage = argFull.substring(0, lastDot);
- argName = argFull.substring(lastDot+1);
- } | a=(VOID|ID) { argName = $a.text; }) RIGHTPAREN
- RETURNS LEFTPAREN (rp=FULL_ID {
- String retFull = $rp.text;
- int lastDot = retFull.lastIndexOf('.');
- retPackage = retFull.substring(0, lastDot);
- retName = retFull.substring(lastDot+1);
- } | r=(VOID|ID) { retName = $r.text; }) RIGHTPAREN {
- rm = service.addRpcMethod($n.text, argName, argPackage, retName, retPackage);
- proto.addAnnotationsTo(rm);
- } rpc_body_block[proto, rm]? SEMICOLON!
- ;
-
-rpc_body_block [Proto proto, Service.RpcMethod rm]
- : LEFTCURLY option_entry[proto, rm]* RIGHTCURLY {
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
- }
- ;
-
-extend_block [Proto proto, Message parent]
-@init {
- Extension extension = null;
-}
- : EXTEND (
- FULL_ID {
- String fullType = $FULL_ID.text;
- int lastDot = fullType.lastIndexOf('.');
- String packageName = fullType.substring(0, lastDot);
- String type = fullType.substring(lastDot+1);
- extension = new Extension(proto, parent, packageName, type);
- } | ID { extension = new Extension(proto, parent, null, $ID.text); } ) {
- if(parent==null)
- proto.addExtension(extension);
- else
- parent.addNestedExtension(extension);
-
- proto.addAnnotationsTo(extension);
- }
- LEFTCURLY (extend_body[proto, extension])* RIGHTCURLY {
- if(!proto.annotations.isEmpty())
- throw new IllegalStateException("Misplaced annotations: " + proto.annotations);
-
- } (SEMICOLON?)!
- ;
-
-extend_body [Proto proto, Extension extension]
- : message_field[proto, extension]
- | annotation_entry[proto]
- ;
-
-ignore_block
- : LEFTCURLY ignore_block_body* RIGHTCURLY
- ;
-
-ignore_block_body
- : (LEFTCURLY)=> ignore_block
- | ~RIGHTCURLY
- ;
-
[13/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
deleted file mode 100644
index d0f5c4a..0000000
--- a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-<#list ["", "Nullable", "Repeated"] as holderMode>
-<#assign nullMode = holderMode />
-<#if holderMode == "Repeated"><#assign nullMode = "Nullable" /></#if>
-
-<#assign lowerName = minor.class?uncap_first />
-<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-<#assign name = minor.class?cap_first />
-<#assign javaType = (minor.javaType!type.javaType) />
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-<#assign safeType=friendlyType />
-<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-<#assign fields = minor.fields!type.fields />
-
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${holderMode}${name}HolderReaderImpl.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-
-import org.apache.drill.exec.expr.holders.*;
-import org.apache.hadoop.io.Text;
-import org.joda.time.Period;
-
-// Source code generated using FreeMarker template ${.template_name}
-
-@SuppressWarnings("unused")
-public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
-
- private ${nullMode}${name}Holder holder;
-<#if holderMode == "Repeated" >
- private int index = -1;
- private ${holderMode}${name}Holder repeatedHolder;
-</#if>
-
- public ${holderMode}${name}HolderReaderImpl(${holderMode}${name}Holder holder) {
-<#if holderMode == "Repeated" >
- this.holder = new ${nullMode}${name}Holder();
- this.repeatedHolder = holder;
-<#else>
- this.holder = holder;
-</#if>
- }
-
- @Override
- public int size() {
-<#if holderMode == "Repeated">
- return repeatedHolder.end - repeatedHolder.start;
-<#else>
- throw new UnsupportedOperationException("You can't call size on a Holder value reader.");
-</#if>
- }
-
- @Override
- public boolean next() {
-<#if holderMode == "Repeated">
- if(index + 1 < repeatedHolder.end) {
- index++;
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
- return true;
- } else {
- return false;
- }
-<#else>
- throw new UnsupportedOperationException("You can't call next on a single value reader.");
-</#if>
-
- }
-
- @Override
- public void setPosition(int index) {
- throw new UnsupportedOperationException("You can't call next on a single value reader.");
- }
-
- @Override
- public MajorType getType() {
-<#if holderMode == "Repeated">
- return this.repeatedHolder.TYPE;
-<#else>
- return this.holder.TYPE;
-</#if>
- }
-
- @Override
- public boolean isSet() {
- <#if holderMode == "Repeated">
- return this.repeatedHolder.end!=this.repeatedHolder.start;
- <#elseif nullMode == "Nullable">
- return this.holder.isSet == 1;
- <#else>
- return true;
- </#if>
-
- }
-
-<#if holderMode != "Repeated">
-@Override
- public void read(${name}Holder h) {
- <#list fields as field>
- h.${field.name} = holder.${field.name};
- </#list>
- }
-
- @Override
- public void read(Nullable${name}Holder h) {
- <#list fields as field>
- h.${field.name} = holder.${field.name};
- </#list>
- h.isSet = isSet() ? 1 : 0;
- }
-</#if>
-
-<#if holderMode == "Repeated">
- @Override
- public ${friendlyType} read${safeType}(int index){
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
- ${friendlyType} value = read${safeType}();
- if (this.index > -1) {
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + this.index, holder);
- }
- return value;
- }
-</#if>
-
- @Override
- public ${friendlyType} read${safeType}(){
-<#if nullMode == "Nullable">
- if (!isSet()) {
- return null;
- }
-</#if>
-
-<#if type.major == "VarLen">
-
- int length = holder.end - holder.start;
- byte[] value = new byte [length];
- holder.buffer.getBytes(holder.start, value, 0, length);
-
-<#if minor.class == "VarBinary">
- return value;
-<#elseif minor.class == "Var16Char">
- return new String(value);
-<#elseif minor.class == "VarChar">
- Text text = new Text();
- text.set(value);
- return text;
-</#if>
-
-<#elseif minor.class == "Interval">
- Period p = new Period();
- return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "IntervalDay">
- Period p = new Period();
- return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "Decimal9" ||
- minor.class == "Decimal18" >
- BigInteger value = BigInteger.valueOf(holder.value);
- return new BigDecimal(value, holder.scale);
-
-<#elseif minor.class == "Decimal28Dense" ||
- minor.class == "Decimal38Dense">
- return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale,
- holder.maxPrecision,
- holder.WIDTH);
-
-<#elseif minor.class == "Decimal28Sparse" ||
- minor.class == "Decimal38Sparse">
- return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale);
-
-<#elseif minor.class == "Bit" >
- return new Boolean(holder.value != 0);
-<#else>
- ${friendlyType} value = new ${friendlyType}(this.holder.value);
- return value;
-</#if>
-
- }
-
- @Override
- public Object readObject() {
-<#if holderMode == "Repeated" >
- List<Object> valList = Lists.newArrayList();
- for (int i = repeatedHolder.start; i < repeatedHolder.end; i++) {
- valList.add(repeatedHolder.vector.getAccessor().getObject(i));
- }
- return valList;
-<#else>
- return readSingleObject();
-</#if>
- }
-
- private Object readSingleObject() {
-<#if nullMode == "Nullable">
- if (!isSet()) {
- return null;
- }
-</#if>
-
-<#if type.major == "VarLen">
- int length = holder.end - holder.start;
- byte[] value = new byte [length];
- holder.buffer.getBytes(holder.start, value, 0, length);
-
-<#if minor.class == "VarBinary">
- return value;
-<#elseif minor.class == "Var16Char">
- return new String(value);
-<#elseif minor.class == "VarChar">
- Text text = new Text();
- text.set(value);
- return text;
-</#if>
-
-<#elseif minor.class == "Interval">
- Period p = new Period();
- return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "IntervalDay">
- Period p = new Period();
- return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "Decimal9" ||
- minor.class == "Decimal18" >
- BigInteger value = BigInteger.valueOf(holder.value);
- return new BigDecimal(value, holder.scale);
-
-<#elseif minor.class == "Decimal28Dense" ||
- minor.class == "Decimal38Dense">
- return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale,
- holder.maxPrecision,
- holder.WIDTH);
-
-<#elseif minor.class == "Decimal28Sparse" ||
- minor.class == "Decimal38Sparse">
- return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale);
-
-<#elseif minor.class == "Bit" >
- return new Boolean(holder.value != 0);
-<#else>
- ${friendlyType} value = new ${friendlyType}(this.holder.value);
- return value;
-</#if>
- }
-
-<#if holderMode != "Repeated" && nullMode != "Nullable">
- public void copyAsValue(${minor.class?cap_first}Writer writer){
- writer.write(holder);
- }
-</#if>
-}
-
-</#list>
-</#list>
-</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/ListWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ListWriters.java b/exec/java-exec/src/main/codegen/templates/ListWriters.java
deleted file mode 100644
index 16d41ec..0000000
--- a/exec/java-exec/src/main/codegen/templates/ListWriters.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-
-<#list ["Single", "Repeated"] as mode>
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${mode}ListWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-<#if mode == "Single">
- <#assign containerClass = "AbstractContainerVector" />
- <#assign index = "idx()">
-<#else>
- <#assign containerClass = "RepeatedListVector" />
- <#assign index = "currentChildIndex">
-</#if>
-
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using FreeMarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-public class ${mode}ListWriter extends AbstractFieldWriter {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${mode}ListWriter.class);
-
- static enum Mode { INIT, IN_MAP, IN_LIST <#list vv.types as type><#list type.minor as minor>, IN_${minor.class?upper_case}</#list></#list> }
-
- private final String name;
- protected final ${containerClass} container;
- private Mode mode = Mode.INIT;
- private FieldWriter writer;
- protected RepeatedValueVector innerVector;
-
- <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
- public ${mode}ListWriter(String name, ${containerClass} container, FieldWriter parent){
- super(parent);
- this.name = name;
- this.container = container;
- }
-
- public ${mode}ListWriter(${containerClass} container, FieldWriter parent){
- super(parent);
- this.name = null;
- this.container = container;
- }
-
- @Override
- public void allocate() {
- if(writer != null) {
- writer.allocate();
- }
-
- <#if mode == "Repeated">
- container.allocateNew();
- </#if>
- }
-
- @Override
- public void clear() {
- if (writer != null) {
- writer.clear();
- }
- }
-
- @Override
- public void close() {
- clear();
- container.close();
- if (innerVector != null) {
- innerVector.close();
- }
- }
-
- @Override
- public int getValueCapacity() {
- return innerVector == null ? 0 : innerVector.getValueCapacity();
- }
-
- public void setValueCount(int count){
- if(innerVector != null) innerVector.getMutator().setValueCount(count);
- }
-
- @Override
- public MapWriter map() {
- switch(mode) {
- case INIT:
- int vectorCount = container.size();
- final RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class);
- innerVector = vector;
- writer = new RepeatedMapWriter(vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_MAP;
- return writer;
- case IN_MAP:
- return writer;
- }
-
- throw UserException.unsupportedError().message(getUnsupportedErrorMsg("MAP", mode.name())).build(logger);
-
- }
-
- @Override
- public ListWriter list() {
- switch(mode) {
- case INIT:
- final int vectorCount = container.size();
- final RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class);
- innerVector = vector;
- writer = new RepeatedListWriter(null, vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_LIST;
- return writer;
- case IN_LIST:
- return writer;
- }
-
- throw UserException.unsupportedError().message(getUnsupportedErrorMsg("LIST", mode.name())).build(logger);
-
- }
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-
- private static final MajorType ${upperName}_TYPE = Types.repeated(MinorType.${upperName});
-
- @Override
- public ${capName}Writer ${lowerName}() {
- switch(mode) {
- case INIT:
- final int vectorCount = container.size();
- final Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class);
- innerVector = vector;
- writer = new Repeated${capName}WriterImpl(vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_${upperName};
- return writer;
- case IN_${upperName}:
- return writer;
- }
-
- throw UserException.unsupportedError().message(getUnsupportedErrorMsg("${upperName}", mode.name())).build(logger);
-
- }
- </#list></#list>
-
- public MaterializedField getField() {
- return container.getField();
- }
-
- <#if mode == "Repeated">
-
- public void startList() {
- final RepeatedListVector list = (RepeatedListVector) container;
- final RepeatedListVector.RepeatedMutator mutator = list.getMutator();
-
- // make sure that the current vector can support the end position of this list.
- if(container.getValueCapacity() <= idx()) {
- mutator.setValueCount(idx()+1);
- }
-
- // update the repeated vector to state that there is current+1 objects.
- final RepeatedListHolder h = new RepeatedListHolder();
- list.getAccessor().get(idx(), h);
- if (h.start >= h.end) {
- mutator.startNewValue(idx());
- }
- currentChildIndex = container.getMutator().add(idx());
- if(writer != null) {
- writer.setPosition(currentChildIndex);
- }
- }
-
- public void endList() {
- // noop, we initialize state at start rather than end.
- }
- <#else>
-
- public void setPosition(int index) {
- super.setPosition(index);
- if(writer != null) {
- writer.setPosition(index);
- }
- }
-
- public void startList() {
- // noop
- }
-
- public void endList() {
- // noop
- }
- </#if>
-
- private String getUnsupportedErrorMsg(String expected, String found) {
- final String f = found.substring(3);
- return String.format("In a list of type %s, encountered a value of type %s. "+
- "Drill does not support lists of different types.",
- f, expected
- );
- }
-}
-</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/MapWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java
deleted file mode 100644
index 93f2edb..0000000
--- a/exec/java-exec/src/main/codegen/templates/MapWriters.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<#list ["Single", "Repeated"] as mode>
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${mode}MapWriter.java" />
-<#if mode == "Single">
-<#assign containerClass = "MapVector" />
-<#assign index = "idx()">
-<#else>
-<#assign containerClass = "RepeatedMapVector" />
-<#assign index = "currentChildIndex">
-</#if>
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-import java.util.Map;
-
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
-import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-import org.apache.drill.exec.vector.complex.writer.FieldWriter;
-
-import com.google.common.collect.Maps;
-
-/*
- * This class is generated using FreeMarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-public class ${mode}MapWriter extends AbstractFieldWriter {
-
- protected final ${containerClass} container;
- private final Map<String, FieldWriter> fields = Maps.newHashMap();
- <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
-
- private final boolean unionEnabled;
-
- public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled) {
- super(parent);
- this.container = container;
- this.unionEnabled = unionEnabled;
- }
-
- public ${mode}MapWriter(${containerClass} container, FieldWriter parent) {
- this(container, parent, false);
- }
-
- @Override
- public int getValueCapacity() {
- return container.getValueCapacity();
- }
-
- @Override
- public boolean isEmptyMap() {
- return 0 == container.size();
- }
-
- @Override
- public MaterializedField getField() {
- return container.getField();
- }
-
- @Override
- public MapWriter map(String name) {
- FieldWriter writer = fields.get(name.toLowerCase());
- if(writer == null){
- int vectorCount=container.size();
- MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class);
- if(!unionEnabled){
- writer = new SingleMapWriter(vector, this);
- } else {
- writer = new PromotableWriter(vector, container);
- }
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- fields.put(name.toLowerCase(), writer);
- }
- return writer;
- }
-
- @Override
- public void close() throws Exception {
- clear();
- container.close();
- }
-
- @Override
- public void allocate() {
- container.allocateNew();
- for(final FieldWriter w : fields.values()) {
- w.allocate();
- }
- }
-
- @Override
- public void clear() {
- container.clear();
- for(final FieldWriter w : fields.values()) {
- w.clear();
- }
- }
-
- @Override
- public ListWriter list(String name) {
- FieldWriter writer = fields.get(name.toLowerCase());
- int vectorCount = container.size();
- if(writer == null) {
- if (!unionEnabled){
- writer = new SingleListWriter(name,container,this);
- } else{
- writer = new PromotableWriter(container.addOrGet(name, Types.optional(MinorType.LIST), ListVector.class), container);
- }
- if (container.size() > vectorCount) {
- writer.allocate();
- }
- writer.setPosition(${index});
- fields.put(name.toLowerCase(), writer);
- }
- return writer;
- }
-
- <#if mode == "Repeated">
- public void start() {
- // update the repeated vector to state that there is current+1 objects.
- final RepeatedMapHolder h = new RepeatedMapHolder();
- final RepeatedMapVector map = (RepeatedMapVector) container;
- final RepeatedMapVector.Mutator mutator = map.getMutator();
-
- // Make sure that the current vector can support the end position of this list.
- if(container.getValueCapacity() <= idx()) {
- mutator.setValueCount(idx()+1);
- }
-
- map.getAccessor().get(idx(), h);
- if (h.start >= h.end) {
- container.getMutator().startNewValue(idx());
- }
- currentChildIndex = container.getMutator().add(idx());
- for(final FieldWriter w : fields.values()) {
- w.setPosition(currentChildIndex);
- }
- }
-
-
- public void end() {
- // noop
- }
- <#else>
-
- public void setValueCount(int count) {
- container.getMutator().setValueCount(count);
- }
-
- @Override
- public void setPosition(int index) {
- super.setPosition(index);
- for(final FieldWriter w: fields.values()) {
- w.setPosition(index);
- }
- }
-
- @Override
- public void start() {
- }
-
- @Override
- public void end() {
- }
-
- </#if>
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#assign vectName = capName />
- <#assign vectName = "Nullable${capName}" />
-
- <#if minor.class?starts_with("Decimal") >
- public ${minor.class}Writer ${lowerName}(String name) {
- // returns existing writer
- final FieldWriter writer = fields.get(name.toLowerCase());
- assert writer != null;
- return writer;
- }
-
- public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) {
- final MajorType ${upperName}_TYPE = Types.withScaleAndPrecision(MinorType.${upperName}, DataMode.OPTIONAL, scale, precision);
- <#else>
- private static final MajorType ${upperName}_TYPE = Types.optional(MinorType.${upperName});
- @Override
- public ${minor.class}Writer ${lowerName}(String name) {
- </#if>
- FieldWriter writer = fields.get(name.toLowerCase());
- if(writer == null) {
- ValueVector vector;
- ValueVector currentVector = container.getChild(name);
- if (unionEnabled){
- ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
- writer = new PromotableWriter(v, container);
- vector = v;
- } else {
- ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
- writer = new ${vectName}WriterImpl(v, this);
- vector = v;
- }
- if (currentVector == null || currentVector != vector) {
- vector.allocateNewSafe();
- }
- writer.setPosition(${index});
- fields.put(name.toLowerCase(), writer);
- }
- return writer;
- }
-
- </#list></#list>
-
-}
-</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/NullReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullReader.java b/exec/java-exec/src/main/codegen/templates/NullReader.java
deleted file mode 100644
index 62aa33e..0000000
--- a/exec/java-exec/src/main/codegen/templates/NullReader.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/NullReader.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-import org.apache.drill.common.types.TypeProtos;
-
-
-@SuppressWarnings("unused")
-public class NullReader extends AbstractBaseReader implements FieldReader{
-
- public static final NullReader INSTANCE = new NullReader();
- public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(Types.repeated(TypeProtos.MinorType.NULL));
- public static final NullReader EMPTY_MAP_INSTANCE = new NullReader(Types.required(TypeProtos.MinorType.MAP));
- private MajorType type;
-
- private NullReader(){
- super();
- type = Types.NULL;
- }
-
- private NullReader(MajorType type){
- super();
- this.type = type;
- }
-
- @Override
- public MajorType getType() {
- return type;
- }
-
- public void copyAsValue(MapWriter writer) {}
-
- public void copyAsValue(ListWriter writer) {}
-
- public void copyAsValue(UnionWriter writer) {}
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- public void read(${name}Holder holder){
- throw new UnsupportedOperationException("NullReader cannot write into non-nullable holder");
- }
-
- public void read(Nullable${name}Holder holder){
- holder.isSet = 0;
- }
-
- public void read(int arrayIndex, ${name}Holder holder){
- throw new ArrayIndexOutOfBoundsException();
- }
-
- public void copyAsValue(${minor.class}Writer writer){}
- public void copyAsField(String name, ${minor.class}Writer writer){}
-
- public void read(int arrayIndex, Nullable${name}Holder holder){
- throw new ArrayIndexOutOfBoundsException();
- }
- </#list></#list>
-
- public int size(){
- return 0;
- }
-
- public boolean isSet(){
- return false;
- }
-
- public boolean next(){
- return false;
- }
-
- public RepeatedMapReader map(){
- return this;
- }
-
- public RepeatedListReader list(){
- return this;
- }
-
- public MapReader map(String name){
- return this;
- }
-
- public ListReader list(String name){
- return this;
- }
-
- public FieldReader reader(String name){
- return this;
- }
-
- public FieldReader reader(){
- return this;
- }
-
- private void fail(String name){
- throw new IllegalArgumentException(String.format("You tried to read a %s type when you are using a ValueReader of type %s.", name, this.getClass().getSimpleName()));
- }
-
- <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
- "Character", "DateTime", "Period", "Double", "Float",
- "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
- <#assign safeType=friendlyType />
- <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-
- public ${friendlyType} read${safeType}(int arrayIndex){
- return null;
- }
-
- public ${friendlyType} read${safeType}(){
- return null;
- }
- </#list>
-
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
deleted file mode 100644
index fe91fe6..0000000
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ /dev/null
@@ -1,636 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
-
-import java.lang.Override;
-import java.lang.UnsupportedOperationException;
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-
-<#assign className = "Nullable${minor.class}Vector" />
-<#assign valuesName = "${minor.class}Vector" />
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${className}.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/**
- * Nullable${minor.class} implements a vector of values which could be null. Elements in the vector
- * are first checked against a fixed length vector of boolean values. Then the element is retrieved
- * from the base class (if not null).
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-@SuppressWarnings("unused")
-public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector{
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
-
- private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
-
- private final MaterializedField bitsField = MaterializedField.create("$bits$", Types.required(MinorType.UINT1));
- private final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
- private final ${valuesName} values = new ${minor.class}Vector(field, allocator);
-
- private final Mutator mutator = new Mutator();
- private final Accessor accessor = new Accessor();
-
- public ${className}(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- @Override
- public FieldReader getReader(){
- return reader;
- }
-
- @Override
- public int getValueCapacity(){
- return Math.min(bits.getValueCapacity(), values.getValueCapacity());
- }
-
- @Override
- public DrillBuf[] getBuffers(boolean clear) {
- final DrillBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), DrillBuf.class);
- if (clear) {
- for (final DrillBuf buffer:buffers) {
- buffer.retain(1);
- }
- clear();
- }
- return buffers;
- }
-
- @Override
- public void close() {
- bits.close();
- values.close();
- super.close();
- }
-
- @Override
- public void clear() {
- bits.clear();
- values.clear();
- super.clear();
- }
-
- @Override
- public int getBufferSize(){
- return values.getBufferSize() + bits.getBufferSize();
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- return values.getBufferSizeFor(valueCount)
- + bits.getBufferSizeFor(valueCount);
- }
-
- @Override
- public DrillBuf getBuffer() {
- return values.getBuffer();
- }
-
- @Override
- public ${valuesName} getValuesVector() {
- return values;
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- bits.setInitialCapacity(numRecords);
- values.setInitialCapacity(numRecords);
- }
-
- @Override
- public SerializedField.Builder getMetadataBuilder() {
- return super.getMetadataBuilder()
- .addChild(bits.getMetadata())
- .addChild(values.getMetadata());
- }
-
- @Override
- public void allocateNew() {
- if(!allocateNewSafe()){
- throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
- }
- }
-
- @Override
- public boolean allocateNewSafe() {
- /* Boolean to keep track if all the memory allocations were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- success = values.allocateNewSafe() && bits.allocateNewSafe();
- } finally {
- if (!success) {
- clear();
- }
- }
- bits.zeroVector();
- mutator.reset();
- accessor.reset();
- return success;
- }
-
- <#if type.major == "VarLen">
- @Override
- public void allocateNew(int totalBytes, int valueCount) {
- try {
- values.allocateNew(totalBytes, valueCount);
- bits.allocateNew(valueCount);
- } catch(DrillRuntimeException e) {
- clear();
- throw e;
- }
- bits.zeroVector();
- mutator.reset();
- accessor.reset();
- }
-
- public void reset() {
- bits.zeroVector();
- mutator.reset();
- accessor.reset();
- super.reset();
- }
-
- @Override
- public int getByteCapacity(){
- return values.getByteCapacity();
- }
-
- @Override
- public int getCurrentSizeInBytes(){
- return values.getCurrentSizeInBytes();
- }
-
- <#else>
- @Override
- public void allocateNew(int valueCount) {
- try {
- values.allocateNew(valueCount);
- bits.allocateNew(valueCount+1);
- } catch(OutOfMemoryRuntimeException e) {
- clear();
- throw e;
- }
- bits.zeroVector();
- mutator.reset();
- accessor.reset();
- }
-
- @Override
- public void reset() {
- bits.zeroVector();
- mutator.reset();
- accessor.reset();
- super.reset();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void zeroVector() {
- bits.zeroVector();
- values.zeroVector();
- }
- </#if>
-
-
- @Override
- public void load(SerializedField metadata, DrillBuf buffer) {
- clear();
- // the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
- final SerializedField bitsField = metadata.getChild(0);
- bits.load(bitsField, buffer);
-
- final int capacity = buffer.capacity();
- final int bitsLength = bitsField.getBufferLength();
- final SerializedField valuesField = metadata.getChild(1);
- values.load(valuesField, buffer.slice(bitsLength, capacity - bitsLength));
- }
-
- @Override
- public TransferPair getTransferPair(){
- return new TransferImpl(getField());
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref){
- return new TransferImpl(getField().withPath(ref));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new TransferImpl((Nullable${minor.class}Vector) to);
- }
-
- public void transferTo(Nullable${minor.class}Vector target){
- bits.transferTo(target.bits);
- values.transferTo(target.values);
- <#if type.major == "VarLen">
- target.mutator.lastSet = mutator.lastSet;
- </#if>
- clear();
- }
-
- public void splitAndTransferTo(int startIndex, int length, Nullable${minor.class}Vector target) {
- bits.splitAndTransferTo(startIndex, length, target.bits);
- values.splitAndTransferTo(startIndex, length, target.values);
- <#if type.major == "VarLen">
- target.mutator.lastSet = length - 1;
- </#if>
- }
-
- private class TransferImpl implements TransferPair {
- Nullable${minor.class}Vector to;
-
- public TransferImpl(MaterializedField field){
- to = new Nullable${minor.class}Vector(field, allocator);
- }
-
- public TransferImpl(Nullable${minor.class}Vector to){
- this.to = to;
- }
-
- @Override
- public Nullable${minor.class}Vector getTo(){
- return to;
- }
-
- @Override
- public void transfer(){
- transferTo(to);
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- splitAndTransferTo(startIndex, length, to);
- }
-
- @Override
- public void copyValueSafe(int fromIndex, int toIndex) {
- to.copyFromSafe(fromIndex, toIndex, Nullable${minor.class}Vector.this);
- }
- }
-
- @Override
- public Accessor getAccessor(){
- return accessor;
- }
-
- @Override
- public Mutator getMutator(){
- return mutator;
- }
-
- public ${minor.class}Vector convertToRequiredVector(){
- ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
- if (v.data != null) {
- v.data.release(1);
- }
- v.data = values.data;
- v.data.retain(1);
- clear();
- return v;
- }
-
- public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
- final Accessor fromAccessor = from.getAccessor();
- if (!fromAccessor.isNull(fromIndex)) {
- mutator.set(thisIndex, fromAccessor.get(fromIndex));
- }
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
- <#if type.major == "VarLen">
- mutator.fillEmpties(thisIndex);
- </#if>
- values.copyFromSafe(fromIndex, thisIndex, from);
- bits.getMutator().setSafe(thisIndex, 1);
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
- <#if type.major == "VarLen">
- mutator.fillEmpties(thisIndex);
- </#if>
- bits.copyFromSafe(fromIndex, thisIndex, from.bits);
- values.copyFromSafe(fromIndex, thisIndex, from.values);
- }
-
- public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
- final UInt1Vector.Accessor bAccessor = bits.getAccessor();
- final ${valuesName}.Accessor vAccessor = values.getAccessor();
-
- /**
- * Get the element at the specified position.
- *
- * @param index position of the value
- * @return value of the element, if not null
- * @throws NullValueException if the value is null
- */
- public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
- if (isNull(index)) {
- throw new IllegalStateException("Can't get a null value");
- }
- return vAccessor.get(index);
- }
-
- @Override
- public boolean isNull(int index) {
- return isSet(index) == 0;
- }
-
- public int isSet(int index){
- return bAccessor.get(index);
- }
-
- <#if type.major == "VarLen">
- public long getStartEnd(int index){
- return vAccessor.getStartEnd(index);
- }
-
- @Override
- public int getValueLength(int index) {
- return values.getAccessor().getValueLength(index);
- }
- </#if>
-
- public void get(int index, Nullable${minor.class}Holder holder){
- vAccessor.get(index, holder);
- holder.isSet = bAccessor.get(index);
-
- <#if minor.class.startsWith("Decimal")>
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
- </#if>
- }
-
- @Override
- public ${friendlyType} getObject(int index) {
- if (isNull(index)) {
- return null;
- }else{
- return vAccessor.getObject(index);
- }
- }
-
- <#if minor.class == "Interval" || minor.class == "IntervalDay" || minor.class == "IntervalYear">
- public StringBuilder getAsStringBuilder(int index) {
- if (isNull(index)) {
- return null;
- }else{
- return vAccessor.getAsStringBuilder(index);
- }
- }
- </#if>
-
- @Override
- public int getValueCount(){
- return bits.getAccessor().getValueCount();
- }
-
- public void reset(){}
- }
-
- public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> {
- private int setCount;
- <#if type.major = "VarLen"> private int lastSet = -1;</#if>
-
- private Mutator(){
- }
-
- public ${valuesName} getVectorWithValues(){
- return values;
- }
-
- @Override
- public void setIndexDefined(int index){
- bits.getMutator().set(index, 1);
- }
-
- /**
- * Set the variable length element at the specified index to the supplied byte array.
- *
- * @param index position of the bit to set
- * @param bytes array of bytes to write
- */
- public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
- setCount++;
- final ${valuesName}.Mutator valuesMutator = values.getMutator();
- final UInt1Vector.Mutator bitsMutator = bits.getMutator();
- <#if type.major == "VarLen">
- for (int i = lastSet + 1; i < index; i++) {
- valuesMutator.set(i, emptyByteArray);
- }
- </#if>
- bitsMutator.set(index, 1);
- valuesMutator.set(index, value);
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
- <#if type.major == "VarLen">
-
- private void fillEmpties(int index){
- final ${valuesName}.Mutator valuesMutator = values.getMutator();
- for (int i = lastSet; i < index; i++) {
- valuesMutator.setSafe(i + 1, emptyByteArray);
- }
- while(index > bits.getValueCapacity()) {
- bits.reAlloc();
- }
- lastSet = index;
- }
-
- @Override
- public void setValueLengthSafe(int index, int length) {
- values.getMutator().setValueLengthSafe(index, length);
- lastSet = index;
- }
- </#if>
-
- public void setSafe(int index, byte[] value, int start, int length) {
- <#if type.major != "VarLen">
- throw new UnsupportedOperationException();
- <#else>
- fillEmpties(index);
-
- bits.getMutator().setSafe(index, 1);
- values.getMutator().setSafe(index, value, start, length);
- setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
- </#if>
- }
-
- public void setSafe(int index, ByteBuffer value, int start, int length) {
- <#if type.major != "VarLen">
- throw new UnsupportedOperationException();
- <#else>
- fillEmpties(index);
-
- bits.getMutator().setSafe(index, 1);
- values.getMutator().setSafe(index, value, start, length);
- setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
- </#if>
- }
-
- public void setNull(int index){
- bits.getMutator().setSafe(index, 0);
- }
-
- public void setSkipNull(int index, ${minor.class}Holder holder){
- values.getMutator().set(index, holder);
- }
-
- public void setSkipNull(int index, Nullable${minor.class}Holder holder){
- values.getMutator().set(index, holder);
- }
-
-
- public void set(int index, Nullable${minor.class}Holder holder){
- final ${valuesName}.Mutator valuesMutator = values.getMutator();
- <#if type.major == "VarLen">
- for (int i = lastSet + 1; i < index; i++) {
- valuesMutator.set(i, emptyByteArray);
- }
- </#if>
- bits.getMutator().set(index, holder.isSet);
- valuesMutator.set(index, holder);
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
- public void set(int index, ${minor.class}Holder holder){
- final ${valuesName}.Mutator valuesMutator = values.getMutator();
- <#if type.major == "VarLen">
- for (int i = lastSet + 1; i < index; i++) {
- valuesMutator.set(i, emptyByteArray);
- }
- </#if>
- bits.getMutator().set(index, 1);
- valuesMutator.set(index, holder);
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
- public boolean isSafe(int outIndex) {
- return outIndex < Nullable${minor.class}Vector.this.getValueCapacity();
- }
-
- <#assign fields = minor.fields!type.fields />
- public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){
- final ${valuesName}.Mutator valuesMutator = values.getMutator();
- <#if type.major == "VarLen">
- for (int i = lastSet + 1; i < index; i++) {
- valuesMutator.set(i, emptyByteArray);
- }
- </#if>
- bits.getMutator().set(index, isSet);
- valuesMutator.set(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
- public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
- <#if type.major == "VarLen">
- fillEmpties(index);
- </#if>
-
- bits.getMutator().setSafe(index, isSet);
- values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
- setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
-
- public void setSafe(int index, Nullable${minor.class}Holder value) {
-
- <#if type.major == "VarLen">
- fillEmpties(index);
- </#if>
- bits.getMutator().setSafe(index, value.isSet);
- values.getMutator().setSafe(index, value);
- setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
- public void setSafe(int index, ${minor.class}Holder value) {
-
- <#if type.major == "VarLen">
- fillEmpties(index);
- </#if>
- bits.getMutator().setSafe(index, 1);
- values.getMutator().setSafe(index, value);
- setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
- <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")>
- public void setSafe(int index, ${minor.javaType!type.javaType} value) {
- <#if type.major == "VarLen">
- fillEmpties(index);
- </#if>
- bits.getMutator().setSafe(index, 1);
- values.getMutator().setSafe(index, value);
- setCount++;
- }
-
- </#if>
-
- @Override
- public void setValueCount(int valueCount) {
- assert valueCount >= 0;
- <#if type.major == "VarLen">
- fillEmpties(valueCount);
- </#if>
- values.getMutator().setValueCount(valueCount);
- bits.getMutator().setValueCount(valueCount);
- }
-
- @Override
- public void generateTestData(int valueCount){
- bits.getMutator().generateTestDataAlt(valueCount);
- values.getMutator().generateTestData(valueCount);
- <#if type.major = "VarLen">lastSet = valueCount;</#if>
- setValueCount(valueCount);
- }
-
- @Override
- public void reset(){
- setCount = 0;
- <#if type.major = "VarLen">lastSet = -1;</#if>
- }
- }
-}
-</#list>
-</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
deleted file mode 100644
index 2f5ec18..0000000
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.Override;
-
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector;
-import org.mortbay.jetty.servlet.Holder;
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-<#assign fields = minor.fields!type.fields />
-
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/Repeated${minor.class}Vector.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/**
- * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
- * repeated protobuf field). The implementation uses two additional value vectors; one to convert
- * the index offset to the underlying element offset, and another to store the number of values
- * in the vector.
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-
-public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>VectorLike {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Repeated${minor.class}Vector.class);
-
- // we maintain local reference to concrete vector type for performance reasons.
- private ${minor.class}Vector values;
- private final FieldReader reader = new Repeated${minor.class}ReaderImpl(Repeated${minor.class}Vector.this);
- private final Mutator mutator = new Mutator();
- private final Accessor accessor = new Accessor();
-
- public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- addOrGetVector(VectorDescriptor.create(Types.required(field.getType().getMinorType())));
- }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- @Override
- public Accessor getAccessor() {
- return accessor;
- }
-
- @Override
- public FieldReader getReader() {
- return reader;
- }
-
- @Override
- public ${minor.class}Vector getDataVector() {
- return values;
- }
-
- @Override
- public TransferPair getTransferPair() {
- return new TransferImpl(getField());
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref){
- return new TransferImpl(getField().withPath(ref));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new TransferImpl((Repeated${minor.class}Vector) to);
- }
-
- @Override
- public AddOrGetResult<${minor.class}Vector> addOrGetVector(VectorDescriptor descriptor) {
- final AddOrGetResult<${minor.class}Vector> result = super.addOrGetVector(descriptor);
- if (result.isCreated()) {
- values = result.getVector();
- }
- return result;
- }
-
- public void transferTo(Repeated${minor.class}Vector target) {
- target.clear();
- offsets.transferTo(target.offsets);
- values.transferTo(target.values);
- clear();
- }
-
- public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) {
- final UInt4Vector.Accessor a = offsets.getAccessor();
- final UInt4Vector.Mutator m = to.offsets.getMutator();
-
- final int startPos = a.get(startIndex);
- final int endPos = a.get(startIndex + groups);
- final int valuesToCopy = endPos - startPos;
-
- values.splitAndTransferTo(startPos, valuesToCopy, to.values);
- to.offsets.clear();
- to.offsets.allocateNew(groups + 1);
- int normalizedPos = 0;
- for (int i=0; i < groups + 1;i++ ) {
- normalizedPos = a.get(startIndex+i) - startPos;
- m.set(i, normalizedPos);
- }
- m.setValueCount(groups == 0 ? 0 : groups + 1);
- }
-
- private class TransferImpl implements TransferPair {
- final Repeated${minor.class}Vector to;
-
- public TransferImpl(MaterializedField field) {
- this.to = new Repeated${minor.class}Vector(field, allocator);
- }
-
- public TransferImpl(Repeated${minor.class}Vector to) {
- this.to = to;
- }
-
- @Override
- public Repeated${minor.class}Vector getTo() {
- return to;
- }
-
- @Override
- public void transfer() {
- transferTo(to);
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- splitAndTransferTo(startIndex, length, to);
- }
-
- @Override
- public void copyValueSafe(int fromIndex, int toIndex) {
- to.copyFromSafe(fromIndex, toIndex, Repeated${minor.class}Vector.this);
- }
- }
-
- public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
- final Accessor vAccessor = v.getAccessor();
- final int count = vAccessor.getInnerValueCountAt(inIndex);
- mutator.startNewValue(outIndex);
- for (int i = 0; i < count; i++) {
- mutator.add(outIndex, vAccessor.get(inIndex, i));
- }
- }
-
- public void copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
- final Accessor vAccessor = v.getAccessor();
- final int count = vAccessor.getInnerValueCountAt(inIndex);
- mutator.startNewValue(outIndex);
- for (int i = 0; i < count; i++) {
- mutator.addSafe(outIndex, vAccessor.get(inIndex, i));
- }
- }
-
- public boolean allocateNewSafe() {
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- if(!offsets.allocateNewSafe()) return false;
- if(!values.allocateNewSafe()) return false;
- success = true;
- } finally {
- if (!success) {
- clear();
- }
- }
- offsets.zeroVector();
- mutator.reset();
- return true;
- }
-
- @Override
- public void allocateNew() {
- try {
- offsets.allocateNew();
- values.allocateNew();
- } catch (OutOfMemoryRuntimeException e) {
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- <#if type.major == "VarLen">
- @Override
- protected SerializedField.Builder getMetadataBuilder() {
- return super.getMetadataBuilder()
- .setVarByteLength(values.getVarByteLength());
- }
-
- public void allocateNew(int totalBytes, int valueCount, int innerValueCount) {
- try {
- offsets.allocateNew(valueCount + 1);
- values.allocateNew(totalBytes, innerValueCount);
- } catch (OutOfMemoryRuntimeException e) {
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- public int getByteCapacity(){
- return values.getByteCapacity();
- }
-
- <#else>
-
- @Override
- public void allocateNew(int valueCount, int innerValueCount) {
- clear();
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to//
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- offsets.allocateNew(valueCount + 1);
- values.allocateNew(innerValueCount);
- } catch(OutOfMemoryRuntimeException e){
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- </#if>
-
- // This is declared a subclass of the accessor declared inside of FixedWidthVector, this is also used for
- // variable length vectors, as they should ahve consistent interface as much as possible, if they need to diverge
- // in the future, the interface shold be declared in the respective value vector superclasses for fixed and variable
- // and we should refer to each in the generation template
- public final class Accessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
- @Override
- public List<${friendlyType}> getObject(int index) {
- final List<${friendlyType}> vals = new JsonStringArrayList<>();
- final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
- final int start = offsetsAccessor.get(index);
- final int end = offsetsAccessor.get(index + 1);
- final ${minor.class}Vector.Accessor valuesAccessor = values.getAccessor();
- for(int i = start; i < end; i++) {
- vals.add(valuesAccessor.getObject(i));
- }
- return vals;
- }
-
- public ${friendlyType} getSingleObject(int index, int arrayIndex) {
- final int start = offsets.getAccessor().get(index);
- return values.getAccessor().getObject(start + arrayIndex);
- }
-
- /**
- * Get a value for the given record. Each element in the repeated field is accessed by
- * the positionIndex param.
- *
- * @param index record containing the repeated field
- * @param positionIndex position within the repeated field
- * @return element at the given position in the given record
- */
- public <#if type.major == "VarLen">byte[]
- <#else>${minor.javaType!type.javaType}
- </#if> get(int index, int positionIndex) {
- return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
- }
-
- public void get(int index, Repeated${minor.class}Holder holder) {
- holder.start = offsets.getAccessor().get(index);
- holder.end = offsets.getAccessor().get(index+1);
- holder.vector = values;
- }
-
- public void get(int index, int positionIndex, ${minor.class}Holder holder) {
- final int offset = offsets.getAccessor().get(index);
- assert offset >= 0;
- assert positionIndex < getInnerValueCountAt(index);
- values.getAccessor().get(offset + positionIndex, holder);
- }
-
- public void get(int index, int positionIndex, Nullable${minor.class}Holder holder) {
- final int offset = offsets.getAccessor().get(index);
- assert offset >= 0;
- if (positionIndex >= getInnerValueCountAt(index)) {
- holder.isSet = 0;
- return;
- }
- values.getAccessor().get(offset + positionIndex, holder);
- }
- }
-
- public final class Mutator extends BaseRepeatedValueVector.BaseRepeatedMutator implements RepeatedMutator {
- private Mutator() {}
-
- /**
- * Add an element to the given record index. This is similar to the set() method in other
- * value vectors, except that it permits setting multiple values for a single record.
- *
- * @param index record of the element to add
- * @param value value to add to the given row
- */
- public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
- int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().set(nextOffset, value);
- offsets.getMutator().set(index+1, nextOffset+1);
- }
-
- <#if type.major == "VarLen">
- public void addSafe(int index, byte[] bytes) {
- addSafe(index, bytes, 0, bytes.length);
- }
-
- public void addSafe(int index, byte[] bytes, int start, int length) {
- final int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().setSafe(nextOffset, bytes, start, length);
- offsets.getMutator().setSafe(index+1, nextOffset+1);
- }
-
- <#else>
-
- public void addSafe(int index, ${minor.javaType!type.javaType} srcValue) {
- final int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().setSafe(nextOffset, srcValue);
- offsets.getMutator().setSafe(index+1, nextOffset+1);
- }
-
- </#if>
-
- public void setSafe(int index, Repeated${minor.class}Holder h) {
- final ${minor.class}Holder ih = new ${minor.class}Holder();
- final ${minor.class}Vector.Accessor hVectorAccessor = h.vector.getAccessor();
- mutator.startNewValue(index);
- for(int i = h.start; i < h.end; i++){
- hVectorAccessor.get(i, ih);
- mutator.addSafe(index, ih);
- }
- }
-
- public void addSafe(int index, ${minor.class}Holder holder) {
- int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().setSafe(nextOffset, holder);
- offsets.getMutator().setSafe(index+1, nextOffset+1);
- }
-
- public void addSafe(int index, Nullable${minor.class}Holder holder) {
- final int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().setSafe(nextOffset, holder);
- offsets.getMutator().setSafe(index+1, nextOffset+1);
- }
-
- <#if (fields?size > 1) && !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
- public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- int nextOffset = offsets.getAccessor().get(arrayIndex+1);
- values.getMutator().setSafe(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
- offsets.getMutator().setSafe(arrayIndex+1, nextOffset+1);
- }
- </#if>
-
- protected void add(int index, ${minor.class}Holder holder) {
- int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().set(nextOffset, holder);
- offsets.getMutator().set(index+1, nextOffset+1);
- }
-
- public void add(int index, Repeated${minor.class}Holder holder) {
-
- ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
- ${minor.class}Holder innerHolder = new ${minor.class}Holder();
-
- for(int i = holder.start; i < holder.end; i++) {
- accessor.get(i, innerHolder);
- add(index, innerHolder);
- }
- }
-
- @Override
- public void generateTestData(final int valCount) {
- final int[] sizes = {1, 2, 0, 6};
- int size = 0;
- int runningOffset = 0;
- final UInt4Vector.Mutator offsetsMutator = offsets.getMutator();
- for(int i = 1; i < valCount + 1; i++, size++) {
- runningOffset += sizes[size % sizes.length];
- offsetsMutator.set(i, runningOffset);
- }
- values.getMutator().generateTestData(valCount * 9);
- setValueCount(size);
- }
-
- @Override
- public void reset() {
- }
- }
-}
-</#list>
-</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
deleted file mode 100644
index b8178cb..0000000
--- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
+++ /dev/null
@@ -1,600 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.drill.exec.vector.complex.UnionVector;
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/TypeHelper.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.expr;
-
-<#include "/@includes/vv_imports.ftl" />
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.vector.accessor.*;
-import org.apache.drill.exec.vector.complex.RepeatedMapVector;
-import org.apache.drill.exec.util.CallBack;
-
-public class TypeHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
-
- private static final int WIDTH_ESTIMATE = 50;
-
- // Default length when casting to varchar : 65536 = 2^16
- // This only defines an absolute maximum for values, setting
- // a high value like this will not inflate the size for small values
- public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
-
- private static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
- return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
- }
-
- private static String buildErrorMessage(final String operation, final MajorType type) {
- return buildErrorMessage(operation, type.getMinorType(), type.getMode());
- }
-
- public static int getSize(MajorType major) {
- switch (major.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
- minor.class?substring(0, 3) == "PRO" ||
- minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
- </#list>
-</#list>
- case FIXEDCHAR: return major.getWidth();
- case FIXED16CHAR: return major.getWidth();
- case FIXEDBINARY: return major.getWidth();
- }
- throw new UnsupportedOperationException(buildErrorMessage("get size", major));
- }
-
- public static SqlAccessor getSqlAccessor(ValueVector vector){
- final MajorType type = vector.getField().getType();
- switch(type.getMinorType()){
- case UNION:
- return new UnionSqlAccessor((UnionVector) vector);
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Accessor((${minor.class}Vector) vector);
- case OPTIONAL:
- return new Nullable${minor.class}Accessor((Nullable${minor.class}Vector) vector);
- case REPEATED:
- return new GenericAccessor(vector);
- }
- </#list>
- </#list>
- case MAP:
- case LIST:
- return new GenericAccessor(vector);
- }
- throw new UnsupportedOperationException(buildErrorMessage("find sql accessor", type));
- }
-
- public static ValueVector getNewVector(SchemaPath parentPath, String name, BufferAllocator allocator, MajorType type, CallBack callback){
- SchemaPath child = parentPath.getChild(name);
- MaterializedField field = MaterializedField.create(child, type);
- return getNewVector(field, allocator, callback);
- }
-
-
- public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return UnionVector.class;
- case MAP:
- switch (mode) {
- case OPTIONAL:
- case REQUIRED:
- return MapVector.class;
- case REPEATED:
- return RepeatedMapVector.class;
- }
-
- case LIST:
- switch (mode) {
- case REPEATED:
- return RepeatedListVector.class;
- case REQUIRED:
- case OPTIONAL:
- return ListVector.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}Vector.class;
- case OPTIONAL:
- return Nullable${minor.class}Vector.class;
- case REPEATED:
- return Repeated${minor.class}Vector.class;
- }
- </#list>
-</#list>
- case GENERIC_OBJECT :
- return ObjectVector.class ;
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
- }
- public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
- switch (type) {
- case MAP:
- switch (mode) {
- case REQUIRED:
- if (!isSingularRepeated)
- return SingleMapReaderImpl.class;
- else
- return SingleLikeRepeatedMapReaderImpl.class;
- case REPEATED:
- return RepeatedMapReaderImpl.class;
- }
- case LIST:
- switch (mode) {
- case REQUIRED:
- return SingleListReaderImpl.class;
- case REPEATED:
- return RepeatedListReaderImpl.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}ReaderImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}ReaderImpl.class;
- case REPEATED:
- return Repeated${minor.class}ReaderImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
- }
-
- public static Class<?> getWriterInterface( MinorType type, DataMode mode){
- switch (type) {
- case UNION: return UnionWriter.class;
- case MAP: return MapWriter.class;
- case LIST: return ListWriter.class;
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}: return ${minor.class}Writer.class;
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
- }
-
- public static Class<?> getWriterImpl( MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return UnionWriter.class;
- case MAP:
- switch (mode) {
- case REQUIRED:
- case OPTIONAL:
- return SingleMapWriter.class;
- case REPEATED:
- return RepeatedMapWriter.class;
- }
- case LIST:
- switch (mode) {
- case REQUIRED:
- case OPTIONAL:
- return UnionListWriter.class;
- case REPEATED:
- return RepeatedListWriter.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}WriterImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}WriterImpl.class;
- case REPEATED:
- return Repeated${minor.class}WriterImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
- }
-
- public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
- switch (type) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}HolderReaderImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}HolderReaderImpl.class;
- case REPEATED:
- return Repeated${minor.class}HolderReaderImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
- }
-
- public static JType getHolderType(JCodeModel model, MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return model._ref(UnionHolder.class);
- case MAP:
- case LIST:
- return model._ref(ComplexHolder.class);
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return model._ref(${minor.class}Holder.class);
- case OPTIONAL:
- return model._ref(Nullable${minor.class}Holder.class);
- case REPEATED:
- return model._ref(Repeated${minor.class}Holder.class);
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- return model._ref(ObjectHolder.class);
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get holder type", type, mode));
- }
-
- public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
- return getNewVector(field, allocator, null);
- }
- public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- MajorType type = field.getType();
-
- switch (type.getMinorType()) {
-
- case UNION:
- return new UnionVector(field, allocator, callBack);
-
- case MAP:
- switch (type.getMode()) {
- case REQUIRED:
- case OPTIONAL:
- return new MapVector(field, allocator, callBack);
- case REPEATED:
- return new RepeatedMapVector(field, allocator, callBack);
- }
- case LIST:
- switch (type.getMode()) {
- case REPEATED:
- return new RepeatedListVector(field, allocator, callBack);
- case OPTIONAL:
- case REQUIRED:
- return new ListVector(field, allocator, callBack);
- }
-<#list vv. types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Vector(field, allocator);
- case OPTIONAL:
- return new Nullable${minor.class}Vector(field, allocator);
- case REPEATED:
- return new Repeated${minor.class}Vector(field, allocator);
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- return new ObjectVector(field, allocator) ;
- default:
- break;
- }
- // All ValueVector types have been handled.
- throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
- }
-
- public static ValueHolder getValue(ValueVector vector, int index) {
- MajorType type = vector.getField().getType();
- ValueHolder holder;
- switch(type.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
- minor.class?starts_with("Decimal28") || minor.class?starts_with("Decimal38")>
- switch (type.getMode()) {
- case REQUIRED:
- holder = new ${minor.class}Holder();
- ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
- return holder;
- case OPTIONAL:
- holder = new Nullable${minor.class}Holder();
- ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
- if (((Nullable${minor.class}Holder)holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
- }
- return holder;
- }
- <#else>
- switch (type.getMode()) {
- case REQUIRED:
- holder = new ${minor.class}Holder();
- ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
- return holder;
- case OPTIONAL:
- holder = new Nullable${minor.class}Holder();
- ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
- if (((Nullable${minor.class}Holder)holder).isSet == 1) {
- ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
- }
- return holder;
- }
- </#if>
- </#list>
-</#list>
- case GENERIC_OBJECT:
- holder = new ObjectHolder();
- ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index) ;
- break;
- }
-
- throw new UnsupportedOperationException(buildErrorMessage("get value", type));
- }
-
- public static void setValue(ValueVector vector, int index, ValueHolder holder) {
- MajorType type = vector.getField().getType();
-
- switch(type.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
- return;
- case OPTIONAL:
- if (((Nullable${minor.class}Holder) holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
- }
- return;
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
- return;
- default:
- throw new UnsupportedOperationException(buildErrorMessage("set value", type));
- }
- }
-
- public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
- MajorType type = vector.getField().getType();
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
- return;
- case OPTIONAL:
- if (((Nullable${minor.class}Holder) holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
- } else {
- ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
- }
- return;
- }
- </#list>
- </#list>
- case GENERIC_OBJECT:
- ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
- default:
- throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
- }
- }
-
- public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
- MajorType type1 = v1.getField().getType();
- MajorType type2 = v2.getField().getType();
-
- if (type1.getMinorType() != type2.getMinorType()) {
- return false;
- }
-
- switch(type1.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) ==
- ((${minor.class}Vector) v2).getAccessor().get(v2index) )
- return true;
- break;
- </#list>
-</#list>
- default:
- break;
- }
- return false;
- }
-
- /**
- * Create a ValueHolder of MajorType.
- * @param type
- * @return
- */
- public static ValueHolder createValueHolder(MajorType type) {
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Holder();
- case OPTIONAL:
- return new Nullable${minor.class}Holder();
- case REPEATED:
- return new Repeated${minor.class}Holder();
- }
- </#list>
- </#list>
- case GENERIC_OBJECT:
- return new ObjectHolder();
- default:
- throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
- }
- }
-
- public static boolean isNull(ValueHolder holder) {
- MajorType type = TypeHelper.getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return true;
- case OPTIONAL:
- return ((Nullable${minor.class}Holder) holder).isSet == 0;
- case REPEATED:
- return true;
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
- }
- }
-
- public static ValueHolder deNullify(ValueHolder holder) {
- MajorType type = TypeHelper.getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return holder;
- case OPTIONAL:
- if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
- ${minor.class}Holder newHolder = new ${minor.class}Holder();
-
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
- </#list>
-
- return newHolder;
- } else {
- throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
- }
- case REPEATED:
- return holder;
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
- }
- }
-
- public static ValueHolder nullify(ValueHolder holder) {
- MajorType type = TypeHelper.getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
- newHolder.isSet = 1;
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
- </#list>
- return newHolder;
- case OPTIONAL:
- return holder;
- case REPEATED:
- throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
- }
- }
-
- public static MajorType getValueHolderType(ValueHolder holder) {
-
- if (0 == 1) {
- return null;
- }
- <#list vv.types as type>
- <#list type.minor as minor>
- else if (holder instanceof ${minor.class}Holder) {
- return ((${minor.class}Holder) holder).TYPE;
- } else if (holder instanceof Nullable${minor.class}Holder) {
- return ((Nullable${minor.class}Holder) holder).TYPE;
- }
- </#list>
- </#list>
-
- throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
-
- }
-
-}
[18/45] drill git commit: DRILL-3987: (REFACTOR) Common and Vector
modules building.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index 49a64cf..0e5b436 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -26,6 +26,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.apache.calcite.rel.RelFieldCollation.Direction;
import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
@@ -37,6 +38,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
@@ -45,8 +47,6 @@ import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.ExternalSort;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
@@ -71,7 +71,6 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.AbstractContainerVector;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.calcite.rel.RelFieldCollation.Direction;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
@@ -145,7 +144,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
SPILL_THRESHOLD = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD);
dirs = Iterators.cycle(config.getStringList(ExecConstants.EXTERNAL_SORT_SPILL_DIRS));
copierAllocator = oContext.getAllocator().getChildAllocator(
- context, PriorityQueueCopier.INITIAL_ALLOCATION, PriorityQueueCopier.MAX_ALLOCATION, true);
+ context.asLimitConsumer(), PriorityQueueCopier.INITIAL_ALLOCATION, PriorityQueueCopier.MAX_ALLOCATION, true);
FragmentHandle handle = context.getHandle();
fileName = String.format("%s/major_fragment_%s/minor_fragment_%s/operator_%s", QueryIdHelper.getQueryId(handle.getQueryId()),
handle.getMajorFragmentId(), handle.getMinorFragmentId(), popConfig.getOperatorId());
@@ -321,7 +320,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
} catch(InterruptedException e) {
return IterOutcome.STOP;
} catch (OutOfMemoryException e) {
- throw new OutOfMemoryRuntimeException(e);
+ throw new OutOfMemoryException(e);
}
}
int count = sv2.getCount();
@@ -343,7 +342,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
// current memory used is more than 95% of memory usage limit of this operator
(totalSizeInMemory > .95 * popConfig.getMaxAllocation()) ||
// current memory used is more than 95% of memory usage limit of this fragment
- (totalSizeInMemory > .95 * oContext.getAllocator().getFragmentLimit()) ||
+ (totalSizeInMemory > .95 * oContext.getAllocator().getLimit()) ||
// Number of incoming batches (BatchGroups) exceed the limit and number of incoming batches accumulated
// since the last spill exceed the defined limit
(batchGroups.size() > SPILL_THRESHOLD && batchesSinceLastSpill >= SPILL_BATCH_GROUP_SIZE)) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
index f06c397..229fe65 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
@@ -23,8 +23,8 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.ops.OperatorStats;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
index 4f91317..65d164d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.record;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
index 3dab51f..dcf9a7d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.record.selection;
import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.record.DeadBuf;
/**
@@ -90,7 +90,7 @@ public class SelectionVector2 implements AutoCloseable {
public boolean allocateNewSafe(int size) {
try {
allocateNew(size);
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
index 3e2adaf..3dfe03f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
@@ -27,7 +27,7 @@ import java.util.List;
import org.apache.drill.exec.memory.BufferAllocator;
import com.google.protobuf.CodedInputStream;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
/**
* Modified version of {@link io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder} that avoids bytebuf copy.
@@ -85,7 +85,7 @@ public class ProtobufLengthDecoder extends ByteToMessageDecoder {
ByteBuf outBuf;
try {
outBuf = allocator.buffer(length);
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
logger.warn("Failure allocating buffer on incoming stream due to memory limits. Current Allocation: {}.", allocator.getAllocatedMemory());
in.resetReaderIndex();
outOfMemoryHandler.handle();
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
index 41285c7..2f1ca56 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
@@ -21,7 +21,7 @@ import java.util.Collection;
import java.util.Map;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.planner.logical.ColumnList;
import org.apache.drill.exec.record.MaterializedField.Key;
import org.apache.drill.exec.vector.ValueVector;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
index f1b55e7..4df1772 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.store;
import java.util.Map;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField.Key;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
index 8160f1c..1d22fed 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
@@ -25,7 +25,7 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
index c742690..d2c2425 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
@@ -22,9 +22,9 @@ import java.util.Map;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index c22392a..91cd112 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -27,7 +27,7 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.AbstractWriter;
import org.apache.drill.exec.physical.base.PhysicalOperator;
@@ -57,7 +57,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-
import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.ParquetFileWriter;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index f784566..a2cd1fa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -30,7 +30,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.planner.physical.WriterPrel;
@@ -47,7 +47,6 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.parquet.column.ColumnWriteStore;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
import org.apache.parquet.column.impl.ColumnWriteStoreV1;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
index f42996b..47d502f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
@@ -31,8 +31,8 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
@@ -45,7 +45,6 @@ import org.apache.drill.exec.vector.complex.RepeatedValueVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.format.FileMetaData;
import org.apache.parquet.format.SchemaElement;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 9e6919b..27536d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -33,8 +33,8 @@ import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
@@ -50,7 +50,6 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
import org.apache.hadoop.fs.Path;
-
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.hadoop.CodecFactory;
import org.apache.parquet.hadoop.metadata.ColumnPath;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
index 32fa31d..22f6bc3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
@@ -25,8 +25,8 @@ import java.util.List;
import java.util.Map;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField.Key;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
deleted file mode 100644
index 2348e62..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-public class AssertionUtil {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
-
-
- public static final boolean ASSERT_ENABLED;
- public static final boolean BOUNDS_CHECKING_ENABLED;
-
- static{
- boolean isAssertEnabled = false;
- assert isAssertEnabled = true;
- ASSERT_ENABLED = isAssertEnabled;
- BOUNDS_CHECKING_ENABLED = ASSERT_ENABLED || !"true".equals(System.getProperty("drill.enable_unsafe_memory_access"));
- }
-
- public static boolean isAssertionsEnabled(){
- return ASSERT_ENABLED;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
deleted file mode 100644
index 6966357..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
+++ /dev/null
@@ -1,730 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import io.netty.buffer.DrillBuf;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.drill.common.util.CoreDecimalUtility;
-import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
-import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
-
-public class DecimalUtility extends CoreDecimalUtility{
-
- public final static int MAX_DIGITS = 9;
- public final static int DIGITS_BASE = 1000000000;
- public final static int DIGITS_MAX = 999999999;
- public final static int integerSize = (Integer.SIZE/8);
-
- public final static String[] decimalToString = {"",
- "0",
- "00",
- "000",
- "0000",
- "00000",
- "000000",
- "0000000",
- "00000000",
- "000000000"};
-
- public final static long[] scale_long_constants = {
- 1,
- 10,
- 100,
- 1000,
- 10000,
- 100000,
- 1000000,
- 10000000,
- 100000000,
- 1000000000,
- 10000000000l,
- 100000000000l,
- 1000000000000l,
- 10000000000000l,
- 100000000000000l,
- 1000000000000000l,
- 10000000000000000l,
- 100000000000000000l,
- 1000000000000000000l};
-
- /*
- * Simple function that returns the static precomputed
- * power of ten, instead of using Math.pow
- */
- public static long getPowerOfTen(int power) {
- assert power >= 0 && power < scale_long_constants.length;
- return scale_long_constants[(power)];
- }
-
- /*
- * Math.pow returns a double and while multiplying with large digits
- * in the decimal data type we encounter noise. So instead of multiplying
- * with Math.pow we use the static constants to perform the multiplication
- */
- public static long adjustScaleMultiply(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input * scale_long_constants[index];
- } else {
- return input / scale_long_constants[index];
- }
- }
-
- public static long adjustScaleDivide(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input / scale_long_constants[index];
- } else {
- return input * scale_long_constants[index];
- }
- }
-
- /* Given the number of actual digits this function returns the
- * number of indexes it will occupy in the array of integers
- * which are stored in base 1 billion
- */
- public static int roundUp(int ndigits) {
- return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
- }
-
- /* Returns a string representation of the given integer
- * If the length of the given integer is less than the
- * passed length, this function will prepend zeroes to the string
- */
- public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
- String value = ((Integer) number).toString();
- int length = value.length();
-
- StringBuilder str = new StringBuilder();
- str.append(decimalToString[desiredLength - length]);
- str.append(value);
-
- return str;
- }
-
- public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
- String value = ((Long) number).toString();
- int length = value.length();
-
- StringBuilder str = new StringBuilder();
-
- // Desired length can be > MAX_DIGITS
- int zeroesLength = desiredLength - length;
- while (zeroesLength > MAX_DIGITS) {
- str.append(decimalToString[MAX_DIGITS]);
- zeroesLength -= MAX_DIGITS;
- }
- str.append(decimalToString[zeroesLength]);
- str.append(value);
-
- return str;
- }
-
- public static BigDecimal getBigDecimalFromIntermediate(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
-
- // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
- return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false);
- }
-
- public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
-
- // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
- return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true);
- }
-
- public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) {
- byte[] value = new byte[length];
- bytebuf.getBytes(start, value, 0, length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
- byte[] value = new byte[length];
- bytebuf.get(value);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- /* Create a BigDecimal object using the data in the DrillBuf.
- * This function assumes that data is provided in a non-dense format
- * It works on both sparse and intermediate representations.
- */
- public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) {
-
- // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
- int actualDigits;
-
- // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out
- BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
-
- BigInteger base = BigInteger.valueOf(DIGITS_BASE);
-
- for (int i = 1; i < nDecimalDigits; i++) {
-
- BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * integerSize)));
- decimalDigits = decimalDigits.multiply(base);
- decimalDigits = decimalDigits.add(temp);
- }
-
- // Truncate any additional padding we might have added
- if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
- BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
- decimalDigits = decimalDigits.divide(truncate);
- }
-
- // set the sign
- if ((data.getInt(startIndex) & 0x80000000) != 0) {
- decimalDigits = decimalDigits.negate();
- }
-
- BigDecimal decimal = new BigDecimal(decimalDigits, scale);
-
- return decimal;
- }
-
- /* This function returns a BigDecimal object from the dense decimal representation.
- * First step is to convert the dense representation into an intermediate representation
- * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object
- */
- public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
-
- /* This method converts the dense representation to
- * an intermediate representation. The intermediate
- * representation has one more integer than the dense
- * representation.
- */
- byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * integerSize)];
-
- // Start storing from the least significant byte of the first integer
- int intermediateIndex = 3;
-
- int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
- int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
-
- int maskIndex;
- int shiftOrder;
- byte shiftBits;
-
- // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
- if (maxPrecision == 38) {
- maskIndex = 0;
- shiftOrder = 6;
- shiftBits = 0x00;
- intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
- } else if (maxPrecision == 28) {
- maskIndex = 1;
- shiftOrder = 4;
- shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
- intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
- } else {
- throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
- }
-
- int inputIndex = 1;
- boolean sign = false;
-
- if ((data.getByte(startIndex) & 0x80) != 0) {
- sign = true;
- }
-
- while (inputIndex < width) {
-
- intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
-
- shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
-
- inputIndex++;
- intermediateIndex++;
-
- if (((inputIndex - 1) % integerSize) == 0) {
- shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
- maskIndex++;
- shiftOrder -= 2;
- }
-
- }
- /* copy the last byte */
- intermediateBytes[intermediateIndex] = shiftBits;
-
- if (sign == true) {
- intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
- }
- DrillBuf intermediate = data.getAllocator().buffer(intermediateBytes.length);
- intermediate.setBytes(0, intermediateBytes);
-
- BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
- intermediate.release();
- return ret;
- }
-
- /*
- * Function converts the BigDecimal and stores it in out internal sparse representation
- */
- public static void getSparseFromBigDecimal(BigDecimal input, DrillBuf data, int startIndex, int scale, int precision, int nDecimalDigits) {
-
- // Initialize the buffer
- for (int i = 0; i < nDecimalDigits; i++) {
- data.setInt(startIndex + (i * integerSize), 0);
- }
-
- boolean sign = false;
-
- if (input.signum() == -1) {
- // negative input
- sign = true;
- input = input.abs();
- }
-
- // Truncate the input as per the scale provided
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- // Separate out the integer part
- BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
-
- int destIndex = nDecimalDigits - roundUp(scale) - 1;
-
- // we use base 1 billion integer digits for out integernal representation
- BigDecimal base = new BigDecimal(DIGITS_BASE);
-
- while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
- // store the modulo as the integer value
- data.setInt(startIndex + (destIndex * integerSize), (integerPart.remainder(base)).intValue());
- destIndex--;
- // Divide by base 1 billion
- integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
- }
-
- /* Sparse representation contains padding of additional zeroes
- * so each digit contains MAX_DIGITS for ease of arithmetic
- */
- int actualDigits;
- if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
- // Pad additional zeroes
- scale = scale + (MAX_DIGITS - actualDigits);
- input = input.setScale(scale, BigDecimal.ROUND_DOWN);
- }
-
- //separate out the fractional part
- BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
-
- destIndex = nDecimalDigits - 1;
-
- while (scale > 0) {
- // Get next set of MAX_DIGITS (9) store it in the DrillBuf
- fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
- BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
-
- data.setInt(startIndex + (destIndex * integerSize), (temp.unscaledValue().intValue()));
- destIndex--;
-
- fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
- scale -= MAX_DIGITS;
- }
-
- // Set the negative sign
- if (sign == true) {
- data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
- }
-
- }
-
-
- public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- return (input.unscaledValue().longValue());
- }
-
- public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
- return BigDecimal.valueOf(input, scale);
- }
-
- public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
- return BigDecimal.valueOf(input, scale);
- }
-
-
- public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) {
-
- int invert = 1;
-
- /* If signs are different then simply look at the
- * sign of the two inputs and determine which is greater
- */
- if (leftSign != rightSign) {
-
- return((leftSign == true) ? -1 : 1);
- } else if(leftSign == true) {
- /* Both inputs are negative, at the end we will
- * have to invert the comparison
- */
- invert = -1;
- }
-
- int cmp = 0;
-
- for (int i = 0; i < width; i++) {
- byte leftByte = left.getByte(leftStart + i);
- byte rightByte = right.getByte(rightStart + i);
- // Unsigned byte comparison
- if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
- cmp = 1;
- break;
- } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
- cmp = -1;
- break;
- }
- }
- cmp *= invert; // invert the comparison if both were negative values
-
- return cmp;
- }
-
- public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) {
- int value = buffer.getInt(start + (index * 4));
-
- if (index == 0) {
- /* the first byte contains sign bit, return value without it */
- value = (value & 0x7FFFFFFF);
- }
- return value;
- }
-
- public static void setInteger(DrillBuf buffer, int start, int index, int value) {
- buffer.setInt(start + (index * 4), value);
- }
-
- public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
-
- int invert = 1;
-
- if (absCompare == false) {
- if (leftSign != rightSign) {
- return (leftSign == true) ? -1 : 1;
- }
-
- // Both values are negative invert the outcome of the comparison
- if (leftSign == true) {
- invert = -1;
- }
- }
-
- int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
- return cmp * invert;
- }
- public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
- /* compute the number of integer digits in each decimal */
- int leftInt = leftPrecision - leftScale;
- int rightInt = rightPrecision - rightScale;
-
- /* compute the number of indexes required for storing integer digits */
- int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt);
- int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt);
-
- /* compute number of indexes required for storing scale */
- int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale);
- int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale);
-
- /* compute index of the most significant integer digits */
- int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
- int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
- int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
-
- /* Discard the zeroes in the integer part */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- break;
- }
-
- /* Digit in this location is zero, decrement the actual number
- * of integer digits
- */
- leftIntRoundedUp--;
- leftIndex1++;
- }
-
- /* If we reached the stop index then the number of integers is zero */
- if (leftIndex1 == leftStopIndex) {
- leftIntRoundedUp = 0;
- }
-
- while (rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- break;
- }
-
- /* Digit in this location is zero, decrement the actual number
- * of integer digits
- */
- rightIntRoundedUp--;
- rightIndex1++;
- }
-
- if (rightIndex1 == rightStopIndex) {
- rightIntRoundedUp = 0;
- }
-
- /* We have the accurate number of non-zero integer digits,
- * if the number of integer digits are different then we can determine
- * which decimal is larger and needn't go down to comparing individual values
- */
- if (leftIntRoundedUp > rightIntRoundedUp) {
- return 1;
- }
- else if (rightIntRoundedUp > leftIntRoundedUp) {
- return -1;
- }
-
- /* The number of integer digits are the same, set the each index
- * to the first non-zero integer and compare each digit
- */
- leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
- }
-
- /* The integer part of both the decimal's are equal, now compare
- * each individual fractional part. Set the index to be at the
- * beginning of the fractional part
- */
- leftIndex1 = leftStopIndex;
- rightIndex1 = rightStopIndex;
-
- /* Stop indexes will be the end of the array */
- leftStopIndex = nDecimalDigits;
- rightStopIndex = nDecimalDigits;
-
- /* compare the two fractional parts of the decimal */
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
- }
-
- /* Till now the fractional part of the decimals are equal, check
- * if one of the decimal has fractional part that is remaining
- * and is non-zero
- */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- return 1;
- }
- leftIndex1++;
- }
-
- while(rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- return -1;
- }
- rightIndex1++;
- }
-
- /* Both decimal values are equal */
- return 0;
- }
-
- public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
- byte[] value = Arrays.copyOfRange(bytes, start, start + length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
- int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale);
- int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale);
-
- if (desiredScale < currentScale) {
-
- boolean roundUp = false;
-
- //Extract the first digit to be truncated to check if we need to round up
- int truncatedScaleIndex = desiredScale + 1;
- if (truncatedScaleIndex <= currentScale) {
- int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
- extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex);
- int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
- int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
- if (temp != 0) {
- extractDigit = extractDigit / (int) (Math.pow(10, temp));
- }
- if ((extractDigit % 10) > 4) {
- roundUp = true;
- }
- }
-
- // Get the source index beyond which we will truncate
- int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
- int srcIndex = srcIntIndex + newScaleRoundedUp;
-
- // Truncate the remaining fractional part, move the integer part
- int destIndex = nDecimalDigits - 1;
- if (srcIndex != destIndex) {
- while (srcIndex >= 0) {
- setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
- }
-
- // Set the remaining portion of the decimal to be zeroes
- while (destIndex >= 0) {
- setInteger(result, start, destIndex--, 0);
- }
- srcIndex = nDecimalDigits - 1;
- }
-
- // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
- int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
- if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
- truncateFactor = (int) Math.pow(10, truncateFactor);
- int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
- fractionalDigits /= truncateFactor;
- setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
- }
-
- // Finally round up the digit if needed
- if (roundUp == true) {
- srcIndex = nDecimalDigits - 1;
- int carry;
- if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
- carry = truncateFactor;
- } else {
- carry = 1;
- }
-
- while (srcIndex >= 0) {
- int value = getIntegerFromSparseBuffer(result, start, srcIndex);
- value += carry;
-
- if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
- setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
- carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
- } else {
- setInteger(result, start, srcIndex--, value);
- carry = 0;
- break;
- }
- }
- }
- } else if (desiredScale > currentScale) {
- // Add fractional digits to the decimal
-
- // Check if we need to shift the decimal digits to the left
- if (newScaleRoundedUp > origScaleRoundedUp) {
- int srcIndex = 0;
- int destIndex = newScaleRoundedUp - origScaleRoundedUp;
-
- // Check while extending scale, we are not overwriting integer part
- while (srcIndex < destIndex) {
- if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
- throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
- }
- }
-
- srcIndex = 0;
- while (destIndex < nDecimalDigits) {
- setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
- }
-
- // Clear the remaining part
- while (srcIndex < nDecimalDigits) {
- setInteger(result, start, srcIndex++, 0);
- }
- }
- }
- }
-
- public static int getFirstFractionalDigit(int decimal, int scale) {
- if (scale == 0) {
- return 0;
- }
- int temp = (int) adjustScaleDivide(decimal, scale - 1);
- return Math.abs(temp % 10);
- }
-
- public static int getFirstFractionalDigit(long decimal, int scale) {
- if (scale == 0) {
- return 0;
- }
- long temp = adjustScaleDivide(decimal, scale - 1);
- return (int) (Math.abs(temp % 10));
- }
-
- public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) {
- if (scale == 0) {
- return 0;
- }
-
- int index = nDecimalDigits - roundUp(scale);
- return (int) (adjustScaleDivide(data.getInt(start + (index * integerSize)), MAX_DIGITS - 1));
- }
-
- public static int compareSparseSamePrecScale(DrillBuf left, int lStart, byte[] right, int length) {
- // check the sign first
- boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
- boolean rSign = ByteFunctionHelpers.getSign(right);
- int cmp = 0;
-
- if (lSign != rSign) {
- return (lSign == false) ? 1 : -1;
- }
-
- // invert the comparison if we are comparing negative numbers
- int invert = (lSign == true) ? -1 : 1;
-
- // compare byte by byte
- int n = 0;
- int lPos = lStart;
- int rPos = 0;
- while (n < length/4) {
- int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
- int rightInt = ByteFunctionHelpers.getInteger(right, n);
- if (leftInt != rightInt) {
- cmp = (leftInt - rightInt ) > 0 ? 1 : -1;
- break;
- }
- n++;
- }
- return cmp * invert;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
index d52cb5d..3f2e3f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
import org.apache.drill.common.AutoCloseables;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.BitControl.Collector;
import org.apache.drill.exec.record.RawFragmentBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
index cfe5b6b..753c75d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
@@ -29,8 +29,8 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.BitData;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index e17eba5..6adbfc0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -43,8 +43,7 @@ import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.DistributedSemaphore;
import org.apache.drill.exec.coord.DistributedSemaphore.DistributedLease;
import org.apache.drill.exec.exception.OptimizerException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.opt.BasicOptimizer;
@@ -247,7 +246,7 @@ public class Foreman implements Runnable {
throw new IllegalStateException();
}
injector.injectChecked(queryContext.getExecutionControls(), "run-try-end", ForemanException.class);
- } catch (final OutOfMemoryException | OutOfMemoryRuntimeException e) {
+ } catch (final OutOfMemoryException e) {
moveToState(QueryState.FAILED, UserException.memoryError(e).build(logger));
} catch (final ForemanException e) {
moveToState(QueryState.FAILED, e);
@@ -407,7 +406,7 @@ public class Foreman implements Runnable {
setupRootFragment(rootPlanFragment, work.getRootOperator());
setupNonRootFragments(planFragments);
- drillbitContext.getAllocator().resetFragmentLimits(); // TODO a global effect for this query?!?
+ drillbitContext.getAllocator().resetLimits(); // TODO a global effect for this query?!?
moveToState(QueryState.RUNNING, null);
logger.debug("Fragments running.");
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
index 094316e..9101c13 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
@@ -29,7 +29,7 @@ import org.apache.drill.common.SerializedExecutor;
import org.apache.drill.common.concurrent.ExtendedLatch;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.coord.ClusterCoordinator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.FragmentContext.ExecutorState;
import org.apache.drill.exec.physical.base.FragmentRoot;
@@ -261,7 +261,7 @@ public class FragmentExecutor implements Runnable {
}
});
- } catch (OutOfMemoryError | OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryError | OutOfMemoryException e) {
if (!(e instanceof OutOfMemoryError) || "Direct buffer memory".equals(e.getMessage())) {
fail(UserException.memoryError(e).build(logger));
} else {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java b/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java
index 28f6390..735aaa2 100644
--- a/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java
+++ b/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java
@@ -28,7 +28,7 @@ import java.util.Map;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.store.parquet.ColumnDataReader;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -217,7 +217,7 @@ public class ColumnChunkIncReadStore implements PageReadStore {
}
in.close();
return null;
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
throw e; // throw as it is
} catch (Exception e) {
throw new DrillRuntimeException("Error reading page." +
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
index 8f16c44..1844c32 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.client.PrintingResultsListener;
import org.apache.drill.exec.client.QuerySubmitter.Format;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.user.AwaitableUserResultsListener;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
index 10fd5da..67cabff 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
@@ -18,7 +18,7 @@
package org.apache.drill;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import org.apache.drill.exec.testing.Controls;
@@ -42,7 +42,7 @@ public class TestAllocationException extends BaseTestQuery {
final String controls = Controls.newBuilder()
.addException(TopLevelAllocator.class,
TopLevelAllocator.CHILD_BUFFER_INJECTION_SITE,
- OutOfMemoryRuntimeException.class,
+ OutOfMemoryException.class,
200,
1
).build();
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
index d522663..65adc02 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
@@ -170,7 +170,7 @@ public class TestAllocators {
try {
oContext31.getAllocator().buffer(4400000);
fail("Fragment 3 should fail to allocate buffer");
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
// expected
}
@@ -179,7 +179,7 @@ public class TestAllocators {
try {
DrillBuf b32 = oContext32.getAllocator().buffer(4400000);
b32.release();
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
fail("Fragment 3 failed to allocate buffer");
} finally {
closeOp(oContext32);
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
index f269bd7..0124f9e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
@@ -29,8 +29,8 @@ import java.util.List;
import java.util.Random;
import org.apache.drill.PlanTestBase;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.physical.MinorFragmentEndpoint;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
index a32322b..a10a76d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.server;
import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.glassfish.hk2.api.Factory;
public class DrillClientFactory implements Factory<DrillClient> {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/pom.xml
----------------------------------------------------------------------
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 4bd968e..6001944 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -56,6 +56,12 @@
<artifactId>hppc</artifactId>
<version>0.4.2</version>
</dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
</dependencies>
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/includes/vv_imports.ftl b/exec/vector/src/main/codegen/includes/vv_imports.ftl
index 0954dcf..e6dc973 100644
--- a/exec/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/vector/src/main/codegen/includes/vv_imports.ftl
@@ -45,7 +45,7 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import java.util.Arrays;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/BasicTypeHelper.java b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
new file mode 100644
index 0000000..723687a
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
@@ -0,0 +1,544 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/BasicTypeHelper.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr;
+
+<#include "/@includes/vv_imports.ftl" />
+import org.apache.drill.exec.vector.complex.UnionVector;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.complex.RepeatedMapVector;
+import org.apache.drill.exec.util.CallBack;
+
+public class BasicTypeHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicTypeHelper.class);
+
+ private static final int WIDTH_ESTIMATE = 50;
+
+ // Default length when casting to varchar : 65536 = 2^16
+ // This only defines an absolute maximum for values, setting
+ // a high value like this will not inflate the size for small values
+ public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
+
+ private static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
+ return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
+ }
+
+ private static String buildErrorMessage(final String operation, final MajorType type) {
+ return buildErrorMessage(operation, type.getMinorType(), type.getMode());
+ }
+
+ public static int getSize(MajorType major) {
+ switch (major.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
+ minor.class?substring(0, 3) == "PRO" ||
+ minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
+ </#list>
+</#list>
+ case FIXEDCHAR: return major.getWidth();
+ case FIXED16CHAR: return major.getWidth();
+ case FIXEDBINARY: return major.getWidth();
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get size", major));
+ }
+
+ public static ValueVector getNewVector(SchemaPath parentPath, String name, BufferAllocator allocator, MajorType type, CallBack callback){
+ SchemaPath child = parentPath.getChild(name);
+ MaterializedField field = MaterializedField.create(child, type);
+ return getNewVector(field, allocator, callback);
+ }
+
+
+ public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
+ switch (type) {
+ case UNION:
+ return UnionVector.class;
+ case MAP:
+ switch (mode) {
+ case OPTIONAL:
+ case REQUIRED:
+ return MapVector.class;
+ case REPEATED:
+ return RepeatedMapVector.class;
+ }
+
+ case LIST:
+ switch (mode) {
+ case REPEATED:
+ return RepeatedListVector.class;
+ case REQUIRED:
+ case OPTIONAL:
+ return ListVector.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}Vector.class;
+ case OPTIONAL:
+ return Nullable${minor.class}Vector.class;
+ case REPEATED:
+ return Repeated${minor.class}Vector.class;
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT :
+ return ObjectVector.class ;
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
+ }
+ public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
+ switch (type) {
+ case MAP:
+ switch (mode) {
+ case REQUIRED:
+ if (!isSingularRepeated)
+ return SingleMapReaderImpl.class;
+ else
+ return SingleLikeRepeatedMapReaderImpl.class;
+ case REPEATED:
+ return RepeatedMapReaderImpl.class;
+ }
+ case LIST:
+ switch (mode) {
+ case REQUIRED:
+ return SingleListReaderImpl.class;
+ case REPEATED:
+ return RepeatedListReaderImpl.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}ReaderImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}ReaderImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}ReaderImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
+ }
+
+ public static Class<?> getWriterInterface( MinorType type, DataMode mode){
+ switch (type) {
+ case UNION: return UnionWriter.class;
+ case MAP: return MapWriter.class;
+ case LIST: return ListWriter.class;
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}: return ${minor.class}Writer.class;
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
+ }
+
+ public static Class<?> getWriterImpl( MinorType type, DataMode mode){
+ switch (type) {
+ case UNION:
+ return UnionWriter.class;
+ case MAP:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return SingleMapWriter.class;
+ case REPEATED:
+ return RepeatedMapWriter.class;
+ }
+ case LIST:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return UnionListWriter.class;
+ case REPEATED:
+ return RepeatedListWriter.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}WriterImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}WriterImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}WriterImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
+ }
+
+ public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
+ switch (type) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}HolderReaderImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}HolderReaderImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}HolderReaderImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
+ }
+
+ public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
+ return getNewVector(field, allocator, null);
+ }
+ public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+ MajorType type = field.getType();
+
+ switch (type.getMinorType()) {
+
+ case UNION:
+ return new UnionVector(field, allocator, callBack);
+
+ case MAP:
+ switch (type.getMode()) {
+ case REQUIRED:
+ case OPTIONAL:
+ return new MapVector(field, allocator, callBack);
+ case REPEATED:
+ return new RepeatedMapVector(field, allocator, callBack);
+ }
+ case LIST:
+ switch (type.getMode()) {
+ case REPEATED:
+ return new RepeatedListVector(field, allocator, callBack);
+ case OPTIONAL:
+ case REQUIRED:
+ return new ListVector(field, allocator, callBack);
+ }
+<#list vv. types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Vector(field, allocator);
+ case OPTIONAL:
+ return new Nullable${minor.class}Vector(field, allocator);
+ case REPEATED:
+ return new Repeated${minor.class}Vector(field, allocator);
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ return new ObjectVector(field, allocator) ;
+ default:
+ break;
+ }
+ // All ValueVector types have been handled.
+ throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
+ }
+
+ public static ValueHolder getValue(ValueVector vector, int index) {
+ MajorType type = vector.getField().getType();
+ ValueHolder holder;
+ switch(type.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
+ minor.class?starts_with("Decimal28") || minor.class?starts_with("Decimal38")>
+ switch (type.getMode()) {
+ case REQUIRED:
+ holder = new ${minor.class}Holder();
+ ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
+ return holder;
+ case OPTIONAL:
+ holder = new Nullable${minor.class}Holder();
+ ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
+ if (((Nullable${minor.class}Holder)holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
+ }
+ return holder;
+ }
+ <#else>
+ switch (type.getMode()) {
+ case REQUIRED:
+ holder = new ${minor.class}Holder();
+ ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
+ return holder;
+ case OPTIONAL:
+ holder = new Nullable${minor.class}Holder();
+ ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
+ if (((Nullable${minor.class}Holder)holder).isSet == 1) {
+ ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
+ }
+ return holder;
+ }
+ </#if>
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ holder = new ObjectHolder();
+ ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index) ;
+ break;
+ }
+
+ throw new UnsupportedOperationException(buildErrorMessage("get value", type));
+ }
+
+ public static void setValue(ValueVector vector, int index, ValueHolder holder) {
+ MajorType type = vector.getField().getType();
+
+ switch(type.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
+ return;
+ case OPTIONAL:
+ if (((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
+ }
+ return;
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
+ return;
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("set value", type));
+ }
+ }
+
+ public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
+ MajorType type = vector.getField().getType();
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
+ return;
+ case OPTIONAL:
+ if (((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
+ } else {
+ ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
+ }
+ return;
+ }
+ </#list>
+ </#list>
+ case GENERIC_OBJECT:
+ ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
+ }
+ }
+
+ public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
+ MajorType type1 = v1.getField().getType();
+ MajorType type2 = v2.getField().getType();
+
+ if (type1.getMinorType() != type2.getMinorType()) {
+ return false;
+ }
+
+ switch(type1.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) ==
+ ((${minor.class}Vector) v2).getAccessor().get(v2index) )
+ return true;
+ break;
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ return false;
+ }
+
+ /**
+ * Create a ValueHolder of MajorType.
+ * @param type
+ * @return
+ */
+ public static ValueHolder createValueHolder(MajorType type) {
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Holder();
+ case OPTIONAL:
+ return new Nullable${minor.class}Holder();
+ case REPEATED:
+ return new Repeated${minor.class}Holder();
+ }
+ </#list>
+ </#list>
+ case GENERIC_OBJECT:
+ return new ObjectHolder();
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
+ }
+ }
+
+ public static boolean isNull(ValueHolder holder) {
+ MajorType type = getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return true;
+ case OPTIONAL:
+ return ((Nullable${minor.class}Holder) holder).isSet == 0;
+ case REPEATED:
+ return true;
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
+ }
+ }
+
+ public static ValueHolder deNullify(ValueHolder holder) {
+ MajorType type = getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return holder;
+ case OPTIONAL:
+ if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ${minor.class}Holder newHolder = new ${minor.class}Holder();
+
+ <#assign fields = minor.fields!type.fields />
+ <#list fields as field>
+ newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
+ </#list>
+
+ return newHolder;
+ } else {
+ throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
+ }
+ case REPEATED:
+ return holder;
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
+ }
+ }
+
+ public static ValueHolder nullify(ValueHolder holder) {
+ MajorType type = getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
+ newHolder.isSet = 1;
+ <#assign fields = minor.fields!type.fields />
+ <#list fields as field>
+ newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
+ </#list>
+ return newHolder;
+ case OPTIONAL:
+ return holder;
+ case REPEATED:
+ throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
+ }
+ }
+
+ public static MajorType getValueHolderType(ValueHolder holder) {
+
+ if (0 == 1) {
+ return null;
+ }
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ else if (holder instanceof ${minor.class}Holder) {
+ return ((${minor.class}Holder) holder).TYPE;
+ } else if (holder instanceof Nullable${minor.class}Holder) {
+ return ((Nullable${minor.class}Holder) holder).TYPE;
+ }
+ </#list>
+ </#list>
+
+ throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index d9d5fae..f67614a 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -94,7 +94,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
@Override
public void allocateNew() {
if(!allocateNewSafe()){
- throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
+ throw new OutOfMemoryException("Failure while allocating buffer.");
}
}
@@ -124,7 +124,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
* Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
*
* @param valueCount
- * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer
+ * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer
*/
@Override
public void allocateNew(final int valueCount) {
@@ -154,7 +154,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
/**
* Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
*
- * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer
+ * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer
*/
public void reAlloc() {
final long newAllocationSize = allocationSizeInBytes * 2L;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
index fe91fe6..13bdd4f 100644
--- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -138,7 +138,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public void allocateNew() {
if(!allocateNewSafe()){
- throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
+ throw new OutOfMemoryException("Failure while allocating buffer.");
}
}
@@ -201,7 +201,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
try {
values.allocateNew(valueCount);
bits.allocateNew(valueCount+1);
- } catch(OutOfMemoryRuntimeException e) {
+ } catch(OutOfMemoryException e) {
clear();
throw e;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
index 2f5ec18..21f5616 100644
--- a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
@@ -204,7 +204,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
try {
offsets.allocateNew();
values.allocateNew();
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
clear();
throw e;
}
@@ -223,7 +223,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
try {
offsets.allocateNew(valueCount + 1);
values.allocateNew(totalBytes, innerValueCount);
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
clear();
throw e;
}
@@ -249,7 +249,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
try {
offsets.allocateNew(valueCount + 1);
values.allocateNew(innerValueCount);
- } catch(OutOfMemoryRuntimeException e){
+ } catch(OutOfMemoryException e){
clear();
throw e;
}
[14/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
DRILL-3987: (MOVE) Extract key vector, field reader, complex/field writer classes.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9969d8bd
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9969d8bd
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9969d8bd
Branch: refs/heads/master
Commit: 9969d8bd12b090409f1a9badb904e81aeabc2aa7
Parents: 9b88d59
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Nov 7 15:54:35 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 17:00:45 2015 -0800
----------------------------------------------------------------------
exec/java-exec/pom.xml | 5 +
.../src/main/codegen/data/ValueVectorTypes.tdd | 168 ----
.../codegen/templates/AbstractFieldReader.java | 124 ---
.../codegen/templates/AbstractFieldWriter.java | 147 ----
.../AbstractPromotableFieldWriter.java | 142 ----
.../src/main/codegen/templates/BaseReader.java | 73 --
.../src/main/codegen/templates/BaseWriter.java | 100 ---
.../main/codegen/templates/ComplexCopier.java | 131 ---
.../main/codegen/templates/ComplexReaders.java | 183 -----
.../main/codegen/templates/ComplexWriters.java | 159 ----
.../codegen/templates/FixedValueVectors.java | 815 -------------------
.../codegen/templates/HolderReaderImpl.java | 292 -------
.../src/main/codegen/templates/ListWriters.java | 234 ------
.../src/main/codegen/templates/MapWriters.java | 241 ------
.../src/main/codegen/templates/NullReader.java | 140 ----
.../codegen/templates/NullableValueVectors.java | 636 ---------------
.../codegen/templates/RepeatedValueVectors.java | 427 ----------
.../src/main/codegen/templates/TypeHelper.java | 600 --------------
.../main/codegen/templates/UnionListWriter.java | 185 -----
.../src/main/codegen/templates/UnionReader.java | 196 -----
.../src/main/codegen/templates/UnionVector.java | 479 -----------
.../src/main/codegen/templates/UnionWriter.java | 228 ------
.../main/codegen/templates/ValueHolders.java | 116 ---
.../templates/VariableLengthVectors.java | 645 ---------------
.../src/main/java/io/netty/buffer/DrillBuf.java | 763 -----------------
.../java/io/netty/buffer/FakeAllocator.java | 164 ----
.../main/java/io/netty/buffer/LargeBuffer.java | 350 --------
.../netty/buffer/PooledByteBufAllocatorL.java | 258 ------
.../netty/buffer/UnsafeDirectLittleEndian.java | 247 ------
.../exception/OversizedAllocationException.java | 51 --
.../exception/SchemaChangeRuntimeException.java | 42 -
.../drill/exec/expr/holders/ComplexHolder.java | 25 -
.../drill/exec/expr/holders/ObjectHolder.java | 37 -
.../exec/expr/holders/RepeatedListHolder.java | 23 -
.../exec/expr/holders/RepeatedMapHolder.java | 23 -
.../drill/exec/expr/holders/UnionHolder.java | 37 -
.../drill/exec/expr/holders/ValueHolder.java | 31 -
.../drill/exec/memory/BufferAllocator.java | 136 ----
.../memory/OutOfMemoryRuntimeException.java | 49 --
.../drill/exec/record/MaterializedField.java | 324 --------
.../apache/drill/exec/record/TransferPair.java | 27 -
.../apache/drill/exec/record/TypedFieldId.java | 272 -------
.../org/apache/drill/exec/util/CallBack.java | 23 -
.../drill/exec/util/JsonStringArrayList.java | 63 --
.../drill/exec/util/JsonStringHashMap.java | 82 --
.../org/apache/drill/exec/util/Pointer.java | 28 -
.../apache/drill/exec/util/TextSerializer.java | 39 -
.../drill/exec/vector/AddOrGetResult.java | 38 -
.../drill/exec/vector/AllocationHelper.java | 61 --
.../drill/exec/vector/BaseDataValueVector.java | 90 --
.../drill/exec/vector/BaseValueVector.java | 123 ---
.../org/apache/drill/exec/vector/BitVector.java | 448 ----------
.../drill/exec/vector/FixedWidthVector.java | 35 -
.../drill/exec/vector/NullableVector.java | 23 -
.../vector/NullableVectorDefinitionSetter.java | 23 -
.../apache/drill/exec/vector/ObjectVector.java | 222 -----
.../drill/exec/vector/SchemaChangeCallBack.java | 52 --
.../drill/exec/vector/ValueHolderHelper.java | 201 -----
.../apache/drill/exec/vector/ValueVector.java | 221 -----
.../drill/exec/vector/VariableWidthVector.java | 51 --
.../drill/exec/vector/VectorDescriptor.java | 83 --
.../apache/drill/exec/vector/VectorTrimmer.java | 33 -
.../apache/drill/exec/vector/ZeroVector.java | 175 ----
.../vector/complex/AbstractContainerVector.java | 148 ----
.../exec/vector/complex/AbstractMapVector.java | 276 -------
.../vector/complex/BaseRepeatedValueVector.java | 258 ------
.../vector/complex/ContainerVectorLike.java | 43 -
.../vector/complex/EmptyValuePopulator.java | 53 --
.../drill/exec/vector/complex/FieldIdUtil.java | 124 ---
.../drill/exec/vector/complex/ListVector.java | 327 --------
.../drill/exec/vector/complex/MapVector.java | 380 ---------
.../drill/exec/vector/complex/Positionable.java | 22 -
.../complex/RepeatedFixedWidthVectorLike.java | 40 -
.../exec/vector/complex/RepeatedListVector.java | 429 ----------
.../exec/vector/complex/RepeatedMapVector.java | 611 --------------
.../vector/complex/RepeatedValueVector.java | 86 --
.../RepeatedVariableWidthVectorLike.java | 35 -
.../drill/exec/vector/complex/StateTool.java | 34 -
.../exec/vector/complex/VectorWithOrdinal.java | 30 -
.../vector/complex/impl/AbstractBaseReader.java | 98 ---
.../vector/complex/impl/AbstractBaseWriter.java | 59 --
.../vector/complex/impl/ComplexWriterImpl.java | 218 -----
.../vector/complex/impl/PromotableWriter.java | 194 -----
.../complex/impl/RepeatedListReaderImpl.java | 145 ----
.../complex/impl/RepeatedMapReaderImpl.java | 192 -----
.../impl/SingleLikeRepeatedMapReaderImpl.java | 89 --
.../complex/impl/SingleListReaderImpl.java | 88 --
.../complex/impl/SingleMapReaderImpl.java | 108 ---
.../vector/complex/impl/UnionListReader.java | 100 ---
.../exec/vector/complex/reader/FieldReader.java | 29 -
.../exec/vector/complex/writer/FieldWriter.java | 27 -
exec/pom.xml | 1 +
exec/vector/pom.xml | 146 ++++
exec/vector/src/main/codegen/config.fmpp | 24 +
.../src/main/codegen/data/ValueVectorTypes.tdd | 168 ++++
.../src/main/codegen/includes/license.ftl | 18 +
.../src/main/codegen/includes/vv_imports.ftl | 74 ++
.../codegen/templates/AbstractFieldReader.java | 124 +++
.../codegen/templates/AbstractFieldWriter.java | 147 ++++
.../AbstractPromotableFieldWriter.java | 142 ++++
.../src/main/codegen/templates/BaseReader.java | 73 ++
.../src/main/codegen/templates/BaseWriter.java | 100 +++
.../main/codegen/templates/ComplexCopier.java | 131 +++
.../main/codegen/templates/ComplexReaders.java | 183 +++++
.../main/codegen/templates/ComplexWriters.java | 159 ++++
.../codegen/templates/FixedValueVectors.java | 815 +++++++++++++++++++
.../codegen/templates/HolderReaderImpl.java | 292 +++++++
.../src/main/codegen/templates/ListWriters.java | 234 ++++++
.../src/main/codegen/templates/MapWriters.java | 241 ++++++
.../src/main/codegen/templates/NullReader.java | 140 ++++
.../codegen/templates/NullableValueVectors.java | 636 +++++++++++++++
.../codegen/templates/RepeatedValueVectors.java | 427 ++++++++++
.../src/main/codegen/templates/TypeHelper.java | 600 ++++++++++++++
.../main/codegen/templates/UnionListWriter.java | 185 +++++
.../src/main/codegen/templates/UnionReader.java | 196 +++++
.../src/main/codegen/templates/UnionVector.java | 479 +++++++++++
.../src/main/codegen/templates/UnionWriter.java | 228 ++++++
.../main/codegen/templates/ValueHolders.java | 116 +++
.../templates/VariableLengthVectors.java | 645 +++++++++++++++
.../src/main/java/io/netty/buffer/DrillBuf.java | 763 +++++++++++++++++
.../java/io/netty/buffer/FakeAllocator.java | 164 ++++
.../main/java/io/netty/buffer/LargeBuffer.java | 350 ++++++++
.../netty/buffer/PooledByteBufAllocatorL.java | 258 ++++++
.../netty/buffer/UnsafeDirectLittleEndian.java | 247 ++++++
.../exception/OversizedAllocationException.java | 51 ++
.../exception/SchemaChangeRuntimeException.java | 42 +
.../drill/exec/expr/holders/ComplexHolder.java | 25 +
.../drill/exec/expr/holders/ObjectHolder.java | 37 +
.../exec/expr/holders/RepeatedListHolder.java | 23 +
.../exec/expr/holders/RepeatedMapHolder.java | 23 +
.../drill/exec/expr/holders/UnionHolder.java | 37 +
.../drill/exec/expr/holders/ValueHolder.java | 31 +
.../drill/exec/memory/BufferAllocator.java | 136 ++++
.../memory/OutOfMemoryRuntimeException.java | 49 ++
.../drill/exec/record/MaterializedField.java | 324 ++++++++
.../apache/drill/exec/record/TransferPair.java | 27 +
.../apache/drill/exec/record/TypedFieldId.java | 272 +++++++
.../org/apache/drill/exec/util/CallBack.java | 23 +
.../drill/exec/util/JsonStringArrayList.java | 63 ++
.../drill/exec/util/JsonStringHashMap.java | 82 ++
.../org/apache/drill/exec/util/Pointer.java | 28 +
.../apache/drill/exec/util/TextSerializer.java | 39 +
.../drill/exec/vector/AddOrGetResult.java | 38 +
.../drill/exec/vector/AllocationHelper.java | 61 ++
.../drill/exec/vector/BaseDataValueVector.java | 90 ++
.../drill/exec/vector/BaseValueVector.java | 123 +++
.../org/apache/drill/exec/vector/BitVector.java | 448 ++++++++++
.../drill/exec/vector/FixedWidthVector.java | 35 +
.../drill/exec/vector/NullableVector.java | 23 +
.../vector/NullableVectorDefinitionSetter.java | 23 +
.../apache/drill/exec/vector/ObjectVector.java | 222 +++++
.../drill/exec/vector/SchemaChangeCallBack.java | 52 ++
.../drill/exec/vector/ValueHolderHelper.java | 201 +++++
.../apache/drill/exec/vector/ValueVector.java | 221 +++++
.../drill/exec/vector/VariableWidthVector.java | 51 ++
.../drill/exec/vector/VectorDescriptor.java | 83 ++
.../apache/drill/exec/vector/VectorTrimmer.java | 33 +
.../apache/drill/exec/vector/ZeroVector.java | 175 ++++
.../vector/complex/AbstractContainerVector.java | 148 ++++
.../exec/vector/complex/AbstractMapVector.java | 276 +++++++
.../vector/complex/BaseRepeatedValueVector.java | 258 ++++++
.../vector/complex/ContainerVectorLike.java | 43 +
.../vector/complex/EmptyValuePopulator.java | 53 ++
.../drill/exec/vector/complex/FieldIdUtil.java | 124 +++
.../drill/exec/vector/complex/ListVector.java | 327 ++++++++
.../drill/exec/vector/complex/MapVector.java | 380 +++++++++
.../drill/exec/vector/complex/Positionable.java | 22 +
.../complex/RepeatedFixedWidthVectorLike.java | 40 +
.../exec/vector/complex/RepeatedListVector.java | 429 ++++++++++
.../exec/vector/complex/RepeatedMapVector.java | 611 ++++++++++++++
.../vector/complex/RepeatedValueVector.java | 86 ++
.../RepeatedVariableWidthVectorLike.java | 35 +
.../drill/exec/vector/complex/StateTool.java | 34 +
.../exec/vector/complex/VectorWithOrdinal.java | 30 +
.../vector/complex/impl/AbstractBaseReader.java | 98 +++
.../vector/complex/impl/AbstractBaseWriter.java | 59 ++
.../vector/complex/impl/ComplexWriterImpl.java | 218 +++++
.../vector/complex/impl/PromotableWriter.java | 194 +++++
.../complex/impl/RepeatedListReaderImpl.java | 145 ++++
.../complex/impl/RepeatedMapReaderImpl.java | 192 +++++
.../impl/SingleLikeRepeatedMapReaderImpl.java | 89 ++
.../complex/impl/SingleListReaderImpl.java | 88 ++
.../complex/impl/SingleMapReaderImpl.java | 108 +++
.../vector/complex/impl/UnionListReader.java | 100 +++
.../exec/vector/complex/reader/FieldReader.java | 29 +
.../exec/vector/complex/writer/FieldWriter.java | 27 +
186 files changed, 15945 insertions(+), 15677 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 2011364..da072d0 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -288,6 +288,11 @@
<artifactId>drill-common</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.drill.exec</groupId>
+ <artifactId>vector</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- <dependency> -->
<!-- <groupId>org.apache.drill.exec</groupId> -->
<!-- <version>${project.version}</version> -->
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd b/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
deleted file mode 100644
index 26bf02d..0000000
--- a/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
+++ /dev/null
@@ -1,168 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http:# www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
- modes: [
- {name: "Optional", prefix: "Nullable"},
- {name: "Required", prefix: ""},
- {name: "Repeated", prefix: "Repeated"}
- ],
- types: [
- {
- major: "Fixed",
- width: 1,
- javaType: "byte",
- boxedType: "Byte",
- fields: [{name: "value", type: "byte"}],
- minor: [
- { class: "TinyInt", valueHolder: "IntHolder" },
- { class: "UInt1", valueHolder: "UInt1Holder" }
- ]
- },
- {
- major: "Fixed",
- width: 2,
- javaType: "char",
- boxedType: "Character",
- fields: [{name: "value", type: "char"}],
- minor: [
- { class: "UInt2", valueHolder: "UInt2Holder"}
- ]
- }, {
- major: "Fixed",
- width: 2,
- javaType: "short",
- boxedType: "Short",
- fields: [{name: "value", type: "short"}],
- minor: [
- { class: "SmallInt", valueHolder: "Int2Holder"},
- ]
- },
- {
- major: "Fixed",
- width: 4,
- javaType: "int",
- boxedType: "Integer",
- fields: [{name: "value", type: "int"}],
- minor: [
- { class: "Int", valueHolder: "IntHolder"},
- { class: "UInt4", valueHolder: "UInt4Holder" },
- { class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]},
- { class: "Time", javaType: "int", friendlyType: "DateTime" },
- { class: "IntervalYear", javaType: "int", friendlyType: "Period" }
- { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal", fields: [{name:"value", type:"int"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
- ]
- },
- {
- major: "Fixed",
- width: 8,
- javaType: "long",
- boxedType: "Long",
- fields: [{name: "value", type: "long"}],
- minor: [
- { class: "BigInt"},
- { class: "UInt8" },
- { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
- { class: "Date", javaType: "long", friendlyType: "DateTime" },
- { class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
- { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal", fields: [{name:"value", type:"long"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
- <#--
- { class: "Money", maxPrecisionDigits: 2, scale: 1, },
- -->
- ]
- },
- {
- major: "Fixed",
- width: 12,
- javaType: "DrillBuf",
- boxedType: "DrillBuf",
- minor: [
- { class: "IntervalDay", millisecondsOffset: 4, friendlyType: "Period", fields: [ {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
- ]
- },
- {
- major: "Fixed",
- width: 16,
- javaType: "DrillBuf"
- boxedType: "DrillBuf",
- minor: [
- { class: "Interval", daysOffset: 4, millisecondsOffset: 8, friendlyType: "Period", fields: [ {name: "months", type: "int"}, {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
- ]
- },
- {
- major: "Fixed",
- width: 12,
- javaType: "DrillBuf",
- boxedType: "DrillBuf",
- minor: [
- <#--
- { class: "TimeTZ" },
- { class: "Interval" }
- -->
- { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 16,
- javaType: "DrillBuf",
- boxedType: "DrillBuf",
-
- minor: [
- { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 24,
- javaType: "DrillBuf",
- boxedType: "DrillBuf",
- minor: [
- { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 20,
- javaType: "DrillBuf",
- boxedType: "DrillBuf",
- minor: [
- { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "VarLen",
- width: 4,
- javaType: "int",
- boxedType: "DrillBuf",
- fields: [{name: "start", type: "int"}, {name: "end", type: "int"}, {name: "buffer", type: "DrillBuf"}],
- minor: [
- { class: "VarBinary" , friendlyType: "byte[]" },
- { class: "VarChar" , friendlyType: "Text" },
- { class: "Var16Char" , friendlyType: "String" }
- ]
- },
- {
- major: "Bit",
- width: 1,
- javaType: "int",
- boxedType: "Integer",
- minor: [
- { class: "Bit" , friendlyType: "Boolean", fields: [{name: "value", type: "int"}] }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
deleted file mode 100644
index 2b7b305..0000000
--- a/exec/java-exec/src/main/codegen/templates/AbstractFieldReader.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/AbstractFieldReader.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-@SuppressWarnings("unused")
-abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
-
- AbstractFieldReader(){
- super();
- }
-
- /**
- * Returns true if the current value of the reader is not null
- * @return
- */
- public boolean isSet() {
- return true;
- }
-
- <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
- "Character", "DateTime", "Period", "Double", "Float",
- "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
- <#assign safeType=friendlyType />
- <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-
- public ${friendlyType} read${safeType}(int arrayIndex){
- fail("read${safeType}(int arrayIndex)");
- return null;
- }
-
- public ${friendlyType} read${safeType}(){
- fail("read${safeType}()");
- return null;
- }
-
- </#list>
-
- public void copyAsValue(MapWriter writer){
- fail("CopyAsValue MapWriter");
- }
- public void copyAsField(String name, MapWriter writer){
- fail("CopyAsField MapWriter");
- }
-
- public void copyAsField(String name, ListWriter writer){
- fail("CopyAsFieldList");
- }
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign boxedType = (minor.boxedType!type.boxedType) />
-
- public void read(${name}Holder holder){
- fail("${name}");
- }
-
- public void read(Nullable${name}Holder holder){
- fail("${name}");
- }
-
- public void read(int arrayIndex, ${name}Holder holder){
- fail("Repeated${name}");
- }
-
- public void read(int arrayIndex, Nullable${name}Holder holder){
- fail("Repeated${name}");
- }
-
- public void copyAsValue(${name}Writer writer){
- fail("CopyAsValue${name}");
- }
- public void copyAsField(String name, ${name}Writer writer){
- fail("CopyAsField${name}");
- }
- </#list></#list>
-
- public FieldReader reader(String name){
- fail("reader(String name)");
- return null;
- }
-
- public FieldReader reader(){
- fail("reader()");
- return null;
-
- }
-
- public int size(){
- fail("size()");
- return -1;
- }
-
- private void fail(String name){
- throw new IllegalArgumentException(String.format("You tried to read a [%s] type when you are using a field reader of type [%s].", name, this.getClass().getSimpleName()));
- }
-
-
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java
deleted file mode 100644
index 7ab5dce..0000000
--- a/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/AbstractFieldWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using freemarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter {
- AbstractFieldWriter(FieldWriter parent) {
- super(parent);
- }
-
- @Override
- public void start() {
- throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
- }
-
- @Override
- public void end() {
- throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
- }
-
- @Override
- public void startList() {
- throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
- }
-
- @Override
- public void endList() {
- throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
- }
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- @Override
- public void write(${name}Holder holder) {
- fail("${name}");
- }
-
- public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- fail("${name}");
- }
-
- </#list></#list>
-
- public void writeNull() {
- fail("${name}");
- }
-
- /**
- * This implementation returns {@code false}.
- * <p>
- * Must be overridden by map writers.
- * </p>
- */
- @Override
- public boolean isEmptyMap() {
- return false;
- }
-
- @Override
- public MapWriter map() {
- fail("Map");
- return null;
- }
-
- @Override
- public ListWriter list() {
- fail("List");
- return null;
- }
-
- @Override
- public MapWriter map(String name) {
- fail("Map");
- return null;
- }
-
- @Override
- public ListWriter list(String name) {
- fail("List");
- return null;
- }
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#if minor.class?starts_with("Decimal") >
- public ${capName}Writer ${lowerName}(String name, int scale, int precision) {
- fail("${capName}");
- return null;
- }
- </#if>
-
- @Override
- public ${capName}Writer ${lowerName}(String name) {
- fail("${capName}");
- return null;
- }
-
- @Override
- public ${capName}Writer ${lowerName}() {
- fail("${capName}");
- return null;
- }
-
- </#list></#list>
-
- public void copyReader(FieldReader reader) {
- fail("Copy FieldReader");
- }
-
- public void copyReaderToField(String name, FieldReader reader) {
- fail("Copy FieldReader to STring");
- }
-
- private void fail(String name) {
- throw new IllegalArgumentException(String.format("You tried to write a %s type when you are using a ValueWriter of type %s.", name, this.getClass().getSimpleName()));
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java
deleted file mode 100644
index f2f8193..0000000
--- a/exec/java-exec/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/AbstractPromotableFieldWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * A FieldWriter which delegates calls to another FieldWriter. The delegate FieldWriter can be promoted to a new type
- * when necessary. Classes that extend this class are responsible for handling promotion.
- *
- * This class is generated using freemarker and the ${.template_name} template.
- *
- */
-@SuppressWarnings("unused")
-abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
- AbstractPromotableFieldWriter(FieldWriter parent) {
- super(parent);
- }
-
- /**
- * Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type
- * @param type
- * @return
- */
- abstract protected FieldWriter getWriter(MinorType type);
-
- /**
- * Return the current FieldWriter
- * @return
- */
- abstract protected FieldWriter getWriter();
-
- @Override
- public void start() {
- getWriter(MinorType.MAP).start();
- }
-
- @Override
- public void end() {
- getWriter(MinorType.MAP).end();
- }
-
- @Override
- public void startList() {
- getWriter(MinorType.LIST).startList();
- }
-
- @Override
- public void endList() {
- getWriter(MinorType.LIST).endList();
- }
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#if !minor.class?starts_with("Decimal") >
- @Override
- public void write(${name}Holder holder) {
- getWriter(MinorType.${name?upper_case}).write(holder);
- }
-
- public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
- }
-
- </#if>
- </#list></#list>
-
- public void writeNull() {
- }
-
- @Override
- public MapWriter map() {
- return getWriter(MinorType.LIST).map();
- }
-
- @Override
- public ListWriter list() {
- return getWriter(MinorType.LIST).list();
- }
-
- @Override
- public MapWriter map(String name) {
- return getWriter(MinorType.MAP).map(name);
- }
-
- @Override
- public ListWriter list(String name) {
- return getWriter(MinorType.MAP).list(name);
- }
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#if !minor.class?starts_with("Decimal") >
-
- @Override
- public ${capName}Writer ${lowerName}(String name) {
- return getWriter(MinorType.MAP).${lowerName}(name);
- }
-
- @Override
- public ${capName}Writer ${lowerName}() {
- return getWriter(MinorType.LIST).${lowerName}();
- }
-
- </#if>
- </#list></#list>
-
- public void copyReader(FieldReader reader) {
- getWriter().copyReader(reader);
- }
-
- public void copyReaderToField(String name, FieldReader reader) {
- getWriter().copyReaderToField(name, reader);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/BaseReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/BaseReader.java b/exec/java-exec/src/main/codegen/templates/BaseReader.java
deleted file mode 100644
index 78f32f4..0000000
--- a/exec/java-exec/src/main/codegen/templates/BaseReader.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/reader/BaseReader.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.reader;
-
-<#include "/@includes/vv_imports.ftl" />
-
-
-
-@SuppressWarnings("unused")
-public interface BaseReader extends Positionable{
- MajorType getType();
- MaterializedField getField();
- void reset();
- void read(UnionHolder holder);
- void read(int index, UnionHolder holder);
- void copyAsValue(UnionWriter writer);
- boolean isSet();
-
- public interface MapReader extends BaseReader, Iterable<String>{
- FieldReader reader(String name);
- }
-
- public interface RepeatedMapReader extends MapReader{
- boolean next();
- int size();
- void copyAsValue(MapWriter writer);
- }
-
- public interface ListReader extends BaseReader{
- FieldReader reader();
- }
-
- public interface RepeatedListReader extends ListReader{
- boolean next();
- int size();
- void copyAsValue(ListWriter writer);
- }
-
- public interface ScalarReader extends
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, </#list></#list>
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> Repeated${name}Reader, </#list></#list>
- BaseReader {}
-
- interface ComplexReader{
- MapReader rootAsMap();
- ListReader rootAsList();
- boolean rootIsMap();
- boolean ok();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/BaseWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/BaseWriter.java b/exec/java-exec/src/main/codegen/templates/BaseWriter.java
deleted file mode 100644
index 8a9ea56..0000000
--- a/exec/java-exec/src/main/codegen/templates/BaseWriter.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/writer/BaseWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.writer;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * File generated from ${.template_name} using FreeMarker.
- */
-@SuppressWarnings("unused")
- public interface BaseWriter extends AutoCloseable, Positionable {
- FieldWriter getParent();
- int getValueCapacity();
-
- public interface MapWriter extends BaseWriter {
-
- MaterializedField getField();
-
- /**
- * Whether this writer is a map writer and is empty (has no children).
- *
- * <p>
- * Intended only for use in determining whether to add dummy vector to
- * avoid empty (zero-column) schema, as in JsonReader.
- * </p>
- *
- */
- boolean isEmptyMap();
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#if minor.class?starts_with("Decimal") >
- ${capName}Writer ${lowerName}(String name, int scale, int precision);
- </#if>
- ${capName}Writer ${lowerName}(String name);
- </#list></#list>
-
- void copyReaderToField(String name, FieldReader reader);
- MapWriter map(String name);
- ListWriter list(String name);
- void start();
- void end();
- }
-
- public interface ListWriter extends BaseWriter {
- void startList();
- void endList();
- MapWriter map();
- ListWriter list();
- void copyReader(FieldReader reader);
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- ${capName}Writer ${lowerName}();
- </#list></#list>
- }
-
- public interface ScalarWriter extends
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Writer, </#list></#list> BaseWriter {}
-
- public interface ComplexWriter {
- void allocate();
- void clear();
- void copyReader(FieldReader reader);
- MapWriter rootAsMap();
- ListWriter rootAsList();
-
- void setPosition(int index);
- void setValueCount(int count);
- void reset();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/ComplexCopier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ComplexCopier.java b/exec/java-exec/src/main/codegen/templates/ComplexCopier.java
deleted file mode 100644
index b8b7616..0000000
--- a/exec/java-exec/src/main/codegen/templates/ComplexCopier.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/ComplexCopier.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using freemarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-public class ComplexCopier {
-
- /**
- * Do a deep copy of the value in input into output
- * @param in
- * @param out
- */
- public static void copy(FieldReader input, FieldWriter output) {
- writeValue(input, output);
- }
-
- private static void writeValue(FieldReader reader, FieldWriter writer) {
- final DataMode m = reader.getType().getMode();
- final MinorType mt = reader.getType().getMinorType();
-
- switch(m){
- case OPTIONAL:
- case REQUIRED:
-
-
- switch (mt) {
-
- case LIST:
- writer.startList();
- while (reader.next()) {
- writeValue(reader.reader(), getListWriterForReader(reader.reader(), writer));
- }
- writer.endList();
- break;
- case MAP:
- writer.start();
- if (reader.isSet()) {
- for(String name : reader){
- FieldReader childReader = reader.reader(name);
- if(childReader.isSet()){
- writeValue(childReader, getMapWriterForReader(childReader, writer, name));
- }
- }
- }
- writer.end();
- break;
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
-
- case ${name?upper_case}:
- if (reader.isSet()) {
- Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
- reader.read(${uncappedName}Holder);
- writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list>);
- }
- break;
-
- </#if>
- </#list></#list>
- }
- break;
- }
- }
-
- private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
- switch (reader.getType().getMinorType()) {
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
- case ${name?upper_case}:
- return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name);
- </#if>
- </#list></#list>
- case MAP:
- return (FieldWriter) writer.map(name);
- case LIST:
- return (FieldWriter) writer.list(name);
- default:
- throw new UnsupportedOperationException(reader.getType().toString());
- }
- }
-
- private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) {
- switch (reader.getType().getMinorType()) {
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
- case ${name?upper_case}:
- return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>();
- </#if>
- </#list></#list>
- case MAP:
- return (FieldWriter) writer.map();
- case LIST:
- return (FieldWriter) writer.list();
- default:
- throw new UnsupportedOperationException(reader.getType().toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/ComplexReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ComplexReaders.java b/exec/java-exec/src/main/codegen/templates/ComplexReaders.java
deleted file mode 100644
index 607b71d..0000000
--- a/exec/java-exec/src/main/codegen/templates/ComplexReaders.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.Override;
-import java.util.List;
-
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.complex.IndexHolder;
-import org.apache.drill.exec.vector.complex.writer.IntervalWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-<#list ["", "Repeated"] as mode>
-<#assign lowerName = minor.class?uncap_first />
-<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-<#assign name = mode + minor.class?cap_first />
-<#assign javaType = (minor.javaType!type.javaType) />
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-<#assign safeType=friendlyType />
-<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-
-<#assign hasFriendly = minor.friendlyType!"no" == "no" />
-
-<#list ["", "Nullable"] as nullMode>
-<#if (mode == "Repeated" && nullMode == "") || mode == "" >
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${nullMode}${name}ReaderImpl.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-@SuppressWarnings("unused")
-public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
-
- private final ${nullMode}${name}Vector vector;
-
- public ${nullMode}${name}ReaderImpl(${nullMode}${name}Vector vector){
- super();
- this.vector = vector;
- }
-
- public MajorType getType(){
- return vector.getField().getType();
- }
-
- public MaterializedField getField(){
- return vector.getField();
- }
-
- public boolean isSet(){
- <#if nullMode == "Nullable">
- return !vector.getAccessor().isNull(idx());
- <#else>
- return true;
- </#if>
- }
-
-
-
-
- <#if mode == "Repeated">
-
- public void copyAsValue(${minor.class?cap_first}Writer writer){
- Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer;
- impl.vector.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public void copyAsField(String name, MapWriter writer){
- Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer.list(name).${lowerName}();
- impl.vector.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public int size(){
- return vector.getAccessor().getInnerValueCountAt(idx());
- }
-
- public void read(int arrayIndex, ${minor.class?cap_first}Holder h){
- vector.getAccessor().get(idx(), arrayIndex, h);
- }
- public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h){
- vector.getAccessor().get(idx(), arrayIndex, h);
- }
-
- public ${friendlyType} read${safeType}(int arrayIndex){
- return vector.getAccessor().getSingleObject(idx(), arrayIndex);
- }
-
-
- public List<Object> readObject(){
- return (List<Object>) (Object) vector.getAccessor().getObject(idx());
- }
-
- <#else>
-
- public void copyAsValue(${minor.class?cap_first}Writer writer){
- ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer;
- impl.vector.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public void copyAsField(String name, MapWriter writer){
- ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
- impl.vector.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- <#if nullMode != "Nullable">
- public void read(${minor.class?cap_first}Holder h){
- vector.getAccessor().get(idx(), h);
- }
- </#if>
-
- public void read(Nullable${minor.class?cap_first}Holder h){
- vector.getAccessor().get(idx(), h);
- }
-
- public ${friendlyType} read${safeType}(){
- return vector.getAccessor().getObject(idx());
- }
-
- public void copyValue(FieldWriter w){
-
- }
-
- public Object readObject(){
- return vector.getAccessor().getObject(idx());
- }
-
-
- </#if>
-}
-</#if>
-</#list>
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/reader/${name}Reader.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.reader;
-
-<#include "/@includes/vv_imports.ftl" />
-@SuppressWarnings("unused")
-public interface ${name}Reader extends BaseReader{
-
- <#if mode == "Repeated">
- public int size();
- public void read(int arrayIndex, ${minor.class?cap_first}Holder h);
- public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h);
- public Object readObject(int arrayIndex);
- public ${friendlyType} read${safeType}(int arrayIndex);
- <#else>
- public void read(${minor.class?cap_first}Holder h);
- public void read(Nullable${minor.class?cap_first}Holder h);
- public Object readObject();
- public ${friendlyType} read${safeType}();
- </#if>
- public boolean isSet();
- public void copyAsValue(${minor.class}Writer writer);
- public void copyAsField(String name, ${minor.class}Writer writer);
-
-}
-
-
-
-</#list>
-</#list>
-</#list>
-
-
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/ComplexWriters.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ComplexWriters.java b/exec/java-exec/src/main/codegen/templates/ComplexWriters.java
deleted file mode 100644
index 71a7f9a..0000000
--- a/exec/java-exec/src/main/codegen/templates/ComplexWriters.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.Override;
-import java.util.Vector;
-
-import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.ValueVector.Mutator;
-import org.apache.drill.exec.vector.complex.IndexHolder;
-import org.apache.drill.exec.vector.complex.reader.FieldReader;
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-<#list ["", "Nullable", "Repeated"] as mode>
-<#assign name = mode + minor.class?cap_first />
-<#assign eName = name />
-<#assign javaType = (minor.javaType!type.javaType) />
-<#assign fields = minor.fields!type.fields />
-
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${eName}WriterImpl.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.impl;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using FreeMarker on the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-public class ${eName}WriterImpl extends AbstractFieldWriter {
-
- private final ${name}Vector.Mutator mutator;
- final ${name}Vector vector;
-
- public ${eName}WriterImpl(${name}Vector vector, AbstractFieldWriter parent) {
- super(parent);
- this.mutator = vector.getMutator();
- this.vector = vector;
- }
-
- @Override
- public MaterializedField getField() {
- return vector.getField();
- }
-
- @Override
- public int getValueCapacity() {
- return vector.getValueCapacity();
- }
-
- @Override
- public void allocate() {
- vector.allocateNew();
- }
-
- @Override
- public void close() {
- vector.close();
- }
-
- @Override
- public void clear() {
- vector.clear();
- }
-
- @Override
- protected int idx() {
- return super.idx();
- }
-
- <#if mode == "Repeated">
-
- public void write(${minor.class?cap_first}Holder h) {
- mutator.addSafe(idx(), h);
- vector.getMutator().setValueCount(idx()+1);
- }
-
- public void write(Nullable${minor.class?cap_first}Holder h) {
- mutator.addSafe(idx(), h);
- vector.getMutator().setValueCount(idx()+1);
- }
-
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
- public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
- vector.getMutator().setValueCount(idx()+1);
- }
- </#if>
-
- public void setPosition(int idx) {
- super.setPosition(idx);
- mutator.startNewValue(idx);
- }
-
-
- <#else>
-
- public void write(${minor.class}Holder h) {
- mutator.setSafe(idx(), h);
- vector.getMutator().setValueCount(idx()+1);
- }
-
- public void write(Nullable${minor.class}Holder h) {
- mutator.setSafe(idx(), h);
- vector.getMutator().setValueCount(idx()+1);
- }
-
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
- public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- mutator.setSafe(idx(), <#if mode == "Nullable">1, </#if><#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
- vector.getMutator().setValueCount(idx()+1);
- }
-
- <#if mode == "Nullable">
-
- public void writeNull() {
- mutator.setNull(idx());
- vector.getMutator().setValueCount(idx()+1);
- }
- </#if>
- </#if>
- </#if>
-}
-
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/writer/${eName}Writer.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector.complex.writer;
-
-<#include "/@includes/vv_imports.ftl" />
-@SuppressWarnings("unused")
-public interface ${eName}Writer extends BaseWriter {
- public void write(${minor.class}Holder h);
-
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
- public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
- </#if>
-}
-
-</#list>
-</#list>
-</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
deleted file mode 100644
index d9d5fae..0000000
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ /dev/null
@@ -1,815 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.Override;
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-
-<#if type.major == "Fixed">
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.vector;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/**
- * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed
- * by position, starting from the logical start of the vector. Values should be pushed onto the
- * vector sequentially, but may be randomly accessed.
- * The width of each element is ${type.width} byte(s)
- * The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
-
- private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
- private final Accessor accessor = new Accessor();
- private final Mutator mutator = new Mutator();
-
- private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
- private int allocationMonitor = 0;
-
- public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- @Override
- public FieldReader getReader(){
- return reader;
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
- return valueCount * ${type.width};
- }
-
- @Override
- public int getValueCapacity(){
- return (int) (data.capacity() *1.0 / ${type.width});
- }
-
- @Override
- public Accessor getAccessor(){
- return accessor;
- }
-
- @Override
- public Mutator getMutator(){
- return mutator;
- }
-
- @Override
- public void setInitialCapacity(final int valueCount) {
- final long size = 1L * valueCount * ${type.width};
- if (size > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
- }
- allocationSizeInBytes = (int)size;
- }
-
- @Override
- public void allocateNew() {
- if(!allocateNewSafe()){
- throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
- }
- }
-
- @Override
- public boolean allocateNewSafe() {
- long curAllocationSize = allocationSizeInBytes;
- if (allocationMonitor > 10) {
- curAllocationSize = Math.max(8, curAllocationSize / 2);
- allocationMonitor = 0;
- } else if (allocationMonitor < -2) {
- curAllocationSize = allocationSizeInBytes * 2L;
- allocationMonitor = 0;
- }
-
- try{
- allocateBytes(curAllocationSize);
- } catch (DrillRuntimeException ex) {
- return false;
- }
- return true;
- }
-
- /**
- * Allocate a new buffer that supports setting at least the provided number of values. May actually be sized bigger
- * depending on underlying buffer rounding size. Must be called prior to using the ValueVector.
- *
- * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
- *
- * @param valueCount
- * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer
- */
- @Override
- public void allocateNew(final int valueCount) {
- allocateBytes(valueCount * ${type.width});
- }
-
- @Override
- public void reset() {
- allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
- allocationMonitor = 0;
- zeroVector();
- super.reset();
- }
-
- private void allocateBytes(final long size) {
- if (size > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
- }
-
- final int curSize = (int)size;
- clear();
- data = allocator.buffer(curSize);
- data.readerIndex(0);
- allocationSizeInBytes = curSize;
- }
-
-/**
- * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
- *
- * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer
- */
- public void reAlloc() {
- final long newAllocationSize = allocationSizeInBytes * 2L;
- if (newAllocationSize > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
- }
-
- logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize);
- final DrillBuf newBuf = allocator.buffer((int)newAllocationSize);
- newBuf.setBytes(0, data, 0, data.capacity());
- final int halfNewCapacity = newBuf.capacity() / 2;
- newBuf.setZero(halfNewCapacity, halfNewCapacity);
- newBuf.writerIndex(data.writerIndex());
- data.release(1);
- data = newBuf;
- allocationSizeInBytes = (int)newAllocationSize;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void zeroVector() {
- data.setZero(0, data.capacity());
- }
-
- @Override
- public void load(SerializedField metadata, DrillBuf buffer) {
- Preconditions.checkArgument(this.field.matches(metadata), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
- final int actualLength = metadata.getBufferLength();
- final int valueCount = metadata.getValueCount();
- final int expectedLength = valueCount * ${type.width};
- assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength);
-
- clear();
- if (data != null) {
- data.release(1);
- }
- data = buffer.slice(0, actualLength);
- data.retain(1);
- data.writerIndex(actualLength);
- }
-
- public TransferPair getTransferPair(){
- return new TransferImpl(getField());
- }
-
- @Override
- public TransferPair getTransferPair(FieldReference ref){
- return new TransferImpl(getField().withPath(ref));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new TransferImpl((${minor.class}Vector) to);
- }
-
- public void transferTo(${minor.class}Vector target){
- target.clear();
- target.data = data;
- target.data.retain(1);
- target.data.writerIndex(data.writerIndex());
- clear();
- }
-
- public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
- final int startPoint = startIndex * ${type.width};
- final int sliceLength = length * ${type.width};
- target.clear();
- target.data = data.slice(startPoint, sliceLength);
- target.data.retain(1);
- target.data.writerIndex(sliceLength);
- }
-
- private class TransferImpl implements TransferPair{
- private ${minor.class}Vector to;
-
- public TransferImpl(MaterializedField field){
- to = new ${minor.class}Vector(field, allocator);
- }
-
- public TransferImpl(${minor.class}Vector to) {
- this.to = to;
- }
-
- @Override
- public ${minor.class}Vector getTo(){
- return to;
- }
-
- @Override
- public void transfer(){
- transferTo(to);
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- splitAndTransferTo(startIndex, length, to);
- }
-
- @Override
- public void copyValueSafe(int fromIndex, int toIndex) {
- to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
- }
- }
-
- public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
- <#if (type.width > 8)>
- from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
- <#else> <#-- type.width <= 8 -->
- data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width},
- from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width})
- );
- </#if> <#-- type.width -->
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
- while(thisIndex >= getValueCapacity()) {
- reAlloc();
- }
- copyFrom(fromIndex, thisIndex, from);
- }
-
- public void decrementAllocationMonitor() {
- if (allocationMonitor > 0) {
- allocationMonitor = 0;
- }
- --allocationMonitor;
- }
-
- private void incrementAllocationMonitor() {
- ++allocationMonitor;
- }
-
- public final class Accessor extends BaseDataValueVector.BaseAccessor {
- @Override
- public int getValueCount() {
- return data.writerIndex() / ${type.width};
- }
-
- @Override
- public boolean isNull(int index){
- return false;
- }
-
- <#if (type.width > 8)>
-
- public ${minor.javaType!type.javaType} get(int index) {
- return data.slice(index * ${type.width}, ${type.width});
- }
-
- <#if (minor.class == "Interval")>
- public void get(int index, ${minor.class}Holder holder){
-
- final int offsetIndex = index * ${type.width};
- holder.months = data.getInt(offsetIndex);
- holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
- holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- }
-
- public void get(int index, Nullable${minor.class}Holder holder){
- final int offsetIndex = index * ${type.width};
- holder.isSet = 1;
- holder.months = data.getInt(offsetIndex);
- holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
- holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- }
-
- @Override
- public ${friendlyType} getObject(int index) {
- final int offsetIndex = index * ${type.width};
- final int months = data.getInt(offsetIndex);
- final int days = data.getInt(offsetIndex + ${minor.daysOffset});
- final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- final Period p = new Period();
- return p.plusMonths(months).plusDays(days).plusMillis(millis);
- }
-
- public StringBuilder getAsStringBuilder(int index) {
-
- final int offsetIndex = index * ${type.width};
-
- int months = data.getInt(offsetIndex);
- final int days = data.getInt(offsetIndex + ${minor.daysOffset});
- int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-
- final int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
- months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
-
- final int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
- millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
-
- final int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
- millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
-
- final long seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
- millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
-
- final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
- final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
- final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
-
-
- return(new StringBuilder().
- append(years).append(yearString).
- append(months).append(monthString).
- append(days).append(dayString).
- append(hours).append(":").
- append(minutes).append(":").
- append(seconds).append(".").
- append(millis));
- }
-
- <#elseif (minor.class == "IntervalDay")>
- public void get(int index, ${minor.class}Holder holder){
-
- final int offsetIndex = index * ${type.width};
- holder.days = data.getInt(offsetIndex);
- holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- }
-
- public void get(int index, Nullable${minor.class}Holder holder){
- final int offsetIndex = index * ${type.width};
- holder.isSet = 1;
- holder.days = data.getInt(offsetIndex);
- holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- }
-
- @Override
- public ${friendlyType} getObject(int index) {
- final int offsetIndex = index * ${type.width};
- final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- final int days = data.getInt(offsetIndex);
- final Period p = new Period();
- return p.plusDays(days).plusMillis(millis);
- }
-
-
- public StringBuilder getAsStringBuilder(int index) {
- final int offsetIndex = index * ${type.width};
-
- int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- final int days = data.getInt(offsetIndex);
-
- final int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
- millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
-
- final int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
- millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
-
- final int seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
- millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
-
- final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
-
- return(new StringBuilder().
- append(days).append(dayString).
- append(hours).append(":").
- append(minutes).append(":").
- append(seconds).append(".").
- append(millis));
- }
-
- <#elseif (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
-
- public void get(int index, ${minor.class}Holder holder) {
- holder.start = index * ${type.width};
- holder.buffer = data;
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
- }
-
- public void get(int index, Nullable${minor.class}Holder holder) {
- holder.isSet = 1;
- holder.start = index * ${type.width};
- holder.buffer = data;
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
- }
-
- @Override
- public ${friendlyType} getObject(int index) {
- <#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")>
- // Get the BigDecimal object
- return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
- <#else>
- return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
- </#if>
- }
-
- <#else>
- public void get(int index, ${minor.class}Holder holder){
- holder.buffer = data;
- holder.start = index * ${type.width};
- }
-
- public void get(int index, Nullable${minor.class}Holder holder){
- holder.isSet = 1;
- holder.buffer = data;
- holder.start = index * ${type.width};
- }
-
- @Override
- public ${friendlyType} getObject(int index) {
- return data.slice(index * ${type.width}, ${type.width})
- }
-
- </#if>
- <#else> <#-- type.width <= 8 -->
-
- public ${minor.javaType!type.javaType} get(int index) {
- return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
- }
-
- <#if type.width == 4>
- public long getTwoAsLong(int index) {
- return data.getLong(index * ${type.width});
- }
-
- </#if>
-
- <#if minor.class == "Date">
- @Override
- public ${friendlyType} getObject(int index) {
- org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
- date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
- return date;
- }
-
- <#elseif minor.class == "TimeStamp">
- @Override
- public ${friendlyType} getObject(int index) {
- org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
- date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
- return date;
- }
-
- <#elseif minor.class == "IntervalYear">
- @Override
- public ${friendlyType} getObject(int index) {
-
- final int value = get(index);
-
- final int years = (value / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
- final int months = (value % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
- final Period p = new Period();
- return p.plusYears(years).plusMonths(months);
- }
-
- public StringBuilder getAsStringBuilder(int index) {
-
- int months = data.getInt(index);
-
- final int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
- months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
-
- final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
- final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
-
- return(new StringBuilder().
- append(years).append(yearString).
- append(months).append(monthString));
- }
-
- <#elseif minor.class == "Time">
- @Override
- public DateTime getObject(int index) {
-
- org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
- time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
- return time;
- }
-
- <#elseif minor.class == "Decimal9" || minor.class == "Decimal18">
- @Override
- public ${friendlyType} getObject(int index) {
-
- final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value());
- return new BigDecimal(value, getField().getScale());
- }
-
- <#else>
- @Override
- public ${friendlyType} getObject(int index) {
- return get(index);
- }
- public ${minor.javaType!type.javaType} getPrimitiveObject(int index) {
- return get(index);
- }
- </#if>
-
- public void get(int index, ${minor.class}Holder holder){
- <#if minor.class.startsWith("Decimal")>
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
- </#if>
-
- holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
- }
-
- public void get(int index, Nullable${minor.class}Holder holder){
- holder.isSet = 1;
- holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
- }
-
-
- </#if> <#-- type.width -->
- }
-
- /**
- * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the
- * vector are accessed by position from the logical start of the vector. Values should be pushed
- * onto the vector sequentially, but may be randomly accessed.
- * The width of each element is ${type.width} byte(s)
- * The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public final class Mutator extends BaseDataValueVector.BaseMutator {
-
- private Mutator(){};
- /**
- * Set the element at the given index to the given value. Note that widths smaller than
- * 32 bits are handled by the DrillBuf interface.
- *
- * @param index position of the bit to set
- * @param value value to set
- */
- <#if (type.width > 8)>
- public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
- data.setBytes(index * ${type.width}, value, 0, ${type.width});
- }
-
- public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- data.setBytes(index * ${type.width}, value, 0, ${type.width});
- }
-
- <#if (minor.class == "Interval")>
- public void set(int index, int months, int days, int milliseconds){
- final int offsetIndex = index * ${type.width};
- data.setInt(offsetIndex, months);
- data.setInt((offsetIndex + ${minor.daysOffset}), days);
- data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
- }
-
- protected void set(int index, ${minor.class}Holder holder){
- set(index, holder.months, holder.days, holder.milliseconds);
- }
-
- protected void set(int index, Nullable${minor.class}Holder holder){
- set(index, holder.months, holder.days, holder.milliseconds);
- }
-
- public void setSafe(int index, int months, int days, int milliseconds){
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, months, days, milliseconds);
- }
-
- public void setSafe(int index, Nullable${minor.class}Holder holder){
- setSafe(index, holder.months, holder.days, holder.milliseconds);
- }
-
- public void setSafe(int index, ${minor.class}Holder holder){
- setSafe(index, holder.months, holder.days, holder.milliseconds);
- }
-
- <#elseif (minor.class == "IntervalDay")>
- public void set(int index, int days, int milliseconds){
- final int offsetIndex = index * ${type.width};
- data.setInt(offsetIndex, days);
- data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
- }
-
- protected void set(int index, ${minor.class}Holder holder){
- set(index, holder.days, holder.milliseconds);
- }
- protected void set(int index, Nullable${minor.class}Holder holder){
- set(index, holder.days, holder.milliseconds);
- }
-
- public void setSafe(int index, int days, int milliseconds){
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, days, milliseconds);
- }
-
- public void setSafe(int index, ${minor.class}Holder holder){
- setSafe(index, holder.days, holder.milliseconds);
- }
-
- public void setSafe(int index, Nullable${minor.class}Holder holder){
- setSafe(index, holder.days, holder.milliseconds);
- }
-
- <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
-
- public void set(int index, ${minor.class}Holder holder){
- set(index, holder.start, holder.buffer);
- }
-
- void set(int index, Nullable${minor.class}Holder holder){
- set(index, holder.start, holder.buffer);
- }
-
- public void setSafe(int index, Nullable${minor.class}Holder holder){
- setSafe(index, holder.start, holder.buffer);
- }
- public void setSafe(int index, ${minor.class}Holder holder){
- setSafe(index, holder.start, holder.buffer);
- }
-
- public void setSafe(int index, int start, DrillBuf buffer){
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, start, buffer);
- }
-
- public void set(int index, int start, DrillBuf buffer){
- data.setBytes(index * ${type.width}, buffer, start, ${type.width});
- }
-
- <#else>
-
- protected void set(int index, ${minor.class}Holder holder){
- set(index, holder.start, holder.buffer);
- }
-
- public void set(int index, Nullable${minor.class}Holder holder){
- set(index, holder.start, holder.buffer);
- }
-
- public void set(int index, int start, DrillBuf buffer){
- data.setBytes(index * ${type.width}, buffer, start, ${type.width});
- }
-
- public void setSafe(int index, ${minor.class}Holder holder){
- setSafe(index, holder.start, holder.buffer);
- }
- public void setSafe(int index, Nullable${minor.class}Holder holder){
- setSafe(index, holder.start, holder.buffer);
- }
-
- public void setSafe(int index, int start, DrillBuf buffer){
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, holder);
- }
-
- public void set(int index, Nullable${minor.class}Holder holder){
- data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
- }
- </#if>
-
- @Override
- public void generateTestData(int count) {
- setValueCount(count);
- boolean even = true;
- final int valueCount = getAccessor().getValueCount();
- for(int i = 0; i < valueCount; i++, even = !even) {
- final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE;
- for(int w = 0; w < ${type.width}; w++){
- data.setByte(i + w, b);
- }
- }
- }
-
- <#else> <#-- type.width <= 8 -->
- public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
- data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
- }
-
- public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, value);
- }
-
- protected void set(int index, ${minor.class}Holder holder){
- data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
- }
-
- public void setSafe(int index, ${minor.class}Holder holder){
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, holder);
- }
-
- protected void set(int index, Nullable${minor.class}Holder holder){
- data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
- }
-
- public void setSafe(int index, Nullable${minor.class}Holder holder){
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, holder);
- }
-
- @Override
- public void generateTestData(int size) {
- setValueCount(size);
- boolean even = true;
- final int valueCount = getAccessor().getValueCount();
- for(int i = 0; i < valueCount; i++, even = !even) {
- if(even){
- set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE);
- }else{
- set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE);
- }
- }
- }
-
- public void generateTestDataAlt(int size) {
- setValueCount(size);
- boolean even = true;
- final int valueCount = getAccessor().getValueCount();
- for(int i = 0; i < valueCount; i++, even = !even) {
- if(even){
- set(i, (${(minor.javaType!type.javaType)}) 1);
- }else{
- set(i, (${(minor.javaType!type.javaType)}) 0);
- }
- }
- }
-
- </#if> <#-- type.width -->
-
- @Override
- public void setValueCount(int valueCount) {
- final int currentValueCapacity = getValueCapacity();
- final int idx = (${type.width} * valueCount);
- while(valueCount > getValueCapacity()) {
- reAlloc();
- }
- if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
- incrementAllocationMonitor();
- } else if (allocationMonitor > 0) {
- allocationMonitor = 0;
- }
- VectorTrimmer.trim(data, idx);
- data.writerIndex(valueCount * ${type.width});
- }
- }
-}
-
-</#if> <#-- type.major -->
-</#list>
-</#list>
[06/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
new file mode 100644
index 0000000..fe91fe6
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -0,0 +1,636 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
+
+import java.lang.Override;
+import java.lang.UnsupportedOperationException;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign className = "Nullable${minor.class}Vector" />
+<#assign valuesName = "${minor.class}Vector" />
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${className}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * Nullable${minor.class} implements a vector of values which could be null. Elements in the vector
+ * are first checked against a fixed length vector of boolean values. Then the element is retrieved
+ * from the base class (if not null).
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+@SuppressWarnings("unused")
+public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector{
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
+
+ private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
+
+ private final MaterializedField bitsField = MaterializedField.create("$bits$", Types.required(MinorType.UINT1));
+ private final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
+ private final ${valuesName} values = new ${minor.class}Vector(field, allocator);
+
+ private final Mutator mutator = new Mutator();
+ private final Accessor accessor = new Accessor();
+
+ public ${className}(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ @Override
+ public FieldReader getReader(){
+ return reader;
+ }
+
+ @Override
+ public int getValueCapacity(){
+ return Math.min(bits.getValueCapacity(), values.getValueCapacity());
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ final DrillBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), DrillBuf.class);
+ if (clear) {
+ for (final DrillBuf buffer:buffers) {
+ buffer.retain(1);
+ }
+ clear();
+ }
+ return buffers;
+ }
+
+ @Override
+ public void close() {
+ bits.close();
+ values.close();
+ super.close();
+ }
+
+ @Override
+ public void clear() {
+ bits.clear();
+ values.clear();
+ super.clear();
+ }
+
+ @Override
+ public int getBufferSize(){
+ return values.getBufferSize() + bits.getBufferSize();
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ return values.getBufferSizeFor(valueCount)
+ + bits.getBufferSizeFor(valueCount);
+ }
+
+ @Override
+ public DrillBuf getBuffer() {
+ return values.getBuffer();
+ }
+
+ @Override
+ public ${valuesName} getValuesVector() {
+ return values;
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ bits.setInitialCapacity(numRecords);
+ values.setInitialCapacity(numRecords);
+ }
+
+ @Override
+ public SerializedField.Builder getMetadataBuilder() {
+ return super.getMetadataBuilder()
+ .addChild(bits.getMetadata())
+ .addChild(values.getMetadata());
+ }
+
+ @Override
+ public void allocateNew() {
+ if(!allocateNewSafe()){
+ throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
+ }
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* Boolean to keep track if all the memory allocations were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ success = values.allocateNewSafe() && bits.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ bits.zeroVector();
+ mutator.reset();
+ accessor.reset();
+ return success;
+ }
+
+ <#if type.major == "VarLen">
+ @Override
+ public void allocateNew(int totalBytes, int valueCount) {
+ try {
+ values.allocateNew(totalBytes, valueCount);
+ bits.allocateNew(valueCount);
+ } catch(DrillRuntimeException e) {
+ clear();
+ throw e;
+ }
+ bits.zeroVector();
+ mutator.reset();
+ accessor.reset();
+ }
+
+ public void reset() {
+ bits.zeroVector();
+ mutator.reset();
+ accessor.reset();
+ super.reset();
+ }
+
+ @Override
+ public int getByteCapacity(){
+ return values.getByteCapacity();
+ }
+
+ @Override
+ public int getCurrentSizeInBytes(){
+ return values.getCurrentSizeInBytes();
+ }
+
+ <#else>
+ @Override
+ public void allocateNew(int valueCount) {
+ try {
+ values.allocateNew(valueCount);
+ bits.allocateNew(valueCount+1);
+ } catch(OutOfMemoryRuntimeException e) {
+ clear();
+ throw e;
+ }
+ bits.zeroVector();
+ mutator.reset();
+ accessor.reset();
+ }
+
+ @Override
+ public void reset() {
+ bits.zeroVector();
+ mutator.reset();
+ accessor.reset();
+ super.reset();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void zeroVector() {
+ bits.zeroVector();
+ values.zeroVector();
+ }
+ </#if>
+
+
+ @Override
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ clear();
+ // the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
+ final SerializedField bitsField = metadata.getChild(0);
+ bits.load(bitsField, buffer);
+
+ final int capacity = buffer.capacity();
+ final int bitsLength = bitsField.getBufferLength();
+ final SerializedField valuesField = metadata.getChild(1);
+ values.load(valuesField, buffer.slice(bitsLength, capacity - bitsLength));
+ }
+
+ @Override
+ public TransferPair getTransferPair(){
+ return new TransferImpl(getField());
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref){
+ return new TransferImpl(getField().withPath(ref));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new TransferImpl((Nullable${minor.class}Vector) to);
+ }
+
+ public void transferTo(Nullable${minor.class}Vector target){
+ bits.transferTo(target.bits);
+ values.transferTo(target.values);
+ <#if type.major == "VarLen">
+ target.mutator.lastSet = mutator.lastSet;
+ </#if>
+ clear();
+ }
+
+ public void splitAndTransferTo(int startIndex, int length, Nullable${minor.class}Vector target) {
+ bits.splitAndTransferTo(startIndex, length, target.bits);
+ values.splitAndTransferTo(startIndex, length, target.values);
+ <#if type.major == "VarLen">
+ target.mutator.lastSet = length - 1;
+ </#if>
+ }
+
+ private class TransferImpl implements TransferPair {
+ Nullable${minor.class}Vector to;
+
+ public TransferImpl(MaterializedField field){
+ to = new Nullable${minor.class}Vector(field, allocator);
+ }
+
+ public TransferImpl(Nullable${minor.class}Vector to){
+ this.to = to;
+ }
+
+ @Override
+ public Nullable${minor.class}Vector getTo(){
+ return to;
+ }
+
+ @Override
+ public void transfer(){
+ transferTo(to);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ splitAndTransferTo(startIndex, length, to);
+ }
+
+ @Override
+ public void copyValueSafe(int fromIndex, int toIndex) {
+ to.copyFromSafe(fromIndex, toIndex, Nullable${minor.class}Vector.this);
+ }
+ }
+
+ @Override
+ public Accessor getAccessor(){
+ return accessor;
+ }
+
+ @Override
+ public Mutator getMutator(){
+ return mutator;
+ }
+
+ public ${minor.class}Vector convertToRequiredVector(){
+ ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
+ if (v.data != null) {
+ v.data.release(1);
+ }
+ v.data = values.data;
+ v.data.retain(1);
+ clear();
+ return v;
+ }
+
+ public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
+ final Accessor fromAccessor = from.getAccessor();
+ if (!fromAccessor.isNull(fromIndex)) {
+ mutator.set(thisIndex, fromAccessor.get(fromIndex));
+ }
+ }
+
+ public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+ <#if type.major == "VarLen">
+ mutator.fillEmpties(thisIndex);
+ </#if>
+ values.copyFromSafe(fromIndex, thisIndex, from);
+ bits.getMutator().setSafe(thisIndex, 1);
+ }
+
+ public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
+ <#if type.major == "VarLen">
+ mutator.fillEmpties(thisIndex);
+ </#if>
+ bits.copyFromSafe(fromIndex, thisIndex, from.bits);
+ values.copyFromSafe(fromIndex, thisIndex, from.values);
+ }
+
+ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
+ final UInt1Vector.Accessor bAccessor = bits.getAccessor();
+ final ${valuesName}.Accessor vAccessor = values.getAccessor();
+
+ /**
+ * Get the element at the specified position.
+ *
+ * @param index position of the value
+ * @return value of the element, if not null
+ * @throws NullValueException if the value is null
+ */
+ public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+ if (isNull(index)) {
+ throw new IllegalStateException("Can't get a null value");
+ }
+ return vAccessor.get(index);
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return isSet(index) == 0;
+ }
+
+ public int isSet(int index){
+ return bAccessor.get(index);
+ }
+
+ <#if type.major == "VarLen">
+ public long getStartEnd(int index){
+ return vAccessor.getStartEnd(index);
+ }
+
+ @Override
+ public int getValueLength(int index) {
+ return values.getAccessor().getValueLength(index);
+ }
+ </#if>
+
+ public void get(int index, Nullable${minor.class}Holder holder){
+ vAccessor.get(index, holder);
+ holder.isSet = bAccessor.get(index);
+
+ <#if minor.class.startsWith("Decimal")>
+ holder.scale = getField().getScale();
+ holder.precision = getField().getPrecision();
+ </#if>
+ }
+
+ @Override
+ public ${friendlyType} getObject(int index) {
+ if (isNull(index)) {
+ return null;
+ }else{
+ return vAccessor.getObject(index);
+ }
+ }
+
+ <#if minor.class == "Interval" || minor.class == "IntervalDay" || minor.class == "IntervalYear">
+ public StringBuilder getAsStringBuilder(int index) {
+ if (isNull(index)) {
+ return null;
+ }else{
+ return vAccessor.getAsStringBuilder(index);
+ }
+ }
+ </#if>
+
+ @Override
+ public int getValueCount(){
+ return bits.getAccessor().getValueCount();
+ }
+
+ public void reset(){}
+ }
+
+ public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> {
+ private int setCount;
+ <#if type.major = "VarLen"> private int lastSet = -1;</#if>
+
+ private Mutator(){
+ }
+
+ public ${valuesName} getVectorWithValues(){
+ return values;
+ }
+
+ @Override
+ public void setIndexDefined(int index){
+ bits.getMutator().set(index, 1);
+ }
+
+ /**
+ * Set the variable length element at the specified index to the supplied byte array.
+ *
+ * @param index position of the bit to set
+ * @param bytes array of bytes to write
+ */
+ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+ setCount++;
+ final ${valuesName}.Mutator valuesMutator = values.getMutator();
+ final UInt1Vector.Mutator bitsMutator = bits.getMutator();
+ <#if type.major == "VarLen">
+ for (int i = lastSet + 1; i < index; i++) {
+ valuesMutator.set(i, emptyByteArray);
+ }
+ </#if>
+ bitsMutator.set(index, 1);
+ valuesMutator.set(index, value);
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ }
+
+ <#if type.major == "VarLen">
+
+ private void fillEmpties(int index){
+ final ${valuesName}.Mutator valuesMutator = values.getMutator();
+ for (int i = lastSet; i < index; i++) {
+ valuesMutator.setSafe(i + 1, emptyByteArray);
+ }
+ while(index > bits.getValueCapacity()) {
+ bits.reAlloc();
+ }
+ lastSet = index;
+ }
+
+ @Override
+ public void setValueLengthSafe(int index, int length) {
+ values.getMutator().setValueLengthSafe(index, length);
+ lastSet = index;
+ }
+ </#if>
+
+ public void setSafe(int index, byte[] value, int start, int length) {
+ <#if type.major != "VarLen">
+ throw new UnsupportedOperationException();
+ <#else>
+ fillEmpties(index);
+
+ bits.getMutator().setSafe(index, 1);
+ values.getMutator().setSafe(index, value, start, length);
+ setCount++;
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ </#if>
+ }
+
+ public void setSafe(int index, ByteBuffer value, int start, int length) {
+ <#if type.major != "VarLen">
+ throw new UnsupportedOperationException();
+ <#else>
+ fillEmpties(index);
+
+ bits.getMutator().setSafe(index, 1);
+ values.getMutator().setSafe(index, value, start, length);
+ setCount++;
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ </#if>
+ }
+
+ public void setNull(int index){
+ bits.getMutator().setSafe(index, 0);
+ }
+
+ public void setSkipNull(int index, ${minor.class}Holder holder){
+ values.getMutator().set(index, holder);
+ }
+
+ public void setSkipNull(int index, Nullable${minor.class}Holder holder){
+ values.getMutator().set(index, holder);
+ }
+
+
+ public void set(int index, Nullable${minor.class}Holder holder){
+ final ${valuesName}.Mutator valuesMutator = values.getMutator();
+ <#if type.major == "VarLen">
+ for (int i = lastSet + 1; i < index; i++) {
+ valuesMutator.set(i, emptyByteArray);
+ }
+ </#if>
+ bits.getMutator().set(index, holder.isSet);
+ valuesMutator.set(index, holder);
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ }
+
+ public void set(int index, ${minor.class}Holder holder){
+ final ${valuesName}.Mutator valuesMutator = values.getMutator();
+ <#if type.major == "VarLen">
+ for (int i = lastSet + 1; i < index; i++) {
+ valuesMutator.set(i, emptyByteArray);
+ }
+ </#if>
+ bits.getMutator().set(index, 1);
+ valuesMutator.set(index, holder);
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ }
+
+ public boolean isSafe(int outIndex) {
+ return outIndex < Nullable${minor.class}Vector.this.getValueCapacity();
+ }
+
+ <#assign fields = minor.fields!type.fields />
+ public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){
+ final ${valuesName}.Mutator valuesMutator = values.getMutator();
+ <#if type.major == "VarLen">
+ for (int i = lastSet + 1; i < index; i++) {
+ valuesMutator.set(i, emptyByteArray);
+ }
+ </#if>
+ bits.getMutator().set(index, isSet);
+ valuesMutator.set(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ }
+
+ public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
+ <#if type.major == "VarLen">
+ fillEmpties(index);
+ </#if>
+
+ bits.getMutator().setSafe(index, isSet);
+ values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
+ setCount++;
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ }
+
+
+ public void setSafe(int index, Nullable${minor.class}Holder value) {
+
+ <#if type.major == "VarLen">
+ fillEmpties(index);
+ </#if>
+ bits.getMutator().setSafe(index, value.isSet);
+ values.getMutator().setSafe(index, value);
+ setCount++;
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ }
+
+ public void setSafe(int index, ${minor.class}Holder value) {
+
+ <#if type.major == "VarLen">
+ fillEmpties(index);
+ </#if>
+ bits.getMutator().setSafe(index, 1);
+ values.getMutator().setSafe(index, value);
+ setCount++;
+ <#if type.major == "VarLen">lastSet = index;</#if>
+ }
+
+ <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")>
+ public void setSafe(int index, ${minor.javaType!type.javaType} value) {
+ <#if type.major == "VarLen">
+ fillEmpties(index);
+ </#if>
+ bits.getMutator().setSafe(index, 1);
+ values.getMutator().setSafe(index, value);
+ setCount++;
+ }
+
+ </#if>
+
+ @Override
+ public void setValueCount(int valueCount) {
+ assert valueCount >= 0;
+ <#if type.major == "VarLen">
+ fillEmpties(valueCount);
+ </#if>
+ values.getMutator().setValueCount(valueCount);
+ bits.getMutator().setValueCount(valueCount);
+ }
+
+ @Override
+ public void generateTestData(int valueCount){
+ bits.getMutator().generateTestDataAlt(valueCount);
+ values.getMutator().generateTestData(valueCount);
+ <#if type.major = "VarLen">lastSet = valueCount;</#if>
+ setValueCount(valueCount);
+ }
+
+ @Override
+ public void reset(){
+ setCount = 0;
+ <#if type.major = "VarLen">lastSet = -1;</#if>
+ }
+ }
+}
+</#list>
+</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
new file mode 100644
index 0000000..2f5ec18
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
@@ -0,0 +1,427 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector;
+import org.mortbay.jetty.servlet.Holder;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+<#assign fields = minor.fields!type.fields />
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/Repeated${minor.class}Vector.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
+ * repeated protobuf field). The implementation uses two additional value vectors; one to convert
+ * the index offset to the underlying element offset, and another to store the number of values
+ * in the vector.
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+
+public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>VectorLike {
+ //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Repeated${minor.class}Vector.class);
+
+ // we maintain local reference to concrete vector type for performance reasons.
+ private ${minor.class}Vector values;
+ private final FieldReader reader = new Repeated${minor.class}ReaderImpl(Repeated${minor.class}Vector.this);
+ private final Mutator mutator = new Mutator();
+ private final Accessor accessor = new Accessor();
+
+ public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ addOrGetVector(VectorDescriptor.create(Types.required(field.getType().getMinorType())));
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public FieldReader getReader() {
+ return reader;
+ }
+
+ @Override
+ public ${minor.class}Vector getDataVector() {
+ return values;
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ return new TransferImpl(getField());
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref){
+ return new TransferImpl(getField().withPath(ref));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new TransferImpl((Repeated${minor.class}Vector) to);
+ }
+
+ @Override
+ public AddOrGetResult<${minor.class}Vector> addOrGetVector(VectorDescriptor descriptor) {
+ final AddOrGetResult<${minor.class}Vector> result = super.addOrGetVector(descriptor);
+ if (result.isCreated()) {
+ values = result.getVector();
+ }
+ return result;
+ }
+
+ public void transferTo(Repeated${minor.class}Vector target) {
+ target.clear();
+ offsets.transferTo(target.offsets);
+ values.transferTo(target.values);
+ clear();
+ }
+
+ public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) {
+ final UInt4Vector.Accessor a = offsets.getAccessor();
+ final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+ final int startPos = a.get(startIndex);
+ final int endPos = a.get(startIndex + groups);
+ final int valuesToCopy = endPos - startPos;
+
+ values.splitAndTransferTo(startPos, valuesToCopy, to.values);
+ to.offsets.clear();
+ to.offsets.allocateNew(groups + 1);
+ int normalizedPos = 0;
+ for (int i=0; i < groups + 1;i++ ) {
+ normalizedPos = a.get(startIndex+i) - startPos;
+ m.set(i, normalizedPos);
+ }
+ m.setValueCount(groups == 0 ? 0 : groups + 1);
+ }
+
+ private class TransferImpl implements TransferPair {
+ final Repeated${minor.class}Vector to;
+
+ public TransferImpl(MaterializedField field) {
+ this.to = new Repeated${minor.class}Vector(field, allocator);
+ }
+
+ public TransferImpl(Repeated${minor.class}Vector to) {
+ this.to = to;
+ }
+
+ @Override
+ public Repeated${minor.class}Vector getTo() {
+ return to;
+ }
+
+ @Override
+ public void transfer() {
+ transferTo(to);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ splitAndTransferTo(startIndex, length, to);
+ }
+
+ @Override
+ public void copyValueSafe(int fromIndex, int toIndex) {
+ to.copyFromSafe(fromIndex, toIndex, Repeated${minor.class}Vector.this);
+ }
+ }
+
+ public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
+ final Accessor vAccessor = v.getAccessor();
+ final int count = vAccessor.getInnerValueCountAt(inIndex);
+ mutator.startNewValue(outIndex);
+ for (int i = 0; i < count; i++) {
+ mutator.add(outIndex, vAccessor.get(inIndex, i));
+ }
+ }
+
+ public void copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
+ final Accessor vAccessor = v.getAccessor();
+ final int count = vAccessor.getInnerValueCountAt(inIndex);
+ mutator.startNewValue(outIndex);
+ for (int i = 0; i < count; i++) {
+ mutator.addSafe(outIndex, vAccessor.get(inIndex, i));
+ }
+ }
+
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if(!offsets.allocateNewSafe()) return false;
+ if(!values.allocateNewSafe()) return false;
+ success = true;
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ offsets.zeroVector();
+ mutator.reset();
+ return true;
+ }
+
+ @Override
+ public void allocateNew() {
+ try {
+ offsets.allocateNew();
+ values.allocateNew();
+ } catch (OutOfMemoryRuntimeException e) {
+ clear();
+ throw e;
+ }
+ offsets.zeroVector();
+ mutator.reset();
+ }
+
+ <#if type.major == "VarLen">
+ @Override
+ protected SerializedField.Builder getMetadataBuilder() {
+ return super.getMetadataBuilder()
+ .setVarByteLength(values.getVarByteLength());
+ }
+
+ public void allocateNew(int totalBytes, int valueCount, int innerValueCount) {
+ try {
+ offsets.allocateNew(valueCount + 1);
+ values.allocateNew(totalBytes, innerValueCount);
+ } catch (OutOfMemoryRuntimeException e) {
+ clear();
+ throw e;
+ }
+ offsets.zeroVector();
+ mutator.reset();
+ }
+
+ public int getByteCapacity(){
+ return values.getByteCapacity();
+ }
+
+ <#else>
+
+ @Override
+ public void allocateNew(int valueCount, int innerValueCount) {
+ clear();
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to//
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ offsets.allocateNew(valueCount + 1);
+ values.allocateNew(innerValueCount);
+ } catch(OutOfMemoryRuntimeException e){
+ clear();
+ throw e;
+ }
+ offsets.zeroVector();
+ mutator.reset();
+ }
+
+ </#if>
+
+ // This is declared a subclass of the accessor declared inside of FixedWidthVector, this is also used for
+ // variable length vectors, as they should ahve consistent interface as much as possible, if they need to diverge
+ // in the future, the interface shold be declared in the respective value vector superclasses for fixed and variable
+ // and we should refer to each in the generation template
+ public final class Accessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
+ @Override
+ public List<${friendlyType}> getObject(int index) {
+ final List<${friendlyType}> vals = new JsonStringArrayList<>();
+ final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
+ final int start = offsetsAccessor.get(index);
+ final int end = offsetsAccessor.get(index + 1);
+ final ${minor.class}Vector.Accessor valuesAccessor = values.getAccessor();
+ for(int i = start; i < end; i++) {
+ vals.add(valuesAccessor.getObject(i));
+ }
+ return vals;
+ }
+
+ public ${friendlyType} getSingleObject(int index, int arrayIndex) {
+ final int start = offsets.getAccessor().get(index);
+ return values.getAccessor().getObject(start + arrayIndex);
+ }
+
+ /**
+ * Get a value for the given record. Each element in the repeated field is accessed by
+ * the positionIndex param.
+ *
+ * @param index record containing the repeated field
+ * @param positionIndex position within the repeated field
+ * @return element at the given position in the given record
+ */
+ public <#if type.major == "VarLen">byte[]
+ <#else>${minor.javaType!type.javaType}
+ </#if> get(int index, int positionIndex) {
+ return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
+ }
+
+ public void get(int index, Repeated${minor.class}Holder holder) {
+ holder.start = offsets.getAccessor().get(index);
+ holder.end = offsets.getAccessor().get(index+1);
+ holder.vector = values;
+ }
+
+ public void get(int index, int positionIndex, ${minor.class}Holder holder) {
+ final int offset = offsets.getAccessor().get(index);
+ assert offset >= 0;
+ assert positionIndex < getInnerValueCountAt(index);
+ values.getAccessor().get(offset + positionIndex, holder);
+ }
+
+ public void get(int index, int positionIndex, Nullable${minor.class}Holder holder) {
+ final int offset = offsets.getAccessor().get(index);
+ assert offset >= 0;
+ if (positionIndex >= getInnerValueCountAt(index)) {
+ holder.isSet = 0;
+ return;
+ }
+ values.getAccessor().get(offset + positionIndex, holder);
+ }
+ }
+
+ public final class Mutator extends BaseRepeatedValueVector.BaseRepeatedMutator implements RepeatedMutator {
+ private Mutator() {}
+
+ /**
+ * Add an element to the given record index. This is similar to the set() method in other
+ * value vectors, except that it permits setting multiple values for a single record.
+ *
+ * @param index record of the element to add
+ * @param value value to add to the given row
+ */
+ public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+ int nextOffset = offsets.getAccessor().get(index+1);
+ values.getMutator().set(nextOffset, value);
+ offsets.getMutator().set(index+1, nextOffset+1);
+ }
+
+ <#if type.major == "VarLen">
+ public void addSafe(int index, byte[] bytes) {
+ addSafe(index, bytes, 0, bytes.length);
+ }
+
+ public void addSafe(int index, byte[] bytes, int start, int length) {
+ final int nextOffset = offsets.getAccessor().get(index+1);
+ values.getMutator().setSafe(nextOffset, bytes, start, length);
+ offsets.getMutator().setSafe(index+1, nextOffset+1);
+ }
+
+ <#else>
+
+ public void addSafe(int index, ${minor.javaType!type.javaType} srcValue) {
+ final int nextOffset = offsets.getAccessor().get(index+1);
+ values.getMutator().setSafe(nextOffset, srcValue);
+ offsets.getMutator().setSafe(index+1, nextOffset+1);
+ }
+
+ </#if>
+
+ public void setSafe(int index, Repeated${minor.class}Holder h) {
+ final ${minor.class}Holder ih = new ${minor.class}Holder();
+ final ${minor.class}Vector.Accessor hVectorAccessor = h.vector.getAccessor();
+ mutator.startNewValue(index);
+ for(int i = h.start; i < h.end; i++){
+ hVectorAccessor.get(i, ih);
+ mutator.addSafe(index, ih);
+ }
+ }
+
+ public void addSafe(int index, ${minor.class}Holder holder) {
+ int nextOffset = offsets.getAccessor().get(index+1);
+ values.getMutator().setSafe(nextOffset, holder);
+ offsets.getMutator().setSafe(index+1, nextOffset+1);
+ }
+
+ public void addSafe(int index, Nullable${minor.class}Holder holder) {
+ final int nextOffset = offsets.getAccessor().get(index+1);
+ values.getMutator().setSafe(nextOffset, holder);
+ offsets.getMutator().setSafe(index+1, nextOffset+1);
+ }
+
+ <#if (fields?size > 1) && !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+ public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ int nextOffset = offsets.getAccessor().get(arrayIndex+1);
+ values.getMutator().setSafe(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ offsets.getMutator().setSafe(arrayIndex+1, nextOffset+1);
+ }
+ </#if>
+
+ protected void add(int index, ${minor.class}Holder holder) {
+ int nextOffset = offsets.getAccessor().get(index+1);
+ values.getMutator().set(nextOffset, holder);
+ offsets.getMutator().set(index+1, nextOffset+1);
+ }
+
+ public void add(int index, Repeated${minor.class}Holder holder) {
+
+ ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
+ ${minor.class}Holder innerHolder = new ${minor.class}Holder();
+
+ for(int i = holder.start; i < holder.end; i++) {
+ accessor.get(i, innerHolder);
+ add(index, innerHolder);
+ }
+ }
+
+ @Override
+ public void generateTestData(final int valCount) {
+ final int[] sizes = {1, 2, 0, 6};
+ int size = 0;
+ int runningOffset = 0;
+ final UInt4Vector.Mutator offsetsMutator = offsets.getMutator();
+ for(int i = 1; i < valCount + 1; i++, size++) {
+ runningOffset += sizes[size % sizes.length];
+ offsetsMutator.set(i, runningOffset);
+ }
+ values.getMutator().generateTestData(valCount * 9);
+ setValueCount(size);
+ }
+
+ @Override
+ public void reset() {
+ }
+ }
+}
+</#list>
+</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/TypeHelper.java b/exec/vector/src/main/codegen/templates/TypeHelper.java
new file mode 100644
index 0000000..b8178cb
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/TypeHelper.java
@@ -0,0 +1,600 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.exec.vector.complex.UnionVector;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/TypeHelper.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr;
+
+<#include "/@includes/vv_imports.ftl" />
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.accessor.*;
+import org.apache.drill.exec.vector.complex.RepeatedMapVector;
+import org.apache.drill.exec.util.CallBack;
+
+public class TypeHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
+
+ private static final int WIDTH_ESTIMATE = 50;
+
+ // Default length when casting to varchar : 65536 = 2^16
+ // This only defines an absolute maximum for values, setting
+ // a high value like this will not inflate the size for small values
+ public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
+
+ private static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
+ return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
+ }
+
+ private static String buildErrorMessage(final String operation, final MajorType type) {
+ return buildErrorMessage(operation, type.getMinorType(), type.getMode());
+ }
+
+ public static int getSize(MajorType major) {
+ switch (major.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
+ minor.class?substring(0, 3) == "PRO" ||
+ minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
+ </#list>
+</#list>
+ case FIXEDCHAR: return major.getWidth();
+ case FIXED16CHAR: return major.getWidth();
+ case FIXEDBINARY: return major.getWidth();
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get size", major));
+ }
+
+ public static SqlAccessor getSqlAccessor(ValueVector vector){
+ final MajorType type = vector.getField().getType();
+ switch(type.getMinorType()){
+ case UNION:
+ return new UnionSqlAccessor((UnionVector) vector);
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Accessor((${minor.class}Vector) vector);
+ case OPTIONAL:
+ return new Nullable${minor.class}Accessor((Nullable${minor.class}Vector) vector);
+ case REPEATED:
+ return new GenericAccessor(vector);
+ }
+ </#list>
+ </#list>
+ case MAP:
+ case LIST:
+ return new GenericAccessor(vector);
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("find sql accessor", type));
+ }
+
+ public static ValueVector getNewVector(SchemaPath parentPath, String name, BufferAllocator allocator, MajorType type, CallBack callback){
+ SchemaPath child = parentPath.getChild(name);
+ MaterializedField field = MaterializedField.create(child, type);
+ return getNewVector(field, allocator, callback);
+ }
+
+
+ public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
+ switch (type) {
+ case UNION:
+ return UnionVector.class;
+ case MAP:
+ switch (mode) {
+ case OPTIONAL:
+ case REQUIRED:
+ return MapVector.class;
+ case REPEATED:
+ return RepeatedMapVector.class;
+ }
+
+ case LIST:
+ switch (mode) {
+ case REPEATED:
+ return RepeatedListVector.class;
+ case REQUIRED:
+ case OPTIONAL:
+ return ListVector.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}Vector.class;
+ case OPTIONAL:
+ return Nullable${minor.class}Vector.class;
+ case REPEATED:
+ return Repeated${minor.class}Vector.class;
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT :
+ return ObjectVector.class ;
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
+ }
+ public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
+ switch (type) {
+ case MAP:
+ switch (mode) {
+ case REQUIRED:
+ if (!isSingularRepeated)
+ return SingleMapReaderImpl.class;
+ else
+ return SingleLikeRepeatedMapReaderImpl.class;
+ case REPEATED:
+ return RepeatedMapReaderImpl.class;
+ }
+ case LIST:
+ switch (mode) {
+ case REQUIRED:
+ return SingleListReaderImpl.class;
+ case REPEATED:
+ return RepeatedListReaderImpl.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}ReaderImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}ReaderImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}ReaderImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
+ }
+
+ public static Class<?> getWriterInterface( MinorType type, DataMode mode){
+ switch (type) {
+ case UNION: return UnionWriter.class;
+ case MAP: return MapWriter.class;
+ case LIST: return ListWriter.class;
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}: return ${minor.class}Writer.class;
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
+ }
+
+ public static Class<?> getWriterImpl( MinorType type, DataMode mode){
+ switch (type) {
+ case UNION:
+ return UnionWriter.class;
+ case MAP:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return SingleMapWriter.class;
+ case REPEATED:
+ return RepeatedMapWriter.class;
+ }
+ case LIST:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return UnionListWriter.class;
+ case REPEATED:
+ return RepeatedListWriter.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}WriterImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}WriterImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}WriterImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
+ }
+
+ public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
+ switch (type) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}HolderReaderImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}HolderReaderImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}HolderReaderImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
+ }
+
+ public static JType getHolderType(JCodeModel model, MinorType type, DataMode mode){
+ switch (type) {
+ case UNION:
+ return model._ref(UnionHolder.class);
+ case MAP:
+ case LIST:
+ return model._ref(ComplexHolder.class);
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return model._ref(${minor.class}Holder.class);
+ case OPTIONAL:
+ return model._ref(Nullable${minor.class}Holder.class);
+ case REPEATED:
+ return model._ref(Repeated${minor.class}Holder.class);
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ return model._ref(ObjectHolder.class);
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get holder type", type, mode));
+ }
+
+ public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
+ return getNewVector(field, allocator, null);
+ }
+ public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+ MajorType type = field.getType();
+
+ switch (type.getMinorType()) {
+
+ case UNION:
+ return new UnionVector(field, allocator, callBack);
+
+ case MAP:
+ switch (type.getMode()) {
+ case REQUIRED:
+ case OPTIONAL:
+ return new MapVector(field, allocator, callBack);
+ case REPEATED:
+ return new RepeatedMapVector(field, allocator, callBack);
+ }
+ case LIST:
+ switch (type.getMode()) {
+ case REPEATED:
+ return new RepeatedListVector(field, allocator, callBack);
+ case OPTIONAL:
+ case REQUIRED:
+ return new ListVector(field, allocator, callBack);
+ }
+<#list vv. types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Vector(field, allocator);
+ case OPTIONAL:
+ return new Nullable${minor.class}Vector(field, allocator);
+ case REPEATED:
+ return new Repeated${minor.class}Vector(field, allocator);
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ return new ObjectVector(field, allocator) ;
+ default:
+ break;
+ }
+ // All ValueVector types have been handled.
+ throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
+ }
+
+ public static ValueHolder getValue(ValueVector vector, int index) {
+ MajorType type = vector.getField().getType();
+ ValueHolder holder;
+ switch(type.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
+ minor.class?starts_with("Decimal28") || minor.class?starts_with("Decimal38")>
+ switch (type.getMode()) {
+ case REQUIRED:
+ holder = new ${minor.class}Holder();
+ ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
+ return holder;
+ case OPTIONAL:
+ holder = new Nullable${minor.class}Holder();
+ ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
+ if (((Nullable${minor.class}Holder)holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
+ }
+ return holder;
+ }
+ <#else>
+ switch (type.getMode()) {
+ case REQUIRED:
+ holder = new ${minor.class}Holder();
+ ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
+ return holder;
+ case OPTIONAL:
+ holder = new Nullable${minor.class}Holder();
+ ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
+ if (((Nullable${minor.class}Holder)holder).isSet == 1) {
+ ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
+ }
+ return holder;
+ }
+ </#if>
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ holder = new ObjectHolder();
+ ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index) ;
+ break;
+ }
+
+ throw new UnsupportedOperationException(buildErrorMessage("get value", type));
+ }
+
+ public static void setValue(ValueVector vector, int index, ValueHolder holder) {
+ MajorType type = vector.getField().getType();
+
+ switch(type.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
+ return;
+ case OPTIONAL:
+ if (((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
+ }
+ return;
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
+ return;
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("set value", type));
+ }
+ }
+
+ public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
+ MajorType type = vector.getField().getType();
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
+ return;
+ case OPTIONAL:
+ if (((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
+ } else {
+ ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
+ }
+ return;
+ }
+ </#list>
+ </#list>
+ case GENERIC_OBJECT:
+ ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
+ }
+ }
+
+ public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
+ MajorType type1 = v1.getField().getType();
+ MajorType type2 = v2.getField().getType();
+
+ if (type1.getMinorType() != type2.getMinorType()) {
+ return false;
+ }
+
+ switch(type1.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) ==
+ ((${minor.class}Vector) v2).getAccessor().get(v2index) )
+ return true;
+ break;
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ return false;
+ }
+
+ /**
+ * Create a ValueHolder of MajorType.
+ * @param type
+ * @return
+ */
+ public static ValueHolder createValueHolder(MajorType type) {
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Holder();
+ case OPTIONAL:
+ return new Nullable${minor.class}Holder();
+ case REPEATED:
+ return new Repeated${minor.class}Holder();
+ }
+ </#list>
+ </#list>
+ case GENERIC_OBJECT:
+ return new ObjectHolder();
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
+ }
+ }
+
+ public static boolean isNull(ValueHolder holder) {
+ MajorType type = TypeHelper.getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return true;
+ case OPTIONAL:
+ return ((Nullable${minor.class}Holder) holder).isSet == 0;
+ case REPEATED:
+ return true;
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
+ }
+ }
+
+ public static ValueHolder deNullify(ValueHolder holder) {
+ MajorType type = TypeHelper.getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return holder;
+ case OPTIONAL:
+ if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ${minor.class}Holder newHolder = new ${minor.class}Holder();
+
+ <#assign fields = minor.fields!type.fields />
+ <#list fields as field>
+ newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
+ </#list>
+
+ return newHolder;
+ } else {
+ throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
+ }
+ case REPEATED:
+ return holder;
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
+ }
+ }
+
+ public static ValueHolder nullify(ValueHolder holder) {
+ MajorType type = TypeHelper.getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
+ newHolder.isSet = 1;
+ <#assign fields = minor.fields!type.fields />
+ <#list fields as field>
+ newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
+ </#list>
+ return newHolder;
+ case OPTIONAL:
+ return holder;
+ case REPEATED:
+ throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
+ }
+ }
+
+ public static MajorType getValueHolderType(ValueHolder holder) {
+
+ if (0 == 1) {
+ return null;
+ }
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ else if (holder instanceof ${minor.class}Holder) {
+ return ((${minor.class}Holder) holder).TYPE;
+ } else if (holder instanceof Nullable${minor.class}Holder) {
+ return ((Nullable${minor.class}Holder) holder).TYPE;
+ }
+ </#list>
+ </#list>
+
+ throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/UnionListWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/UnionListWriter.java b/exec/vector/src/main/codegen/templates/UnionListWriter.java
new file mode 100644
index 0000000..c676769
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/UnionListWriter.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.UnsupportedOperationException;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionListWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+
+@SuppressWarnings("unused")
+public class UnionListWriter extends AbstractFieldWriter {
+
+ private ListVector vector;
+ private UInt4Vector offsets;
+ private PromotableWriter writer;
+ private boolean inMap = false;
+ private String mapName;
+ private int lastIndex = 0;
+
+ public UnionListWriter(ListVector vector) {
+ super(null);
+ this.vector = vector;
+ this.writer = new PromotableWriter(vector.getDataVector(), vector);
+ this.offsets = vector.getOffsetVector();
+ }
+
+ public UnionListWriter(ListVector vector, AbstractFieldWriter parent) {
+ this(vector);
+ }
+
+ @Override
+ public void allocate() {
+ vector.allocateNew();
+ }
+
+ @Override
+ public void clear() {
+ vector.clear();
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return null;
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return vector.getValueCapacity();
+ }
+
+ @Override
+ public void close() throws Exception {
+
+ }
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+
+ <#if !minor.class?starts_with("Decimal")>
+
+ @Override
+ public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>() {
+ return this;
+ }
+
+ @Override
+ public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
+ assert inMap;
+ mapName = name;
+ final int nextOffset = offsets.getAccessor().get(idx() + 1);
+ vector.getMutator().setNotNull(idx());
+ writer.setPosition(nextOffset);
+ ${name}Writer ${uncappedName}Writer = writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name);
+ return ${uncappedName}Writer;
+ }
+
+ </#if>
+
+ </#list></#list>
+
+ @Override
+ public MapWriter map() {
+ inMap = true;
+ return this;
+ }
+
+ @Override
+ public ListWriter list() {
+ final int nextOffset = offsets.getAccessor().get(idx() + 1);
+ vector.getMutator().setNotNull(idx());
+ offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
+ writer.setPosition(nextOffset);
+ return writer;
+ }
+
+ @Override
+ public ListWriter list(String name) {
+ final int nextOffset = offsets.getAccessor().get(idx() + 1);
+ vector.getMutator().setNotNull(idx());
+ writer.setPosition(nextOffset);
+ ListWriter listWriter = writer.list(name);
+ return listWriter;
+ }
+
+ @Override
+ public MapWriter map(String name) {
+ MapWriter mapWriter = writer.map(name);
+ return mapWriter;
+ }
+
+ @Override
+ public void startList() {
+ vector.getMutator().startNewValue(idx());
+ }
+
+ @Override
+ public void endList() {
+
+ }
+
+ @Override
+ public void start() {
+ assert inMap;
+ final int nextOffset = offsets.getAccessor().get(idx() + 1);
+ vector.getMutator().setNotNull(idx());
+ offsets.getMutator().setSafe(idx() + 1, nextOffset);
+ writer.setPosition(nextOffset);
+ }
+
+ @Override
+ public void end() {
+ if (inMap) {
+ inMap = false;
+ final int nextOffset = offsets.getAccessor().get(idx() + 1);
+ offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
+ }
+ }
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+
+ <#if !minor.class?starts_with("Decimal")>
+
+ @Override
+ public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ assert !inMap;
+ final int nextOffset = offsets.getAccessor().get(idx() + 1);
+ vector.getMutator().setNotNull(idx());
+ writer.setPosition(nextOffset);
+ writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
+ }
+
+ </#if>
+
+ </#list></#list>
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/UnionReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/UnionReader.java b/exec/vector/src/main/codegen/templates/UnionReader.java
new file mode 100644
index 0000000..46a32ee
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/UnionReader.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.vector.complex.impl.NullReader;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionReader.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+@SuppressWarnings("unused")
+public class UnionReader extends AbstractFieldReader {
+
+ private BaseReader[] readers = new BaseReader[43];
+ public UnionVector data;
+
+ public UnionReader(UnionVector data) {
+ this.data = data;
+ }
+
+ private static MajorType[] TYPES = new MajorType[43];
+
+ static {
+ for (MinorType minorType : MinorType.values()) {
+ TYPES[minorType.getNumber()] = Types.optional(minorType);
+ }
+ }
+
+ public MajorType getType() {
+ return TYPES[data.getTypeValue(idx())];
+ }
+
+ public boolean isSet(){
+ return !data.getAccessor().isNull(idx());
+ }
+
+ public void read(UnionHolder holder) {
+ holder.reader = this;
+ holder.isSet = this.isSet() ? 1 : 0;
+ }
+
+ public void read(int index, UnionHolder holder) {
+ getList().read(index, holder);
+ }
+
+ private FieldReader getReaderForIndex(int index) {
+ int typeValue = data.getTypeValue(index);
+ FieldReader reader = (FieldReader) readers[typeValue];
+ if (reader != null) {
+ return reader;
+ }
+ switch (typeValue) {
+ case 0:
+ return NullReader.INSTANCE;
+ case MinorType.MAP_VALUE:
+ return (FieldReader) getMap();
+ case MinorType.LIST_VALUE:
+ return (FieldReader) getList();
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+ case MinorType.${name?upper_case}_VALUE:
+ return (FieldReader) get${name}();
+ </#if>
+ </#list></#list>
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + MinorType.valueOf(typeValue));
+ }
+ }
+
+ private SingleMapReaderImpl mapReader;
+
+ private MapReader getMap() {
+ if (mapReader == null) {
+ mapReader = (SingleMapReaderImpl) data.getMap().getReader();
+ mapReader.setPosition(idx());
+ readers[MinorType.MAP_VALUE] = mapReader;
+ }
+ return mapReader;
+ }
+
+ private UnionListReader listReader;
+
+ private FieldReader getList() {
+ if (listReader == null) {
+ listReader = new UnionListReader(data.getList());
+ listReader.setPosition(idx());
+ readers[MinorType.LIST_VALUE] = listReader;
+ }
+ return listReader;
+ }
+
+ @Override
+ public java.util.Iterator<String> iterator() {
+ return getMap().iterator();
+ }
+
+ @Override
+ public void copyAsValue(UnionWriter writer) {
+ writer.data.copyFrom(idx(), writer.idx(), data);
+ }
+
+ <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
+ "Character", "DateTime", "Period", "Double", "Float",
+ "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
+ <#assign safeType=friendlyType />
+ <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+
+ @Override
+ public ${friendlyType} read${safeType}() {
+ return getReaderForIndex(idx()).read${safeType}();
+ }
+
+ </#list>
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign uncappedName = name?uncap_first/>
+ <#assign boxedType = (minor.boxedType!type.boxedType) />
+ <#assign javaType = (minor.javaType!type.javaType) />
+ <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+ <#assign safeType=friendlyType />
+ <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+ <#if !minor.class?starts_with("Decimal")>
+
+ private Nullable${name}ReaderImpl ${uncappedName}Reader;
+
+ private Nullable${name}ReaderImpl get${name}() {
+ if (${uncappedName}Reader == null) {
+ ${uncappedName}Reader = new Nullable${name}ReaderImpl(data.get${name}Vector());
+ ${uncappedName}Reader.setPosition(idx());
+ readers[MinorType.${name?upper_case}_VALUE] = ${uncappedName}Reader;
+ }
+ return ${uncappedName}Reader;
+ }
+
+ public void read(Nullable${name}Holder holder){
+ getReaderForIndex(idx()).read(holder);
+ }
+
+ public void copyAsValue(${name}Writer writer){
+ getReaderForIndex(idx()).copyAsValue(writer);
+ }
+ </#if>
+ </#list></#list>
+
+ @Override
+ public void copyAsValue(ListWriter writer) {
+ ComplexCopier.copy(this, (FieldWriter) writer);
+ }
+
+ @Override
+ public void setPosition(int index) {
+ super.setPosition(index);
+ for (BaseReader reader : readers) {
+ if (reader != null) {
+ reader.setPosition(index);
+ }
+ }
+ }
+
+ public FieldReader reader(String name){
+ return getMap().reader(name);
+ }
+
+ public FieldReader reader() {
+ return getList().reader();
+ }
+
+ public boolean next() {
+ return getReaderForIndex(idx()).next();
+ }
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/UnionVector.java b/exec/vector/src/main/codegen/templates/UnionVector.java
new file mode 100644
index 0000000..dfcccb2
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/UnionVector.java
@@ -0,0 +1,479 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/UnionVector.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex;
+
+<#include "/@includes/vv_imports.ftl" />
+import java.util.Iterator;
+import org.apache.drill.exec.vector.complex.impl.ComplexCopier;
+import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.common.expression.PathSegment;
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+
+
+/**
+ * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each
+ * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods,
+ * as well as the addOrGet method.
+ *
+ * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
+ * each time the vector is accessed.
+ */
+public class UnionVector implements ValueVector {
+
+ private MaterializedField field;
+ private BufferAllocator allocator;
+ private Accessor accessor = new Accessor();
+ private Mutator mutator = new Mutator();
+ private int valueCount;
+
+ private MapVector internalMap;
+ private UInt1Vector typeVector;
+
+ private MapVector mapVector;
+ private ListVector listVector;
+
+ private FieldReader reader;
+ private NullableBitVector bit;
+
+ private int singleType = 0;
+ private ValueVector singleVector;
+ private MajorType majorType;
+
+ private final CallBack callBack;
+
+ public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+ this.field = field.clone();
+ this.allocator = allocator;
+ this.internalMap = new MapVector("internal", allocator, callBack);
+ this.typeVector = internalMap.addOrGet("types", Types.required(MinorType.UINT1), UInt1Vector.class);
+ this.field.addChild(internalMap.getField().clone());
+ this.majorType = field.getType();
+ this.callBack = callBack;
+ }
+
+ public List<MinorType> getSubTypes() {
+ return majorType.getSubTypeList();
+ }
+
+ private void addSubType(MinorType type) {
+ majorType = MajorType.newBuilder(this.majorType).addSubType(type).build();
+ if (callBack != null) {
+ callBack.doWork();
+ }
+ }
+
+ private static final MajorType MAP_TYPE = Types.optional(MinorType.MAP);
+
+ public MapVector getMap() {
+ if (mapVector == null) {
+ int vectorCount = internalMap.size();
+ mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class);
+ addSubType(MinorType.MAP);
+ if (internalMap.size() > vectorCount) {
+ mapVector.allocateNew();
+ }
+ }
+ return mapVector;
+ }
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+
+ private Nullable${name}Vector ${uncappedName}Vector;
+ private static final MajorType ${name?upper_case}_TYPE = Types.optional(MinorType.${name?upper_case});
+
+ public Nullable${name}Vector get${name}Vector() {
+ if (${uncappedName}Vector == null) {
+ int vectorCount = internalMap.size();
+ ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class);
+ addSubType(MinorType.${name?upper_case});
+ if (internalMap.size() > vectorCount) {
+ ${uncappedName}Vector.allocateNew();
+ }
+ }
+ return ${uncappedName}Vector;
+ }
+
+ </#if>
+
+ </#list></#list>
+
+ private static final MajorType LIST_TYPE = Types.optional(MinorType.LIST);
+
+ public ListVector getList() {
+ if (listVector == null) {
+ int vectorCount = internalMap.size();
+ listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class);
+ addSubType(MinorType.LIST);
+ if (internalMap.size() > vectorCount) {
+ listVector.allocateNew();
+ }
+ }
+ return listVector;
+ }
+
+ public int getTypeValue(int index) {
+ return typeVector.getAccessor().get(index);
+ }
+
+ public UInt1Vector getTypeVector() {
+ return typeVector;
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryRuntimeException {
+ internalMap.allocateNew();
+ if (typeVector != null) {
+ typeVector.zeroVector();
+ }
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ boolean safe = internalMap.allocateNewSafe();
+ if (safe) {
+ if (typeVector != null) {
+ typeVector.zeroVector();
+ }
+ }
+ return safe;
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity());
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public void clear() {
+ internalMap.clear();
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ return new TransferImpl(field);
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ return new TransferImpl(field.withPath(ref));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector target) {
+ return new TransferImpl((UnionVector) target);
+ }
+
+ public void transferTo(UnionVector target) {
+ internalMap.makeTransferPair(target.internalMap).transfer();
+ target.valueCount = valueCount;
+ target.majorType = majorType;
+ }
+
+ public void copyFrom(int inIndex, int outIndex, UnionVector from) {
+ from.getReader().setPosition(inIndex);
+ getWriter().setPosition(outIndex);
+ ComplexCopier.copy(from.reader, mutator.writer);
+ }
+
+ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
+ copyFrom(inIndex, outIndex, from);
+ }
+
+ public void addVector(ValueVector v) {
+ String name = v.getField().getType().getMinorType().name().toLowerCase();
+ Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
+ internalMap.putChild(name, v);
+ addSubType(v.getField().getType().getMinorType());
+ }
+
+ private class TransferImpl implements TransferPair {
+
+ UnionVector to;
+
+ public TransferImpl(MaterializedField field) {
+ to = new UnionVector(field, allocator, null);
+ }
+
+ public TransferImpl(UnionVector to) {
+ this.to = to;
+ }
+
+ @Override
+ public void transfer() {
+ transferTo(to);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+
+ }
+
+ @Override
+ public ValueVector getTo() {
+ return to;
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) {
+ this.to.copyFrom(from, to, UnionVector.this);
+ }
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ @Override
+ public FieldReader getReader() {
+ if (reader == null) {
+ reader = new UnionReader(this);
+ }
+ return reader;
+ }
+
+ public FieldWriter getWriter() {
+ if (mutator.writer == null) {
+ mutator.writer = new UnionWriter(this);
+ }
+ return mutator.writer;
+ }
+
+ @Override
+ public UserBitShared.SerializedField getMetadata() {
+ SerializedField.Builder b = getField() //
+ .getAsBuilder() //
+ .setBufferLength(getBufferSize()) //
+ .setValueCount(valueCount);
+
+ b.addChild(internalMap.getMetadata());
+ return b.build();
+ }
+
+ @Override
+ public int getBufferSize() {
+ return internalMap.getBufferSize();
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ long bufferSize = 0;
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ bufferSize += v.getBufferSizeFor(valueCount);
+ }
+
+ return (int) bufferSize;
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ return internalMap.getBuffers(clear);
+ }
+
+ @Override
+ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+ valueCount = metadata.getValueCount();
+
+ internalMap.load(metadata.getChild(0), buffer);
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ List<ValueVector> vectors = Lists.newArrayList(internalMap.iterator());
+ vectors.add(typeVector);
+ return vectors.iterator();
+ }
+
+ public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
+ if (seg.isNamed()) {
+ ValueVector v = getMap();
+ if (v != null) {
+ return ((AbstractContainerVector) v).getFieldIdIfMatches(builder, addToBreadCrumb, seg);
+ } else {
+ return null;
+ }
+ } else if (seg.isArray()) {
+ ValueVector v = getList();
+ if (v != null) {
+ return ((ListVector) v).getFieldIdIfMatches(builder, addToBreadCrumb, seg);
+ }
+ else return null;
+ }
+ return null;
+ }
+
+ public class Accessor extends BaseValueVector.BaseAccessor {
+
+
+ @Override
+ public Object getObject(int index) {
+ int type = typeVector.getAccessor().get(index);
+ switch (type) {
+ case 0:
+ return null;
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+ case MinorType.${name?upper_case}_VALUE:
+ return get${name}Vector().getAccessor().getObject(index);
+ </#if>
+
+ </#list></#list>
+ case MinorType.MAP_VALUE:
+ return getMap().getAccessor().getObject(index);
+ case MinorType.LIST_VALUE:
+ return getList().getAccessor().getObject(index);
+ default:
+ throw new UnsupportedOperationException("Cannot support type: " + MinorType.valueOf(type));
+ }
+ }
+
+ public byte[] get(int index) {
+ return null;
+ }
+
+ public void get(int index, ComplexHolder holder) {
+ }
+
+ public void get(int index, UnionHolder holder) {
+ if (reader == null) {
+ reader = new UnionReader(UnionVector.this);
+ }
+ reader.setPosition(index);
+ holder.reader = reader;
+ }
+
+ @Override
+ public int getValueCount() {
+ return valueCount;
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return typeVector.getAccessor().get(index) == 0;
+ }
+
+ public int isSet(int index) {
+ return isNull(index) ? 0 : 1;
+ }
+ }
+
+ public class Mutator extends BaseValueVector.BaseMutator {
+
+ UnionWriter writer;
+
+ @Override
+ public void setValueCount(int valueCount) {
+ UnionVector.this.valueCount = valueCount;
+ internalMap.getMutator().setValueCount(valueCount);
+ }
+
+ public void setSafe(int index, UnionHolder holder) {
+ FieldReader reader = holder.reader;
+ if (writer == null) {
+ writer = new UnionWriter(UnionVector.this);
+ }
+ writer.setPosition(index);
+ MinorType type = reader.getType().getMinorType();
+ switch (type) {
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+ case ${name?upper_case}:
+ Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
+ reader.read(${uncappedName}Holder);
+ setSafe(index, ${uncappedName}Holder);
+ break;
+ </#if>
+ </#list></#list>
+ case MAP: {
+ ComplexCopier.copy(reader, writer);
+ break;
+ }
+ case LIST: {
+ ComplexCopier.copy(reader, writer);
+ break;
+ }
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+ public void setSafe(int index, Nullable${name}Holder holder) {
+ setType(index, MinorType.${name?upper_case});
+ get${name}Vector().getMutator().setSafe(index, holder);
+ }
+
+ </#if>
+ </#list></#list>
+
+ public void setType(int index, MinorType type) {
+ typeVector.getMutator().setSafe(index, type.getNumber());
+ }
+
+ @Override
+ public void reset() { }
+
+ @Override
+ public void generateTestData(int values) { }
+ }
+}
[07/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/ComplexReaders.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/ComplexReaders.java b/exec/vector/src/main/codegen/templates/ComplexReaders.java
new file mode 100644
index 0000000..607b71d
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/ComplexReaders.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+import java.util.List;
+
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.complex.IndexHolder;
+import org.apache.drill.exec.vector.complex.writer.IntervalWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#list ["", "Repeated"] as mode>
+<#assign lowerName = minor.class?uncap_first />
+<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+<#assign name = mode + minor.class?cap_first />
+<#assign javaType = (minor.javaType!type.javaType) />
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+<#assign safeType=friendlyType />
+<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+
+<#assign hasFriendly = minor.friendlyType!"no" == "no" />
+
+<#list ["", "Nullable"] as nullMode>
+<#if (mode == "Repeated" && nullMode == "") || mode == "" >
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${nullMode}${name}ReaderImpl.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+@SuppressWarnings("unused")
+public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
+
+ private final ${nullMode}${name}Vector vector;
+
+ public ${nullMode}${name}ReaderImpl(${nullMode}${name}Vector vector){
+ super();
+ this.vector = vector;
+ }
+
+ public MajorType getType(){
+ return vector.getField().getType();
+ }
+
+ public MaterializedField getField(){
+ return vector.getField();
+ }
+
+ public boolean isSet(){
+ <#if nullMode == "Nullable">
+ return !vector.getAccessor().isNull(idx());
+ <#else>
+ return true;
+ </#if>
+ }
+
+
+
+
+ <#if mode == "Repeated">
+
+ public void copyAsValue(${minor.class?cap_first}Writer writer){
+ Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer;
+ impl.vector.copyFromSafe(idx(), impl.idx(), vector);
+ }
+
+ public void copyAsField(String name, MapWriter writer){
+ Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer.list(name).${lowerName}();
+ impl.vector.copyFromSafe(idx(), impl.idx(), vector);
+ }
+
+ public int size(){
+ return vector.getAccessor().getInnerValueCountAt(idx());
+ }
+
+ public void read(int arrayIndex, ${minor.class?cap_first}Holder h){
+ vector.getAccessor().get(idx(), arrayIndex, h);
+ }
+ public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h){
+ vector.getAccessor().get(idx(), arrayIndex, h);
+ }
+
+ public ${friendlyType} read${safeType}(int arrayIndex){
+ return vector.getAccessor().getSingleObject(idx(), arrayIndex);
+ }
+
+
+ public List<Object> readObject(){
+ return (List<Object>) (Object) vector.getAccessor().getObject(idx());
+ }
+
+ <#else>
+
+ public void copyAsValue(${minor.class?cap_first}Writer writer){
+ ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer;
+ impl.vector.copyFromSafe(idx(), impl.idx(), vector);
+ }
+
+ public void copyAsField(String name, MapWriter writer){
+ ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
+ impl.vector.copyFromSafe(idx(), impl.idx(), vector);
+ }
+
+ <#if nullMode != "Nullable">
+ public void read(${minor.class?cap_first}Holder h){
+ vector.getAccessor().get(idx(), h);
+ }
+ </#if>
+
+ public void read(Nullable${minor.class?cap_first}Holder h){
+ vector.getAccessor().get(idx(), h);
+ }
+
+ public ${friendlyType} read${safeType}(){
+ return vector.getAccessor().getObject(idx());
+ }
+
+ public void copyValue(FieldWriter w){
+
+ }
+
+ public Object readObject(){
+ return vector.getAccessor().getObject(idx());
+ }
+
+
+ </#if>
+}
+</#if>
+</#list>
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/reader/${name}Reader.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.reader;
+
+<#include "/@includes/vv_imports.ftl" />
+@SuppressWarnings("unused")
+public interface ${name}Reader extends BaseReader{
+
+ <#if mode == "Repeated">
+ public int size();
+ public void read(int arrayIndex, ${minor.class?cap_first}Holder h);
+ public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h);
+ public Object readObject(int arrayIndex);
+ public ${friendlyType} read${safeType}(int arrayIndex);
+ <#else>
+ public void read(${minor.class?cap_first}Holder h);
+ public void read(Nullable${minor.class?cap_first}Holder h);
+ public Object readObject();
+ public ${friendlyType} read${safeType}();
+ </#if>
+ public boolean isSet();
+ public void copyAsValue(${minor.class}Writer writer);
+ public void copyAsField(String name, ${minor.class}Writer writer);
+
+}
+
+
+
+</#list>
+</#list>
+</#list>
+
+
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/ComplexWriters.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/ComplexWriters.java b/exec/vector/src/main/codegen/templates/ComplexWriters.java
new file mode 100644
index 0000000..71a7f9a
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/ComplexWriters.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+import java.util.Vector;
+
+import org.apache.drill.exec.vector.AllocationHelper;
+import org.apache.drill.exec.vector.ValueVector.Mutator;
+import org.apache.drill.exec.vector.complex.IndexHolder;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#list ["", "Nullable", "Repeated"] as mode>
+<#assign name = mode + minor.class?cap_first />
+<#assign eName = name />
+<#assign javaType = (minor.javaType!type.javaType) />
+<#assign fields = minor.fields!type.fields />
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${eName}WriterImpl.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using FreeMarker on the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ${eName}WriterImpl extends AbstractFieldWriter {
+
+ private final ${name}Vector.Mutator mutator;
+ final ${name}Vector vector;
+
+ public ${eName}WriterImpl(${name}Vector vector, AbstractFieldWriter parent) {
+ super(parent);
+ this.mutator = vector.getMutator();
+ this.vector = vector;
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return vector.getField();
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return vector.getValueCapacity();
+ }
+
+ @Override
+ public void allocate() {
+ vector.allocateNew();
+ }
+
+ @Override
+ public void close() {
+ vector.close();
+ }
+
+ @Override
+ public void clear() {
+ vector.clear();
+ }
+
+ @Override
+ protected int idx() {
+ return super.idx();
+ }
+
+ <#if mode == "Repeated">
+
+ public void write(${minor.class?cap_first}Holder h) {
+ mutator.addSafe(idx(), h);
+ vector.getMutator().setValueCount(idx()+1);
+ }
+
+ public void write(Nullable${minor.class?cap_first}Holder h) {
+ mutator.addSafe(idx(), h);
+ vector.getMutator().setValueCount(idx()+1);
+ }
+
+ <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+ public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ vector.getMutator().setValueCount(idx()+1);
+ }
+ </#if>
+
+ public void setPosition(int idx) {
+ super.setPosition(idx);
+ mutator.startNewValue(idx);
+ }
+
+
+ <#else>
+
+ public void write(${minor.class}Holder h) {
+ mutator.setSafe(idx(), h);
+ vector.getMutator().setValueCount(idx()+1);
+ }
+
+ public void write(Nullable${minor.class}Holder h) {
+ mutator.setSafe(idx(), h);
+ vector.getMutator().setValueCount(idx()+1);
+ }
+
+ <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+ public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ mutator.setSafe(idx(), <#if mode == "Nullable">1, </#if><#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ vector.getMutator().setValueCount(idx()+1);
+ }
+
+ <#if mode == "Nullable">
+
+ public void writeNull() {
+ mutator.setNull(idx());
+ vector.getMutator().setValueCount(idx()+1);
+ }
+ </#if>
+ </#if>
+ </#if>
+}
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/writer/${eName}Writer.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.writer;
+
+<#include "/@includes/vv_imports.ftl" />
+@SuppressWarnings("unused")
+public interface ${eName}Writer extends BaseWriter {
+ public void write(${minor.class}Holder h);
+
+ <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+ public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
+ </#if>
+}
+
+</#list>
+</#list>
+</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
new file mode 100644
index 0000000..d9d5fae
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -0,0 +1,815 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+
+<#if type.major == "Fixed">
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed
+ * by position, starting from the logical start of the vector. Values should be pushed onto the
+ * vector sequentially, but may be randomly accessed.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+ private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
+ private final Accessor accessor = new Accessor();
+ private final Mutator mutator = new Mutator();
+
+ private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
+ private int allocationMonitor = 0;
+
+ public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ @Override
+ public FieldReader getReader(){
+ return reader;
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+ return valueCount * ${type.width};
+ }
+
+ @Override
+ public int getValueCapacity(){
+ return (int) (data.capacity() *1.0 / ${type.width});
+ }
+
+ @Override
+ public Accessor getAccessor(){
+ return accessor;
+ }
+
+ @Override
+ public Mutator getMutator(){
+ return mutator;
+ }
+
+ @Override
+ public void setInitialCapacity(final int valueCount) {
+ final long size = 1L * valueCount * ${type.width};
+ if (size > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+ }
+ allocationSizeInBytes = (int)size;
+ }
+
+ @Override
+ public void allocateNew() {
+ if(!allocateNewSafe()){
+ throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
+ }
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ long curAllocationSize = allocationSizeInBytes;
+ if (allocationMonitor > 10) {
+ curAllocationSize = Math.max(8, curAllocationSize / 2);
+ allocationMonitor = 0;
+ } else if (allocationMonitor < -2) {
+ curAllocationSize = allocationSizeInBytes * 2L;
+ allocationMonitor = 0;
+ }
+
+ try{
+ allocateBytes(curAllocationSize);
+ } catch (DrillRuntimeException ex) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Allocate a new buffer that supports setting at least the provided number of values. May actually be sized bigger
+ * depending on underlying buffer rounding size. Must be called prior to using the ValueVector.
+ *
+ * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
+ *
+ * @param valueCount
+ * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer
+ */
+ @Override
+ public void allocateNew(final int valueCount) {
+ allocateBytes(valueCount * ${type.width});
+ }
+
+ @Override
+ public void reset() {
+ allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+ allocationMonitor = 0;
+ zeroVector();
+ super.reset();
+ }
+
+ private void allocateBytes(final long size) {
+ if (size > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+ }
+
+ final int curSize = (int)size;
+ clear();
+ data = allocator.buffer(curSize);
+ data.readerIndex(0);
+ allocationSizeInBytes = curSize;
+ }
+
+/**
+ * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+ *
+ * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer
+ */
+ public void reAlloc() {
+ final long newAllocationSize = allocationSizeInBytes * 2L;
+ if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
+ }
+
+ logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize);
+ final DrillBuf newBuf = allocator.buffer((int)newAllocationSize);
+ newBuf.setBytes(0, data, 0, data.capacity());
+ final int halfNewCapacity = newBuf.capacity() / 2;
+ newBuf.setZero(halfNewCapacity, halfNewCapacity);
+ newBuf.writerIndex(data.writerIndex());
+ data.release(1);
+ data = newBuf;
+ allocationSizeInBytes = (int)newAllocationSize;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void zeroVector() {
+ data.setZero(0, data.capacity());
+ }
+
+ @Override
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ Preconditions.checkArgument(this.field.matches(metadata), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
+ final int actualLength = metadata.getBufferLength();
+ final int valueCount = metadata.getValueCount();
+ final int expectedLength = valueCount * ${type.width};
+ assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength);
+
+ clear();
+ if (data != null) {
+ data.release(1);
+ }
+ data = buffer.slice(0, actualLength);
+ data.retain(1);
+ data.writerIndex(actualLength);
+ }
+
+ public TransferPair getTransferPair(){
+ return new TransferImpl(getField());
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref){
+ return new TransferImpl(getField().withPath(ref));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new TransferImpl((${minor.class}Vector) to);
+ }
+
+ public void transferTo(${minor.class}Vector target){
+ target.clear();
+ target.data = data;
+ target.data.retain(1);
+ target.data.writerIndex(data.writerIndex());
+ clear();
+ }
+
+ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
+ final int startPoint = startIndex * ${type.width};
+ final int sliceLength = length * ${type.width};
+ target.clear();
+ target.data = data.slice(startPoint, sliceLength);
+ target.data.retain(1);
+ target.data.writerIndex(sliceLength);
+ }
+
+ private class TransferImpl implements TransferPair{
+ private ${minor.class}Vector to;
+
+ public TransferImpl(MaterializedField field){
+ to = new ${minor.class}Vector(field, allocator);
+ }
+
+ public TransferImpl(${minor.class}Vector to) {
+ this.to = to;
+ }
+
+ @Override
+ public ${minor.class}Vector getTo(){
+ return to;
+ }
+
+ @Override
+ public void transfer(){
+ transferTo(to);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ splitAndTransferTo(startIndex, length, to);
+ }
+
+ @Override
+ public void copyValueSafe(int fromIndex, int toIndex) {
+ to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
+ }
+ }
+
+ public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+ <#if (type.width > 8)>
+ from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
+ <#else> <#-- type.width <= 8 -->
+ data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width},
+ from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width})
+ );
+ </#if> <#-- type.width -->
+ }
+
+ public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+ while(thisIndex >= getValueCapacity()) {
+ reAlloc();
+ }
+ copyFrom(fromIndex, thisIndex, from);
+ }
+
+ public void decrementAllocationMonitor() {
+ if (allocationMonitor > 0) {
+ allocationMonitor = 0;
+ }
+ --allocationMonitor;
+ }
+
+ private void incrementAllocationMonitor() {
+ ++allocationMonitor;
+ }
+
+ public final class Accessor extends BaseDataValueVector.BaseAccessor {
+ @Override
+ public int getValueCount() {
+ return data.writerIndex() / ${type.width};
+ }
+
+ @Override
+ public boolean isNull(int index){
+ return false;
+ }
+
+ <#if (type.width > 8)>
+
+ public ${minor.javaType!type.javaType} get(int index) {
+ return data.slice(index * ${type.width}, ${type.width});
+ }
+
+ <#if (minor.class == "Interval")>
+ public void get(int index, ${minor.class}Holder holder){
+
+ final int offsetIndex = index * ${type.width};
+ holder.months = data.getInt(offsetIndex);
+ holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
+ holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+ }
+
+ public void get(int index, Nullable${minor.class}Holder holder){
+ final int offsetIndex = index * ${type.width};
+ holder.isSet = 1;
+ holder.months = data.getInt(offsetIndex);
+ holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
+ holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+ }
+
+ @Override
+ public ${friendlyType} getObject(int index) {
+ final int offsetIndex = index * ${type.width};
+ final int months = data.getInt(offsetIndex);
+ final int days = data.getInt(offsetIndex + ${minor.daysOffset});
+ final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+ final Period p = new Period();
+ return p.plusMonths(months).plusDays(days).plusMillis(millis);
+ }
+
+ public StringBuilder getAsStringBuilder(int index) {
+
+ final int offsetIndex = index * ${type.width};
+
+ int months = data.getInt(offsetIndex);
+ final int days = data.getInt(offsetIndex + ${minor.daysOffset});
+ int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+
+ final int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
+ months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
+
+ final int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
+ millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
+
+ final int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
+ millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
+
+ final long seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
+ millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
+
+ final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
+ final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
+ final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
+
+
+ return(new StringBuilder().
+ append(years).append(yearString).
+ append(months).append(monthString).
+ append(days).append(dayString).
+ append(hours).append(":").
+ append(minutes).append(":").
+ append(seconds).append(".").
+ append(millis));
+ }
+
+ <#elseif (minor.class == "IntervalDay")>
+ public void get(int index, ${minor.class}Holder holder){
+
+ final int offsetIndex = index * ${type.width};
+ holder.days = data.getInt(offsetIndex);
+ holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+ }
+
+ public void get(int index, Nullable${minor.class}Holder holder){
+ final int offsetIndex = index * ${type.width};
+ holder.isSet = 1;
+ holder.days = data.getInt(offsetIndex);
+ holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+ }
+
+ @Override
+ public ${friendlyType} getObject(int index) {
+ final int offsetIndex = index * ${type.width};
+ final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+ final int days = data.getInt(offsetIndex);
+ final Period p = new Period();
+ return p.plusDays(days).plusMillis(millis);
+ }
+
+
+ public StringBuilder getAsStringBuilder(int index) {
+ final int offsetIndex = index * ${type.width};
+
+ int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
+ final int days = data.getInt(offsetIndex);
+
+ final int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
+ millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis);
+
+ final int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
+ millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis);
+
+ final int seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
+ millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis);
+
+ final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
+
+ return(new StringBuilder().
+ append(days).append(dayString).
+ append(hours).append(":").
+ append(minutes).append(":").
+ append(seconds).append(".").
+ append(millis));
+ }
+
+ <#elseif (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+
+ public void get(int index, ${minor.class}Holder holder) {
+ holder.start = index * ${type.width};
+ holder.buffer = data;
+ holder.scale = getField().getScale();
+ holder.precision = getField().getPrecision();
+ }
+
+ public void get(int index, Nullable${minor.class}Holder holder) {
+ holder.isSet = 1;
+ holder.start = index * ${type.width};
+ holder.buffer = data;
+ holder.scale = getField().getScale();
+ holder.precision = getField().getPrecision();
+ }
+
+ @Override
+ public ${friendlyType} getObject(int index) {
+ <#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")>
+ // Get the BigDecimal object
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
+ <#else>
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
+ </#if>
+ }
+
+ <#else>
+ public void get(int index, ${minor.class}Holder holder){
+ holder.buffer = data;
+ holder.start = index * ${type.width};
+ }
+
+ public void get(int index, Nullable${minor.class}Holder holder){
+ holder.isSet = 1;
+ holder.buffer = data;
+ holder.start = index * ${type.width};
+ }
+
+ @Override
+ public ${friendlyType} getObject(int index) {
+ return data.slice(index * ${type.width}, ${type.width})
+ }
+
+ </#if>
+ <#else> <#-- type.width <= 8 -->
+
+ public ${minor.javaType!type.javaType} get(int index) {
+ return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+ }
+
+ <#if type.width == 4>
+ public long getTwoAsLong(int index) {
+ return data.getLong(index * ${type.width});
+ }
+
+ </#if>
+
+ <#if minor.class == "Date">
+ @Override
+ public ${friendlyType} getObject(int index) {
+ org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+ date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+ return date;
+ }
+
+ <#elseif minor.class == "TimeStamp">
+ @Override
+ public ${friendlyType} getObject(int index) {
+ org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+ date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+ return date;
+ }
+
+ <#elseif minor.class == "IntervalYear">
+ @Override
+ public ${friendlyType} getObject(int index) {
+
+ final int value = get(index);
+
+ final int years = (value / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
+ final int months = (value % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
+ final Period p = new Period();
+ return p.plusYears(years).plusMonths(months);
+ }
+
+ public StringBuilder getAsStringBuilder(int index) {
+
+ int months = data.getInt(index);
+
+ final int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
+ months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
+
+ final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
+ final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
+
+ return(new StringBuilder().
+ append(years).append(yearString).
+ append(months).append(monthString));
+ }
+
+ <#elseif minor.class == "Time">
+ @Override
+ public DateTime getObject(int index) {
+
+ org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+ time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+ return time;
+ }
+
+ <#elseif minor.class == "Decimal9" || minor.class == "Decimal18">
+ @Override
+ public ${friendlyType} getObject(int index) {
+
+ final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value());
+ return new BigDecimal(value, getField().getScale());
+ }
+
+ <#else>
+ @Override
+ public ${friendlyType} getObject(int index) {
+ return get(index);
+ }
+ public ${minor.javaType!type.javaType} getPrimitiveObject(int index) {
+ return get(index);
+ }
+ </#if>
+
+ public void get(int index, ${minor.class}Holder holder){
+ <#if minor.class.startsWith("Decimal")>
+ holder.scale = getField().getScale();
+ holder.precision = getField().getPrecision();
+ </#if>
+
+ holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+ }
+
+ public void get(int index, Nullable${minor.class}Holder holder){
+ holder.isSet = 1;
+ holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+ }
+
+
+ </#if> <#-- type.width -->
+ }
+
+ /**
+ * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the
+ * vector are accessed by position from the logical start of the vector. Values should be pushed
+ * onto the vector sequentially, but may be randomly accessed.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public final class Mutator extends BaseDataValueVector.BaseMutator {
+
+ private Mutator(){};
+ /**
+ * Set the element at the given index to the given value. Note that widths smaller than
+ * 32 bits are handled by the DrillBuf interface.
+ *
+ * @param index position of the bit to set
+ * @param value value to set
+ */
+ <#if (type.width > 8)>
+ public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ data.setBytes(index * ${type.width}, value, 0, ${type.width});
+ }
+
+ public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ data.setBytes(index * ${type.width}, value, 0, ${type.width});
+ }
+
+ <#if (minor.class == "Interval")>
+ public void set(int index, int months, int days, int milliseconds){
+ final int offsetIndex = index * ${type.width};
+ data.setInt(offsetIndex, months);
+ data.setInt((offsetIndex + ${minor.daysOffset}), days);
+ data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
+ }
+
+ protected void set(int index, ${minor.class}Holder holder){
+ set(index, holder.months, holder.days, holder.milliseconds);
+ }
+
+ protected void set(int index, Nullable${minor.class}Holder holder){
+ set(index, holder.months, holder.days, holder.milliseconds);
+ }
+
+ public void setSafe(int index, int months, int days, int milliseconds){
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, months, days, milliseconds);
+ }
+
+ public void setSafe(int index, Nullable${minor.class}Holder holder){
+ setSafe(index, holder.months, holder.days, holder.milliseconds);
+ }
+
+ public void setSafe(int index, ${minor.class}Holder holder){
+ setSafe(index, holder.months, holder.days, holder.milliseconds);
+ }
+
+ <#elseif (minor.class == "IntervalDay")>
+ public void set(int index, int days, int milliseconds){
+ final int offsetIndex = index * ${type.width};
+ data.setInt(offsetIndex, days);
+ data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
+ }
+
+ protected void set(int index, ${minor.class}Holder holder){
+ set(index, holder.days, holder.milliseconds);
+ }
+ protected void set(int index, Nullable${minor.class}Holder holder){
+ set(index, holder.days, holder.milliseconds);
+ }
+
+ public void setSafe(int index, int days, int milliseconds){
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, days, milliseconds);
+ }
+
+ public void setSafe(int index, ${minor.class}Holder holder){
+ setSafe(index, holder.days, holder.milliseconds);
+ }
+
+ public void setSafe(int index, Nullable${minor.class}Holder holder){
+ setSafe(index, holder.days, holder.milliseconds);
+ }
+
+ <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+
+ public void set(int index, ${minor.class}Holder holder){
+ set(index, holder.start, holder.buffer);
+ }
+
+ void set(int index, Nullable${minor.class}Holder holder){
+ set(index, holder.start, holder.buffer);
+ }
+
+ public void setSafe(int index, Nullable${minor.class}Holder holder){
+ setSafe(index, holder.start, holder.buffer);
+ }
+ public void setSafe(int index, ${minor.class}Holder holder){
+ setSafe(index, holder.start, holder.buffer);
+ }
+
+ public void setSafe(int index, int start, DrillBuf buffer){
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, start, buffer);
+ }
+
+ public void set(int index, int start, DrillBuf buffer){
+ data.setBytes(index * ${type.width}, buffer, start, ${type.width});
+ }
+
+ <#else>
+
+ protected void set(int index, ${minor.class}Holder holder){
+ set(index, holder.start, holder.buffer);
+ }
+
+ public void set(int index, Nullable${minor.class}Holder holder){
+ set(index, holder.start, holder.buffer);
+ }
+
+ public void set(int index, int start, DrillBuf buffer){
+ data.setBytes(index * ${type.width}, buffer, start, ${type.width});
+ }
+
+ public void setSafe(int index, ${minor.class}Holder holder){
+ setSafe(index, holder.start, holder.buffer);
+ }
+ public void setSafe(int index, Nullable${minor.class}Holder holder){
+ setSafe(index, holder.start, holder.buffer);
+ }
+
+ public void setSafe(int index, int start, DrillBuf buffer){
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, holder);
+ }
+
+ public void set(int index, Nullable${minor.class}Holder holder){
+ data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
+ }
+ </#if>
+
+ @Override
+ public void generateTestData(int count) {
+ setValueCount(count);
+ boolean even = true;
+ final int valueCount = getAccessor().getValueCount();
+ for(int i = 0; i < valueCount; i++, even = !even) {
+ final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE;
+ for(int w = 0; w < ${type.width}; w++){
+ data.setByte(i + w, b);
+ }
+ }
+ }
+
+ <#else> <#-- type.width <= 8 -->
+ public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
+ }
+
+ public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, value);
+ }
+
+ protected void set(int index, ${minor.class}Holder holder){
+ data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
+ }
+
+ public void setSafe(int index, ${minor.class}Holder holder){
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, holder);
+ }
+
+ protected void set(int index, Nullable${minor.class}Holder holder){
+ data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
+ }
+
+ public void setSafe(int index, Nullable${minor.class}Holder holder){
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, holder);
+ }
+
+ @Override
+ public void generateTestData(int size) {
+ setValueCount(size);
+ boolean even = true;
+ final int valueCount = getAccessor().getValueCount();
+ for(int i = 0; i < valueCount; i++, even = !even) {
+ if(even){
+ set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE);
+ }else{
+ set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE);
+ }
+ }
+ }
+
+ public void generateTestDataAlt(int size) {
+ setValueCount(size);
+ boolean even = true;
+ final int valueCount = getAccessor().getValueCount();
+ for(int i = 0; i < valueCount; i++, even = !even) {
+ if(even){
+ set(i, (${(minor.javaType!type.javaType)}) 1);
+ }else{
+ set(i, (${(minor.javaType!type.javaType)}) 0);
+ }
+ }
+ }
+
+ </#if> <#-- type.width -->
+
+ @Override
+ public void setValueCount(int valueCount) {
+ final int currentValueCapacity = getValueCapacity();
+ final int idx = (${type.width} * valueCount);
+ while(valueCount > getValueCapacity()) {
+ reAlloc();
+ }
+ if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
+ incrementAllocationMonitor();
+ } else if (allocationMonitor > 0) {
+ allocationMonitor = 0;
+ }
+ VectorTrimmer.trim(data, idx);
+ data.writerIndex(valueCount * ${type.width});
+ }
+ }
+}
+
+</#if> <#-- type.major -->
+</#list>
+</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/HolderReaderImpl.java b/exec/vector/src/main/codegen/templates/HolderReaderImpl.java
new file mode 100644
index 0000000..d0f5c4a
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#list ["", "Nullable", "Repeated"] as holderMode>
+<#assign nullMode = holderMode />
+<#if holderMode == "Repeated"><#assign nullMode = "Nullable" /></#if>
+
+<#assign lowerName = minor.class?uncap_first />
+<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+<#assign name = minor.class?cap_first />
+<#assign javaType = (minor.javaType!type.javaType) />
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+<#assign safeType=friendlyType />
+<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+<#assign fields = minor.fields!type.fields />
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${holderMode}${name}HolderReaderImpl.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.hadoop.io.Text;
+import org.joda.time.Period;
+
+// Source code generated using FreeMarker template ${.template_name}
+
+@SuppressWarnings("unused")
+public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
+
+ private ${nullMode}${name}Holder holder;
+<#if holderMode == "Repeated" >
+ private int index = -1;
+ private ${holderMode}${name}Holder repeatedHolder;
+</#if>
+
+ public ${holderMode}${name}HolderReaderImpl(${holderMode}${name}Holder holder) {
+<#if holderMode == "Repeated" >
+ this.holder = new ${nullMode}${name}Holder();
+ this.repeatedHolder = holder;
+<#else>
+ this.holder = holder;
+</#if>
+ }
+
+ @Override
+ public int size() {
+<#if holderMode == "Repeated">
+ return repeatedHolder.end - repeatedHolder.start;
+<#else>
+ throw new UnsupportedOperationException("You can't call size on a Holder value reader.");
+</#if>
+ }
+
+ @Override
+ public boolean next() {
+<#if holderMode == "Repeated">
+ if(index + 1 < repeatedHolder.end) {
+ index++;
+ repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
+ return true;
+ } else {
+ return false;
+ }
+<#else>
+ throw new UnsupportedOperationException("You can't call next on a single value reader.");
+</#if>
+
+ }
+
+ @Override
+ public void setPosition(int index) {
+ throw new UnsupportedOperationException("You can't call next on a single value reader.");
+ }
+
+ @Override
+ public MajorType getType() {
+<#if holderMode == "Repeated">
+ return this.repeatedHolder.TYPE;
+<#else>
+ return this.holder.TYPE;
+</#if>
+ }
+
+ @Override
+ public boolean isSet() {
+ <#if holderMode == "Repeated">
+ return this.repeatedHolder.end!=this.repeatedHolder.start;
+ <#elseif nullMode == "Nullable">
+ return this.holder.isSet == 1;
+ <#else>
+ return true;
+ </#if>
+
+ }
+
+<#if holderMode != "Repeated">
+@Override
+ public void read(${name}Holder h) {
+ <#list fields as field>
+ h.${field.name} = holder.${field.name};
+ </#list>
+ }
+
+ @Override
+ public void read(Nullable${name}Holder h) {
+ <#list fields as field>
+ h.${field.name} = holder.${field.name};
+ </#list>
+ h.isSet = isSet() ? 1 : 0;
+ }
+</#if>
+
+<#if holderMode == "Repeated">
+ @Override
+ public ${friendlyType} read${safeType}(int index){
+ repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
+ ${friendlyType} value = read${safeType}();
+ if (this.index > -1) {
+ repeatedHolder.vector.getAccessor().get(repeatedHolder.start + this.index, holder);
+ }
+ return value;
+ }
+</#if>
+
+ @Override
+ public ${friendlyType} read${safeType}(){
+<#if nullMode == "Nullable">
+ if (!isSet()) {
+ return null;
+ }
+</#if>
+
+<#if type.major == "VarLen">
+
+ int length = holder.end - holder.start;
+ byte[] value = new byte [length];
+ holder.buffer.getBytes(holder.start, value, 0, length);
+
+<#if minor.class == "VarBinary">
+ return value;
+<#elseif minor.class == "Var16Char">
+ return new String(value);
+<#elseif minor.class == "VarChar">
+ Text text = new Text();
+ text.set(value);
+ return text;
+</#if>
+
+<#elseif minor.class == "Interval">
+ Period p = new Period();
+ return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
+
+<#elseif minor.class == "IntervalDay">
+ Period p = new Period();
+ return p.plusDays(holder.days).plusMillis(holder.milliseconds);
+
+<#elseif minor.class == "Decimal9" ||
+ minor.class == "Decimal18" >
+ BigInteger value = BigInteger.valueOf(holder.value);
+ return new BigDecimal(value, holder.scale);
+
+<#elseif minor.class == "Decimal28Dense" ||
+ minor.class == "Decimal38Dense">
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
+ holder.start,
+ holder.nDecimalDigits,
+ holder.scale,
+ holder.maxPrecision,
+ holder.WIDTH);
+
+<#elseif minor.class == "Decimal28Sparse" ||
+ minor.class == "Decimal38Sparse">
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
+ holder.start,
+ holder.nDecimalDigits,
+ holder.scale);
+
+<#elseif minor.class == "Bit" >
+ return new Boolean(holder.value != 0);
+<#else>
+ ${friendlyType} value = new ${friendlyType}(this.holder.value);
+ return value;
+</#if>
+
+ }
+
+ @Override
+ public Object readObject() {
+<#if holderMode == "Repeated" >
+ List<Object> valList = Lists.newArrayList();
+ for (int i = repeatedHolder.start; i < repeatedHolder.end; i++) {
+ valList.add(repeatedHolder.vector.getAccessor().getObject(i));
+ }
+ return valList;
+<#else>
+ return readSingleObject();
+</#if>
+ }
+
+ private Object readSingleObject() {
+<#if nullMode == "Nullable">
+ if (!isSet()) {
+ return null;
+ }
+</#if>
+
+<#if type.major == "VarLen">
+ int length = holder.end - holder.start;
+ byte[] value = new byte [length];
+ holder.buffer.getBytes(holder.start, value, 0, length);
+
+<#if minor.class == "VarBinary">
+ return value;
+<#elseif minor.class == "Var16Char">
+ return new String(value);
+<#elseif minor.class == "VarChar">
+ Text text = new Text();
+ text.set(value);
+ return text;
+</#if>
+
+<#elseif minor.class == "Interval">
+ Period p = new Period();
+ return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
+
+<#elseif minor.class == "IntervalDay">
+ Period p = new Period();
+ return p.plusDays(holder.days).plusMillis(holder.milliseconds);
+
+<#elseif minor.class == "Decimal9" ||
+ minor.class == "Decimal18" >
+ BigInteger value = BigInteger.valueOf(holder.value);
+ return new BigDecimal(value, holder.scale);
+
+<#elseif minor.class == "Decimal28Dense" ||
+ minor.class == "Decimal38Dense">
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
+ holder.start,
+ holder.nDecimalDigits,
+ holder.scale,
+ holder.maxPrecision,
+ holder.WIDTH);
+
+<#elseif minor.class == "Decimal28Sparse" ||
+ minor.class == "Decimal38Sparse">
+ return org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
+ holder.start,
+ holder.nDecimalDigits,
+ holder.scale);
+
+<#elseif minor.class == "Bit" >
+ return new Boolean(holder.value != 0);
+<#else>
+ ${friendlyType} value = new ${friendlyType}(this.holder.value);
+ return value;
+</#if>
+ }
+
+<#if holderMode != "Repeated" && nullMode != "Nullable">
+ public void copyAsValue(${minor.class?cap_first}Writer writer){
+ writer.write(holder);
+ }
+</#if>
+}
+
+</#list>
+</#list>
+</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/ListWriters.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/ListWriters.java b/exec/vector/src/main/codegen/templates/ListWriters.java
new file mode 100644
index 0000000..16d41ec
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/ListWriters.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+
+<#list ["Single", "Repeated"] as mode>
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${mode}ListWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+<#if mode == "Single">
+ <#assign containerClass = "AbstractContainerVector" />
+ <#assign index = "idx()">
+<#else>
+ <#assign containerClass = "RepeatedListVector" />
+ <#assign index = "currentChildIndex">
+</#if>
+
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using FreeMarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ${mode}ListWriter extends AbstractFieldWriter {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${mode}ListWriter.class);
+
+ static enum Mode { INIT, IN_MAP, IN_LIST <#list vv.types as type><#list type.minor as minor>, IN_${minor.class?upper_case}</#list></#list> }
+
+ private final String name;
+ protected final ${containerClass} container;
+ private Mode mode = Mode.INIT;
+ private FieldWriter writer;
+ protected RepeatedValueVector innerVector;
+
+ <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
+ public ${mode}ListWriter(String name, ${containerClass} container, FieldWriter parent){
+ super(parent);
+ this.name = name;
+ this.container = container;
+ }
+
+ public ${mode}ListWriter(${containerClass} container, FieldWriter parent){
+ super(parent);
+ this.name = null;
+ this.container = container;
+ }
+
+ @Override
+ public void allocate() {
+ if(writer != null) {
+ writer.allocate();
+ }
+
+ <#if mode == "Repeated">
+ container.allocateNew();
+ </#if>
+ }
+
+ @Override
+ public void clear() {
+ if (writer != null) {
+ writer.clear();
+ }
+ }
+
+ @Override
+ public void close() {
+ clear();
+ container.close();
+ if (innerVector != null) {
+ innerVector.close();
+ }
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return innerVector == null ? 0 : innerVector.getValueCapacity();
+ }
+
+ public void setValueCount(int count){
+ if(innerVector != null) innerVector.getMutator().setValueCount(count);
+ }
+
+ @Override
+ public MapWriter map() {
+ switch(mode) {
+ case INIT:
+ int vectorCount = container.size();
+ final RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class);
+ innerVector = vector;
+ writer = new RepeatedMapWriter(vector, this);
+ if(vectorCount != container.size()) {
+ writer.allocate();
+ }
+ writer.setPosition(${index});
+ mode = Mode.IN_MAP;
+ return writer;
+ case IN_MAP:
+ return writer;
+ }
+
+ throw UserException.unsupportedError().message(getUnsupportedErrorMsg("MAP", mode.name())).build(logger);
+
+ }
+
+ @Override
+ public ListWriter list() {
+ switch(mode) {
+ case INIT:
+ final int vectorCount = container.size();
+ final RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class);
+ innerVector = vector;
+ writer = new RepeatedListWriter(null, vector, this);
+ if(vectorCount != container.size()) {
+ writer.allocate();
+ }
+ writer.setPosition(${index});
+ mode = Mode.IN_LIST;
+ return writer;
+ case IN_LIST:
+ return writer;
+ }
+
+ throw UserException.unsupportedError().message(getUnsupportedErrorMsg("LIST", mode.name())).build(logger);
+
+ }
+
+ <#list vv.types as type><#list type.minor as minor>
+ <#assign lowerName = minor.class?uncap_first />
+ <#assign upperName = minor.class?upper_case />
+ <#assign capName = minor.class?cap_first />
+ <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+
+ private static final MajorType ${upperName}_TYPE = Types.repeated(MinorType.${upperName});
+
+ @Override
+ public ${capName}Writer ${lowerName}() {
+ switch(mode) {
+ case INIT:
+ final int vectorCount = container.size();
+ final Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class);
+ innerVector = vector;
+ writer = new Repeated${capName}WriterImpl(vector, this);
+ if(vectorCount != container.size()) {
+ writer.allocate();
+ }
+ writer.setPosition(${index});
+ mode = Mode.IN_${upperName};
+ return writer;
+ case IN_${upperName}:
+ return writer;
+ }
+
+ throw UserException.unsupportedError().message(getUnsupportedErrorMsg("${upperName}", mode.name())).build(logger);
+
+ }
+ </#list></#list>
+
+ public MaterializedField getField() {
+ return container.getField();
+ }
+
+ <#if mode == "Repeated">
+
+ public void startList() {
+ final RepeatedListVector list = (RepeatedListVector) container;
+ final RepeatedListVector.RepeatedMutator mutator = list.getMutator();
+
+ // make sure that the current vector can support the end position of this list.
+ if(container.getValueCapacity() <= idx()) {
+ mutator.setValueCount(idx()+1);
+ }
+
+ // update the repeated vector to state that there is current+1 objects.
+ final RepeatedListHolder h = new RepeatedListHolder();
+ list.getAccessor().get(idx(), h);
+ if (h.start >= h.end) {
+ mutator.startNewValue(idx());
+ }
+ currentChildIndex = container.getMutator().add(idx());
+ if(writer != null) {
+ writer.setPosition(currentChildIndex);
+ }
+ }
+
+ public void endList() {
+ // noop, we initialize state at start rather than end.
+ }
+ <#else>
+
+ public void setPosition(int index) {
+ super.setPosition(index);
+ if(writer != null) {
+ writer.setPosition(index);
+ }
+ }
+
+ public void startList() {
+ // noop
+ }
+
+ public void endList() {
+ // noop
+ }
+ </#if>
+
+ private String getUnsupportedErrorMsg(String expected, String found) {
+ final String f = found.substring(3);
+ return String.format("In a list of type %s, encountered a value of type %s. "+
+ "Drill does not support lists of different types.",
+ f, expected
+ );
+ }
+}
+</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/MapWriters.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/MapWriters.java b/exec/vector/src/main/codegen/templates/MapWriters.java
new file mode 100644
index 0000000..93f2edb
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/MapWriters.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<#list ["Single", "Repeated"] as mode>
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${mode}MapWriter.java" />
+<#if mode == "Single">
+<#assign containerClass = "MapVector" />
+<#assign index = "idx()">
+<#else>
+<#assign containerClass = "RepeatedMapVector" />
+<#assign index = "currentChildIndex">
+</#if>
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+import java.util.Map;
+
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
+import org.apache.drill.exec.vector.AllocationHelper;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
+
+import com.google.common.collect.Maps;
+
+/*
+ * This class is generated using FreeMarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class ${mode}MapWriter extends AbstractFieldWriter {
+
+ protected final ${containerClass} container;
+ private final Map<String, FieldWriter> fields = Maps.newHashMap();
+ <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
+
+ private final boolean unionEnabled;
+
+ public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled) {
+ super(parent);
+ this.container = container;
+ this.unionEnabled = unionEnabled;
+ }
+
+ public ${mode}MapWriter(${containerClass} container, FieldWriter parent) {
+ this(container, parent, false);
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return container.getValueCapacity();
+ }
+
+ @Override
+ public boolean isEmptyMap() {
+ return 0 == container.size();
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return container.getField();
+ }
+
+ @Override
+ public MapWriter map(String name) {
+ FieldWriter writer = fields.get(name.toLowerCase());
+ if(writer == null){
+ int vectorCount=container.size();
+ MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class);
+ if(!unionEnabled){
+ writer = new SingleMapWriter(vector, this);
+ } else {
+ writer = new PromotableWriter(vector, container);
+ }
+ if(vectorCount != container.size()) {
+ writer.allocate();
+ }
+ writer.setPosition(${index});
+ fields.put(name.toLowerCase(), writer);
+ }
+ return writer;
+ }
+
+ @Override
+ public void close() throws Exception {
+ clear();
+ container.close();
+ }
+
+ @Override
+ public void allocate() {
+ container.allocateNew();
+ for(final FieldWriter w : fields.values()) {
+ w.allocate();
+ }
+ }
+
+ @Override
+ public void clear() {
+ container.clear();
+ for(final FieldWriter w : fields.values()) {
+ w.clear();
+ }
+ }
+
+ @Override
+ public ListWriter list(String name) {
+ FieldWriter writer = fields.get(name.toLowerCase());
+ int vectorCount = container.size();
+ if(writer == null) {
+ if (!unionEnabled){
+ writer = new SingleListWriter(name,container,this);
+ } else{
+ writer = new PromotableWriter(container.addOrGet(name, Types.optional(MinorType.LIST), ListVector.class), container);
+ }
+ if (container.size() > vectorCount) {
+ writer.allocate();
+ }
+ writer.setPosition(${index});
+ fields.put(name.toLowerCase(), writer);
+ }
+ return writer;
+ }
+
+ <#if mode == "Repeated">
+ public void start() {
+ // update the repeated vector to state that there is current+1 objects.
+ final RepeatedMapHolder h = new RepeatedMapHolder();
+ final RepeatedMapVector map = (RepeatedMapVector) container;
+ final RepeatedMapVector.Mutator mutator = map.getMutator();
+
+ // Make sure that the current vector can support the end position of this list.
+ if(container.getValueCapacity() <= idx()) {
+ mutator.setValueCount(idx()+1);
+ }
+
+ map.getAccessor().get(idx(), h);
+ if (h.start >= h.end) {
+ container.getMutator().startNewValue(idx());
+ }
+ currentChildIndex = container.getMutator().add(idx());
+ for(final FieldWriter w : fields.values()) {
+ w.setPosition(currentChildIndex);
+ }
+ }
+
+
+ public void end() {
+ // noop
+ }
+ <#else>
+
+ public void setValueCount(int count) {
+ container.getMutator().setValueCount(count);
+ }
+
+ @Override
+ public void setPosition(int index) {
+ super.setPosition(index);
+ for(final FieldWriter w: fields.values()) {
+ w.setPosition(index);
+ }
+ }
+
+ @Override
+ public void start() {
+ }
+
+ @Override
+ public void end() {
+ }
+
+ </#if>
+
+ <#list vv.types as type><#list type.minor as minor>
+ <#assign lowerName = minor.class?uncap_first />
+ <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+ <#assign upperName = minor.class?upper_case />
+ <#assign capName = minor.class?cap_first />
+ <#assign vectName = capName />
+ <#assign vectName = "Nullable${capName}" />
+
+ <#if minor.class?starts_with("Decimal") >
+ public ${minor.class}Writer ${lowerName}(String name) {
+ // returns existing writer
+ final FieldWriter writer = fields.get(name.toLowerCase());
+ assert writer != null;
+ return writer;
+ }
+
+ public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) {
+ final MajorType ${upperName}_TYPE = Types.withScaleAndPrecision(MinorType.${upperName}, DataMode.OPTIONAL, scale, precision);
+ <#else>
+ private static final MajorType ${upperName}_TYPE = Types.optional(MinorType.${upperName});
+ @Override
+ public ${minor.class}Writer ${lowerName}(String name) {
+ </#if>
+ FieldWriter writer = fields.get(name.toLowerCase());
+ if(writer == null) {
+ ValueVector vector;
+ ValueVector currentVector = container.getChild(name);
+ if (unionEnabled){
+ ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
+ writer = new PromotableWriter(v, container);
+ vector = v;
+ } else {
+ ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
+ writer = new ${vectName}WriterImpl(v, this);
+ vector = v;
+ }
+ if (currentVector == null || currentVector != vector) {
+ vector.allocateNewSafe();
+ }
+ writer.setPosition(${index});
+ fields.put(name.toLowerCase(), writer);
+ }
+ return writer;
+ }
+
+ </#list></#list>
+
+}
+</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/NullReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullReader.java b/exec/vector/src/main/codegen/templates/NullReader.java
new file mode 100644
index 0000000..62aa33e
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/NullReader.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/NullReader.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+import org.apache.drill.common.types.TypeProtos;
+
+
+@SuppressWarnings("unused")
+public class NullReader extends AbstractBaseReader implements FieldReader{
+
+ public static final NullReader INSTANCE = new NullReader();
+ public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(Types.repeated(TypeProtos.MinorType.NULL));
+ public static final NullReader EMPTY_MAP_INSTANCE = new NullReader(Types.required(TypeProtos.MinorType.MAP));
+ private MajorType type;
+
+ private NullReader(){
+ super();
+ type = Types.NULL;
+ }
+
+ private NullReader(MajorType type){
+ super();
+ this.type = type;
+ }
+
+ @Override
+ public MajorType getType() {
+ return type;
+ }
+
+ public void copyAsValue(MapWriter writer) {}
+
+ public void copyAsValue(ListWriter writer) {}
+
+ public void copyAsValue(UnionWriter writer) {}
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ public void read(${name}Holder holder){
+ throw new UnsupportedOperationException("NullReader cannot write into non-nullable holder");
+ }
+
+ public void read(Nullable${name}Holder holder){
+ holder.isSet = 0;
+ }
+
+ public void read(int arrayIndex, ${name}Holder holder){
+ throw new ArrayIndexOutOfBoundsException();
+ }
+
+ public void copyAsValue(${minor.class}Writer writer){}
+ public void copyAsField(String name, ${minor.class}Writer writer){}
+
+ public void read(int arrayIndex, Nullable${name}Holder holder){
+ throw new ArrayIndexOutOfBoundsException();
+ }
+ </#list></#list>
+
+ public int size(){
+ return 0;
+ }
+
+ public boolean isSet(){
+ return false;
+ }
+
+ public boolean next(){
+ return false;
+ }
+
+ public RepeatedMapReader map(){
+ return this;
+ }
+
+ public RepeatedListReader list(){
+ return this;
+ }
+
+ public MapReader map(String name){
+ return this;
+ }
+
+ public ListReader list(String name){
+ return this;
+ }
+
+ public FieldReader reader(String name){
+ return this;
+ }
+
+ public FieldReader reader(){
+ return this;
+ }
+
+ private void fail(String name){
+ throw new IllegalArgumentException(String.format("You tried to read a %s type when you are using a ValueReader of type %s.", name, this.getClass().getSimpleName()));
+ }
+
+ <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
+ "Character", "DateTime", "Period", "Double", "Float",
+ "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
+ <#assign safeType=friendlyType />
+ <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+
+ public ${friendlyType} read${safeType}(int arrayIndex){
+ return null;
+ }
+
+ public ${friendlyType} read${safeType}(){
+ return null;
+ }
+ </#list>
+
+}
+
+
+
[04/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
new file mode 100644
index 0000000..e1025df
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class ComplexHolder implements ValueHolder {
+ public FieldReader reader;
+ public int isSet;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
new file mode 100644
index 0000000..391a795
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+
+/*
+ * Holder class for the vector ObjectVector. This holder internally stores a
+ * reference to an object. The ObjectVector maintains an array of these objects.
+ * This holder can be used only as workspace variables in aggregate functions.
+ * Using this holder should be avoided and we should stick to native holder types.
+ */
+@Deprecated
+public class ObjectHolder implements ValueHolder {
+ public static final TypeProtos.MajorType TYPE = Types.required(TypeProtos.MinorType.GENERIC_OBJECT);
+
+ public TypeProtos.MajorType getType() {return TYPE;}
+
+ public Object obj;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
new file mode 100644
index 0000000..09746da
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+public final class RepeatedListHolder implements ValueHolder{
+ public int start;
+ public int end;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
new file mode 100644
index 0000000..247f75e
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+public final class RepeatedMapHolder implements ValueHolder{
+ public int start;
+ public int end;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
new file mode 100644
index 0000000..84cdefb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class UnionHolder implements ValueHolder {
+ public static final MajorType TYPE = Types.optional(MinorType.UNION);
+ public FieldReader reader;
+ public int isSet;
+
+ public MajorType getType() {
+ return reader.getType();
+ }
+
+ public boolean isSet() {
+ return isSet == 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
new file mode 100644
index 0000000..2431dcb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+/**
+ * Wrapper object for an individual value in Drill.
+ *
+ * ValueHolders are designed to be mutable wrapper objects for defining clean
+ * APIs that access data in Drill. For performance, object creation is avoided
+ * at all costs throughout execution. For this reason, ValueHolders are
+ * disallowed from implementing any methods, this allows for them to be
+ * replaced by their java primitive inner members during optimization of
+ * run-time generated code.
+ */
+public interface ValueHolder {
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
new file mode 100644
index 0000000..b01534e
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.DrillBuf;
+import io.netty.buffer.UnsafeDirectLittleEndian;
+
+import java.io.Closeable;
+
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.util.Pointer;
+
+/**
+ * Wrapper class to deal with byte buffer allocation. Ensures users only use designated methods. Also allows inser
+ */
+public interface BufferAllocator extends Closeable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BufferAllocator.class);
+
+ /**
+ * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
+ * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
+ *
+ * @param size
+ * The size in bytes.
+ * @return A new ByteBuf.
+ * @throws OutOfMemoryRuntimeException if buffer cannot be allocated
+ */
+ public abstract DrillBuf buffer(int size);
+
+ /**
+ * Allocate a new or reused buffer within provided range. Note that the buffer may technically be larger than the
+ * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
+ *
+ * @param minSize The minimum size in bytes.
+ * @param maxSize The maximum size in bytes.
+ * @return A new ByteBuf.
+ * @throws OutOfMemoryRuntimeException if buffer cannot be allocated
+ */
+ public abstract DrillBuf buffer(int minSize, int maxSize);
+
+ public abstract ByteBufAllocator getUnderlyingAllocator();
+
+ /**
+ * Create a child allocator nested below this one.
+ *
+ * @param context - owning fragment for this allocator
+ * @param initialReservation - specified in bytes
+ * @param maximumReservation - specified in bytes
+ * @param applyFragmentLimit - flag to conditionally enable fragment memory limits
+ * @return - a new buffer allocator owned by the parent it was spawned from
+ * @throws OutOfMemoryException - when off-heap memory has been exhausted
+ */
+ public abstract BufferAllocator getChildAllocator(FragmentContext context, long initialReservation,
+ long maximumReservation, boolean applyFragmentLimit) throws OutOfMemoryException;
+
+ /**
+ * Take over ownership of fragment accounting. Always takes over ownership.
+ * @param buf
+ * @return false if over allocation.
+ */
+ public boolean takeOwnership(DrillBuf buf) ;
+
+ /**
+ * Take over ownership of fragment accounting. Always takes over ownership.
+ * @param buf
+ * @return false if over allocation.
+ */
+ public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut);
+
+ public PreAllocator getNewPreAllocator();
+
+ //public void addFragmentContext(FragmentContext c);
+
+ /**
+ * For Top Level Allocators. Reset the fragment limits for all allocators
+ */
+ public void resetFragmentLimits();
+
+ /**
+ * For Child allocators to set the Fragment limit for the corresponding fragment allocator.
+ * @param l the new fragment limit
+ */
+ public void setFragmentLimit(long l);
+
+ public long getFragmentLimit();
+
+
+ /**
+ * Not thread safe.
+ *
+ * WARNING: unclaimed pre-allocations leak memory. If you call preAllocate(), you must
+ * make sure to ultimately try to get the buffer and release it.
+ */
+ public interface PreAllocator {
+ public boolean preAllocate(int bytes);
+
+ public DrillBuf getAllocation();
+ }
+
+ /**
+ * @param bytes
+ * @return
+ */
+
+ /**
+ *
+ */
+
+ /**
+ * Close and release all buffers generated from this buffer pool.
+ */
+ @Override
+ public abstract void close();
+
+ public abstract long getAllocatedMemory();
+
+ public abstract long getPeakMemoryAllocation();
+
+ public DrillBuf getEmpty();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
new file mode 100644
index 0000000..305eabd
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+public class OutOfMemoryRuntimeException extends DrillRuntimeException{
+
+ public OutOfMemoryRuntimeException() {
+ super();
+
+ }
+
+ public OutOfMemoryRuntimeException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+
+ }
+
+ public OutOfMemoryRuntimeException(String message, Throwable cause) {
+ super(message, cause);
+
+ }
+
+ public OutOfMemoryRuntimeException(String message) {
+ super(message);
+
+ }
+
+ public OutOfMemoryRuntimeException(Throwable cause) {
+ super(cause);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
new file mode 100644
index 0000000..26e1257
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -0,0 +1,324 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Map;
+
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+
+
+public class MaterializedField {
+ private final Key key;
+ // use an ordered set as existing code relies on order (e,g. parquet writer)
+ private final LinkedHashSet<MaterializedField> children;
+
+ private MaterializedField(SchemaPath path, MajorType type) {
+ this(path, type, new LinkedHashSet<MaterializedField>());
+ }
+
+ private MaterializedField(SchemaPath path, MajorType type, LinkedHashSet<MaterializedField> children) {
+ this.key = new Key(path, type);
+ this.children = children;
+ }
+
+ public static MaterializedField create(SerializedField serField){
+ MaterializedField field = new MaterializedField(SchemaPath.create(serField.getNamePart()), serField.getMajorType());
+ for (SerializedField sf:serField.getChildList()) {
+ field.addChild(MaterializedField.create(sf));
+ }
+ return field;
+ }
+
+ /**
+ * Create and return a serialized field based on the current state.
+ */
+ public SerializedField getSerializedField() {
+ SerializedField.Builder serializedFieldBuilder = getAsBuilder();
+ for(MaterializedField childMaterializedField : getChildren()) {
+ serializedFieldBuilder.addChild(childMaterializedField.getSerializedField());
+ }
+ return serializedFieldBuilder.build();
+ }
+
+
+ public SerializedField.Builder getAsBuilder(){
+ return SerializedField.newBuilder() //
+ .setMajorType(key.type) //
+ .setNamePart(key.path.getAsNamePart());
+ }
+
+ public Collection<MaterializedField> getChildren() {
+ return children;
+ }
+
+ public void addChild(MaterializedField field){
+ children.add(field);
+ }
+
+ public MaterializedField clone() {
+ return withPathAndType(getPath(), getType());
+ }
+
+ public MaterializedField withType(MajorType type) {
+ return withPathAndType(getPath(), type);
+ }
+
+ public MaterializedField withPath(SchemaPath path) {
+ return withPathAndType(path, getType());
+ }
+
+ public MaterializedField withPathAndType(final SchemaPath path, final MajorType type) {
+ final LinkedHashSet<MaterializedField> newChildren = new LinkedHashSet<>(children.size());
+ final MaterializedField clone = new MaterializedField(path, type, newChildren);
+ for (final MaterializedField child:children) {
+ newChildren.add(child.clone());
+ }
+ return clone;
+ }
+
+ public String getLastName(){
+ PathSegment seg = key.path.getRootSegment();
+ while (seg.getChild() != null) {
+ seg = seg.getChild();
+ }
+ return seg.getNameSegment().getPath();
+ }
+
+
+ // TODO: rewrite without as direct match rather than conversion then match.
+ public boolean matches(SerializedField field){
+ MaterializedField f = create(field);
+ return f.equals(this);
+ }
+
+ public static MaterializedField create(String path, MajorType type){
+ SchemaPath p = SchemaPath.getSimplePath(path);
+ return create(p, type);
+ }
+
+ public static MaterializedField create(SchemaPath path, MajorType type) {
+ return new MaterializedField(path, type);
+ }
+
+ public SchemaPath getPath(){
+ return key.path;
+ }
+
+ /**
+ * Get the schema path. Deprecated, use getPath() instead.
+ * @return the SchemaPath of this field.
+ */
+ @Deprecated
+ public SchemaPath getAsSchemaPath(){
+ return getPath();
+ }
+
+// public String getName(){
+// StringBuilder sb = new StringBuilder();
+// boolean first = true;
+// for(NamePart np : def.getNameList()){
+// if(np.getType() == Type.ARRAY){
+// sb.append("[]");
+// }else{
+// if(first){
+// first = false;
+// }else{
+// sb.append(".");
+// }
+// sb.append('`');
+// sb.append(np.getName());
+// sb.append('`');
+//
+// }
+// }
+// return sb.toString();
+// }
+
+ public int getWidth() {
+ return key.type.getWidth();
+ }
+
+ public MajorType getType() {
+ return key.type;
+ }
+
+ public int getScale() {
+ return key.type.getScale();
+ }
+ public int getPrecision() {
+ return key.type.getPrecision();
+ }
+ public boolean isNullable() {
+ return key.type.getMode() == DataMode.OPTIONAL;
+ }
+
+ public DataMode getDataMode() {
+ return key.type.getMode();
+ }
+
+ public MaterializedField getOtherNullableVersion(){
+ MajorType mt = key.type;
+ DataMode newDataMode = null;
+ switch (mt.getMode()){
+ case OPTIONAL:
+ newDataMode = DataMode.REQUIRED;
+ break;
+ case REQUIRED:
+ newDataMode = DataMode.OPTIONAL;
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ return new MaterializedField(key.path, mt.toBuilder().setMode(newDataMode).build());
+ }
+
+ public Class<?> getValueClass() {
+ return TypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
+ }
+
+ public boolean matches(SchemaPath path) {
+ if (!path.isSimplePath()) {
+ return false;
+ }
+
+ return key.path.equals(path);
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = 1;
+ // DRILL-1872: Compute hashCode only on key. See also the comment
+ // in MapVector$MapTransferPair
+ result = ((key == null) ? 0 : key.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ MaterializedField other = (MaterializedField) obj;
+ // DRILL-1872: Compute equals only on key. See also the comment
+ // in MapVector$MapTransferPair
+
+ if (key == null) {
+ if (other.key != null) {
+ return false;
+ }
+ } else if (!key.equals(other.key)) {
+ return false;
+ }
+ return true;
+ }
+
+
+ @Override
+ public String toString() {
+ final int maxLen = 10;
+ String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : "";
+ return key.path + "(" + key.type.getMinorType().name() + ":" + key.type.getMode().name() + ")" + childStr;
+ }
+
+
+ private String toString(Collection<?> collection, int maxLen) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("[");
+ int i = 0;
+ for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
+ if (i > 0){
+ builder.append(", ");
+ }
+ builder.append(iterator.next());
+ }
+ builder.append("]");
+ return builder.toString();
+ }
+
+ public Key key() {
+ return key;
+ }
+
+ public String toExpr(){
+ return key.path.toExpr();
+ }
+
+ /**
+ * Since the {@code MaterializedField) itself is mutable, in certain cases, it is not suitable
+ * as a key of a {@link Map}. This inner class allows the {@link MaterializedField} object to be
+ * used for this purpose.
+ */
+ public class Key {
+
+ private final SchemaPath path;
+ private final MajorType type;
+
+ private Key(SchemaPath path, MajorType type) {
+ this.path = path;
+ this.type = type;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((path == null) ? 0 : path.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ Key other = (Key) obj;
+ if (path == null) {
+ if (other.path != null) {
+ return false;
+ }
+ } else if (!path.equals(other.path)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/record/TransferPair.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/TransferPair.java b/exec/vector/src/main/java/org/apache/drill/exec/record/TransferPair.java
new file mode 100644
index 0000000..3585a8c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/TransferPair.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import org.apache.drill.exec.vector.ValueVector;
+
+public interface TransferPair {
+ public void transfer();
+ public void splitAndTransfer(int startIndex, int length);
+ public ValueVector getTo();
+ public void copyValueSafe(int from, int to);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java b/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
new file mode 100644
index 0000000..931f981
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import java.util.Arrays;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.vector.ValueVector;
+
+import com.carrotsearch.hppc.IntArrayList;
+import com.google.common.base.Preconditions;
+
+public class TypedFieldId {
+ final MajorType finalType;
+ final MajorType secondaryFinal;
+ final MajorType intermediateType;
+ final int[] fieldIds;
+ final boolean isHyperReader;
+ final boolean isListVector;
+ final PathSegment remainder;
+
+ public TypedFieldId(MajorType type, int... fieldIds) {
+ this(type, type, type, false, null, fieldIds);
+ }
+
+ public TypedFieldId(MajorType type, IntArrayList breadCrumb, PathSegment remainder) {
+ this(type, type, type, false, remainder, breadCrumb.toArray());
+ }
+
+ public TypedFieldId(MajorType type, boolean isHyper, int... fieldIds) {
+ this(type, type, type, isHyper, null, fieldIds);
+ }
+
+ public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, PathSegment remainder, int... fieldIds) {
+ this(intermediateType, secondaryFinal, finalType, isHyper, false, remainder, fieldIds);
+ }
+
+ public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, boolean isListVector, PathSegment remainder, int... fieldIds) {
+ super();
+ this.intermediateType = intermediateType;
+ this.finalType = finalType;
+ this.secondaryFinal = secondaryFinal;
+ this.fieldIds = fieldIds;
+ this.isHyperReader = isHyper;
+ this.isListVector = isListVector;
+ this.remainder = remainder;
+ }
+
+ public TypedFieldId cloneWithChild(int id) {
+ int[] fieldIds = ArrayUtils.add(this.fieldIds, id);
+ return new TypedFieldId(intermediateType, secondaryFinal, finalType, isHyperReader, remainder, fieldIds);
+ }
+
+ public PathSegment getLastSegment() {
+ if (remainder == null) {
+ return null;
+ }
+ PathSegment seg = remainder;
+ while (seg.getChild() != null) {
+ seg = seg.getChild();
+ }
+ return seg;
+ }
+
+ public TypedFieldId cloneWithRemainder(PathSegment remainder) {
+ return new TypedFieldId(intermediateType, secondaryFinal, finalType, isHyperReader, remainder, fieldIds);
+ }
+
+ public boolean hasRemainder() {
+ return remainder != null;
+ }
+
+ public PathSegment getRemainder() {
+ return remainder;
+ }
+
+ public boolean isHyperReader() {
+ return isHyperReader;
+ }
+
+ public boolean isListVector() {
+ return isListVector;
+ }
+
+ public MajorType getIntermediateType() {
+ return intermediateType;
+ }
+
+ public Class<? extends ValueVector> getIntermediateClass() {
+ return (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(intermediateType.getMinorType(), intermediateType.getMode());
+ }
+
+ public MajorType getFinalType() {
+ return finalType;
+ }
+
+ public int[] getFieldIds() {
+ return fieldIds;
+ }
+
+ public MajorType getSecondaryFinal() {
+ return secondaryFinal;
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ public static class Builder{
+ final IntArrayList ids = new IntArrayList();
+ MajorType finalType;
+ MajorType intermediateType;
+ MajorType secondaryFinal;
+ PathSegment remainder;
+ boolean hyperReader = false;
+ boolean withIndex = false;
+ boolean isListVector = false;
+
+ public Builder addId(int id) {
+ ids.add(id);
+ return this;
+ }
+
+ public Builder withIndex() {
+ withIndex = true;
+ return this;
+ }
+
+ public Builder remainder(PathSegment remainder) {
+ this.remainder = remainder;
+ return this;
+ }
+
+ public Builder hyper() {
+ this.hyperReader = true;
+ return this;
+ }
+
+ public Builder listVector() {
+ this.isListVector = true;
+ return this;
+ }
+
+ public Builder finalType(MajorType finalType) {
+ this.finalType = finalType;
+ return this;
+ }
+
+ public Builder secondaryFinal(MajorType secondaryFinal) {
+ this.secondaryFinal = secondaryFinal;
+ return this;
+ }
+
+ public Builder intermediateType(MajorType intermediateType) {
+ this.intermediateType = intermediateType;
+ return this;
+ }
+
+ public TypedFieldId build() {
+ Preconditions.checkNotNull(intermediateType);
+ Preconditions.checkNotNull(finalType);
+
+ if (intermediateType == null) {
+ intermediateType = finalType;
+ }
+ if (secondaryFinal == null) {
+ secondaryFinal = finalType;
+ }
+
+ MajorType actualFinalType = finalType;
+ //MajorType secondaryFinal = finalType;
+
+ // if this has an index, switch to required type for output
+ //if(withIndex && intermediateType == finalType) actualFinalType = finalType.toBuilder().setMode(DataMode.REQUIRED).build();
+
+ // if this isn't a direct access, switch the final type to nullable as offsets may be null.
+ // TODO: there is a bug here with some things.
+ //if(intermediateType != finalType) actualFinalType = finalType.toBuilder().setMode(DataMode.OPTIONAL).build();
+
+ return new TypedFieldId(intermediateType, secondaryFinal, actualFinalType, hyperReader, isListVector, remainder, ids.toArray());
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(fieldIds);
+ result = prime * result + ((finalType == null) ? 0 : finalType.hashCode());
+ result = prime * result + ((intermediateType == null) ? 0 : intermediateType.hashCode());
+ result = prime * result + (isHyperReader ? 1231 : 1237);
+ result = prime * result + ((remainder == null) ? 0 : remainder.hashCode());
+ result = prime * result + ((secondaryFinal == null) ? 0 : secondaryFinal.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TypedFieldId other = (TypedFieldId) obj;
+ if (!Arrays.equals(fieldIds, other.fieldIds)) {
+ return false;
+ }
+ if (finalType == null) {
+ if (other.finalType != null) {
+ return false;
+ }
+ } else if (!finalType.equals(other.finalType)) {
+ return false;
+ }
+ if (intermediateType == null) {
+ if (other.intermediateType != null) {
+ return false;
+ }
+ } else if (!intermediateType.equals(other.intermediateType)) {
+ return false;
+ }
+ if (isHyperReader != other.isHyperReader) {
+ return false;
+ }
+ if (remainder == null) {
+ if (other.remainder != null) {
+ return false;
+ }
+ } else if (!remainder.equals(other.remainder)) {
+ return false;
+ }
+ if (secondaryFinal == null) {
+ if (other.secondaryFinal != null) {
+ return false;
+ }
+ } else if (!secondaryFinal.equals(other.secondaryFinal)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ final int maxLen = 10;
+ return "TypedFieldId [fieldIds="
+ + (fieldIds != null ? Arrays.toString(Arrays.copyOf(fieldIds, Math.min(fieldIds.length, maxLen))) : null)
+ + ", remainder=" + remainder + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/CallBack.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/CallBack.java b/exec/vector/src/main/java/org/apache/drill/exec/util/CallBack.java
new file mode 100644
index 0000000..0243f8a
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/CallBack.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+
+public interface CallBack {
+ public void doWork();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
new file mode 100644
index 0000000..dd5145d
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+public class JsonStringArrayList<E> extends ArrayList<E> {
+
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ SimpleModule serializer = new SimpleModule("TextSerializer")
+ .addSerializer(Text.class, new TextSerializer());
+ mapper.registerModule(serializer);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof List)) {
+ return false;
+ }
+ List other = (List) obj;
+ return this.size() == other.size() && this.containsAll(other);
+ }
+
+ @Override
+ public final String toString() {
+ try {
+ return mapper.writeValueAsString(this);
+ } catch(JsonProcessingException e) {
+ throw new IllegalStateException("Cannot serialize array list to JSON string", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
new file mode 100644
index 0000000..ac980eb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.io.Text;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+/*
+ * Simple class that extends the regular java.util.HashMap but overrides the
+ * toString() method of the HashMap class to produce a JSON string instead
+ */
+public class JsonStringHashMap<K, V> extends LinkedHashMap<K, V> {
+
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ SimpleModule serializer = new SimpleModule("TextSerializer")
+ .addSerializer(Text.class, new TextSerializer());
+ mapper.registerModule(serializer);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof Map)) {
+ return false;
+ }
+ Map other = (Map) obj;
+ if (this.size() != other.size()) {
+ return false;
+ }
+ for (K key : this.keySet()) {
+ if (this.get(key) == null ) {
+ if (other.get(key) == null) {
+ continue;
+ } else {
+ return false;
+ }
+ }
+ if ( ! this.get(key).equals(other.get(key))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public final String toString() {
+ try {
+ return mapper.writeValueAsString(this);
+ } catch(JsonProcessingException e) {
+ throw new IllegalStateException("Cannot serialize hash map to JSON string", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java b/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
new file mode 100644
index 0000000..074ed76
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+public class Pointer<T> {
+ public T value;
+
+ public Pointer(){}
+
+ public Pointer(T value){
+ this.value = value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
new file mode 100644
index 0000000..87df102
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+
+public class TextSerializer extends StdSerializer<Text> {
+
+ public TextSerializer() {
+ super(Text.class);
+ }
+
+ @Override
+ public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
+ jsonGenerator.writeString(text.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
new file mode 100644
index 0000000..7d1f08d
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import com.google.common.base.Preconditions;
+
+public class AddOrGetResult<V extends ValueVector> {
+ private final V vector;
+ private final boolean created;
+
+ public AddOrGetResult(V vector, boolean created) {
+ this.vector = Preconditions.checkNotNull(vector);
+ this.created = created;
+ }
+
+ public V getVector() {
+ return vector;
+ }
+
+ public boolean isCreated() {
+ return created;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
new file mode 100644
index 0000000..622e2d2
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import org.apache.drill.exec.vector.complex.RepeatedFixedWidthVectorLike;
+import org.apache.drill.exec.vector.complex.RepeatedVariableWidthVectorLike;
+
+public class AllocationHelper {
+// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class);
+
+ public static void allocate(ValueVector v, int valueCount, int bytesPerValue) {
+ allocate(v, valueCount, bytesPerValue, 5);
+ }
+
+ public static void allocatePrecomputedChildCount(ValueVector v, int valueCount, int bytesPerValue, int childValCount){
+ if(v instanceof FixedWidthVector) {
+ ((FixedWidthVector) v).allocateNew(valueCount);
+ } else if (v instanceof VariableWidthVector) {
+ ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount);
+ } else if(v instanceof RepeatedFixedWidthVectorLike) {
+ ((RepeatedFixedWidthVectorLike) v).allocateNew(valueCount, childValCount);
+ } else if(v instanceof RepeatedVariableWidthVectorLike) {
+ ((RepeatedVariableWidthVectorLike) v).allocateNew(childValCount * bytesPerValue, valueCount, childValCount);
+ } else {
+ v.allocateNew();
+ }
+ }
+
+ public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){
+ allocatePrecomputedChildCount(v, valueCount, bytesPerValue, repeatedPerTop * valueCount);
+ }
+
+ /**
+ * Allocates the exact amount if v is fixed width, otherwise falls back to dynamic allocation
+ * @param v value vector we are trying to allocate
+ * @param valueCount size we are trying to allocate
+ * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the memory
+ */
+ public static void allocateNew(ValueVector v, int valueCount) {
+ if (v instanceof FixedWidthVector) {
+ ((FixedWidthVector) v).allocateNew(valueCount);
+ } else {
+ v.allocateNew();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
new file mode 100644
index 0000000..2fc741c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.record.MaterializedField;
+
+
+public abstract class BaseDataValueVector extends BaseValueVector {
+
+ protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
+
+ protected DrillBuf data;
+
+ public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ data = allocator.getEmpty();
+ }
+
+ @Override
+ public void clear() {
+ if (data != null) {
+ data.release();
+ }
+ data = allocator.getEmpty();
+ super.clear();
+ }
+
+ @Override
+ public void close() {
+ clear();
+ if (data != null) {
+ data.release();
+ data = null;
+ }
+ super.close();
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ DrillBuf[] out;
+ if (getBufferSize() == 0) {
+ out = new DrillBuf[0];
+ } else {
+ out = new DrillBuf[]{data};
+ if (clear) {
+ data.readerIndex(0);
+ data.retain(1);
+ }
+ }
+ if (clear) {
+ clear();
+ }
+ return out;
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (getAccessor().getValueCount() == 0) {
+ return 0;
+ }
+ return data.writerIndex();
+ }
+
+ public DrillBuf getBuffer() {
+ return data;
+ }
+
+ /**
+ * This method has a similar effect of allocateNew() without actually clearing and reallocating
+ * the value vector. The purpose is to move the value vector to a "mutate" state
+ */
+ public void reset() {}
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
new file mode 100644
index 0000000..eb5dbcd
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.DrillBuf;
+
+import java.util.Iterator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class BaseValueVector implements ValueVector {
+ private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
+
+ public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
+ public static final int INITIAL_VALUE_ALLOCATION = 4096;
+
+ protected final BufferAllocator allocator;
+ protected final MaterializedField field;
+
+ protected BaseValueVector(MaterializedField field, BufferAllocator allocator) {
+ this.field = Preconditions.checkNotNull(field, "field cannot be null");
+ this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + "[field = " + field + ", ...]";
+ }
+
+ @Override
+ public void clear() {
+ getMutator().reset();
+ }
+
+ @Override
+ public void close() {
+ clear();
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ public MaterializedField getField(FieldReference ref){
+ return getField().withPath(ref);
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ return getTransferPair(new FieldReference(getField().getPath()));
+ }
+
+ @Override
+ public SerializedField getMetadata() {
+ return getMetadataBuilder().build();
+ }
+
+ protected SerializedField.Builder getMetadataBuilder() {
+ return getField().getAsBuilder()
+ .setValueCount(getAccessor().getValueCount())
+ .setBufferLength(getBufferSize());
+ }
+
+ public abstract static class BaseAccessor implements ValueVector.Accessor {
+ protected BaseAccessor() { }
+
+ @Override
+ public boolean isNull(int index) {
+ return false;
+ }
+ }
+
+ public abstract static class BaseMutator implements ValueVector.Mutator {
+ protected BaseMutator() { }
+
+ @Override
+ public void generateTestData(int values) {}
+
+ //TODO: consider making mutator stateless(if possible) on another issue.
+ public void reset() {}
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ public static boolean checkBufRefs(final ValueVector vv) {
+ for(final DrillBuf buffer : vv.getBuffers(false)) {
+ if (buffer.refCnt() <= 0) {
+ throw new IllegalStateException("zero refcount");
+ }
+ }
+
+ return true;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
new file mode 100644
index 0000000..67ba5c4
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -0,0 +1,448 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import com.google.common.base.Preconditions;
+import io.netty.buffer.DrillBuf;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.exception.OversizedAllocationException;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.complex.impl.BitReaderImpl;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+/**
+ * Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start
+ * of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0'
+ * or '1'.
+ */
+public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
+
+ private final FieldReader reader = new BitReaderImpl(BitVector.this);
+ private final Accessor accessor = new Accessor();
+ private final Mutator mutator = new Mutator();
+
+ private int valueCount;
+ private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+ private int allocationMonitor = 0;
+
+ public BitVector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ @Override
+ public FieldReader getReader() {
+ return reader;
+ }
+
+ @Override
+ public int getBufferSize() {
+ return getSizeFromCount(valueCount);
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ return getSizeFromCount(valueCount);
+ }
+
+ private int getSizeFromCount(int valueCount) {
+ return (int) Math.ceil(valueCount / 8.0);
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return (int)Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L);
+ }
+
+ private int getByteIndex(int index) {
+ return (int) Math.floor(index / 8.0);
+ }
+
+ @Override
+ public void setInitialCapacity(final int valueCount) {
+ allocationSizeInBytes = getSizeFromCount(valueCount);
+ }
+
+ @Override
+ public void allocateNew() {
+ if (!allocateNewSafe()) {
+ throw new OutOfMemoryRuntimeException();
+ }
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ long curAllocationSize = allocationSizeInBytes;
+ if (allocationMonitor > 10) {
+ curAllocationSize = Math.max(8, allocationSizeInBytes / 2);
+ allocationMonitor = 0;
+ } else if (allocationMonitor < -2) {
+ curAllocationSize = allocationSizeInBytes * 2L;
+ allocationMonitor = 0;
+ }
+
+ try {
+ allocateBytes(curAllocationSize);
+ } catch (OutOfMemoryRuntimeException ex) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void reset() {
+ valueCount = 0;
+ allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+ allocationMonitor = 0;
+ zeroVector();
+ super.reset();
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount
+ * The number of values which can be contained within this vector.
+ */
+ @Override
+ public void allocateNew(int valueCount) {
+ final int size = getSizeFromCount(valueCount);
+ allocateBytes(size);
+ }
+
+ private void allocateBytes(final long size) {
+ if (size > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+ }
+
+ final int curSize = (int) size;
+ clear();
+ data = allocator.buffer(curSize);
+ zeroVector();
+ allocationSizeInBytes = curSize;
+ }
+
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+ */
+ public void reAlloc() {
+ final long newAllocationSize = allocationSizeInBytes * 2L;
+ if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+ }
+
+ final int curSize = (int)newAllocationSize;
+ final DrillBuf newBuf = allocator.buffer(curSize);
+ newBuf.setZero(0, newBuf.capacity());
+ newBuf.setBytes(0, data, 0, data.capacity());
+ data.release();
+ data = newBuf;
+ allocationSizeInBytes = curSize;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void zeroVector() {
+ data.setZero(0, data.capacity());
+ }
+
+ public void copyFrom(int inIndex, int outIndex, BitVector from) {
+ this.mutator.set(outIndex, from.accessor.get(inIndex));
+ }
+
+ public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) {
+ if (outIndex >= this.getValueCapacity()) {
+ decrementAllocationMonitor();
+ return false;
+ }
+ copyFrom(inIndex, outIndex, from);
+ return true;
+ }
+
+ @Override
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ Preconditions.checkArgument(this.field.matches(metadata), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
+ final int valueCount = metadata.getValueCount();
+ final int expectedLength = getSizeFromCount(valueCount);
+ final int actualLength = metadata.getBufferLength();
+ assert expectedLength == actualLength: "expected and actual buffer sizes do not match";
+
+ clear();
+ data = buffer.slice(0, actualLength);
+ data.retain();
+ this.valueCount = valueCount;
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return new Mutator();
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return new Accessor();
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ return new TransferImpl(getField());
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ return new TransferImpl(getField().withPath(ref));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new TransferImpl((BitVector) to);
+ }
+
+
+ public void transferTo(BitVector target) {
+ target.clear();
+ if (target.data != null) {
+ target.data.release();
+ }
+ target.data = data;
+ target.data.retain(1);
+ target.valueCount = valueCount;
+ clear();
+ }
+
+ public void splitAndTransferTo(int startIndex, int length, BitVector target) {
+ assert startIndex + length <= valueCount;
+ int firstByte = getByteIndex(startIndex);
+ int byteSize = getSizeFromCount(length);
+ int offset = startIndex % 8;
+ if (offset == 0) {
+ target.clear();
+ // slice
+ if (target.data != null) {
+ target.data.release();
+ }
+ target.data = (DrillBuf) data.slice(firstByte, byteSize);
+ target.data.retain(1);
+ } else {
+ // Copy data
+ // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
+ // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
+ // The last byte copied to target is a bit tricky :
+ // 1) if length requires partly byte (length % 8 !=0), copy the remaining bits only.
+ // 2) otherwise, copy the last byte in the same way as to the prior bytes.
+ target.clear();
+ target.allocateNew(length);
+ // TODO maybe do this one word at a time, rather than byte?
+ for(int i = 0; i < byteSize - 1; i++) {
+ target.data.setByte(i, (((this.data.getByte(firstByte + i) & 0xFF) >>> offset) + (this.data.getByte(firstByte + i + 1) << (8 - offset))));
+ }
+ if (length % 8 != 0) {
+ target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
+ } else {
+ target.data.setByte(byteSize - 1,
+ (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) << (8 - offset))));
+ }
+ }
+ target.getMutator().setValueCount(length);
+ }
+
+ private class TransferImpl implements TransferPair {
+ BitVector to;
+
+ public TransferImpl(MaterializedField field) {
+ this.to = new BitVector(field, allocator);
+ }
+
+ public TransferImpl(BitVector to) {
+ this.to = to;
+ }
+
+ @Override
+ public BitVector getTo() {
+ return to;
+ }
+
+ @Override
+ public void transfer() {
+ transferTo(to);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ splitAndTransferTo(startIndex, length, to);
+ }
+
+ @Override
+ public void copyValueSafe(int fromIndex, int toIndex) {
+ to.copyFromSafe(fromIndex, toIndex, BitVector.this);
+ }
+ }
+
+ private void decrementAllocationMonitor() {
+ if (allocationMonitor > 0) {
+ allocationMonitor = 0;
+ }
+ --allocationMonitor;
+ }
+
+ private void incrementAllocationMonitor() {
+ ++allocationMonitor;
+ }
+
+ public class Accessor extends BaseAccessor {
+
+ /**
+ * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the bit was not set.
+ *
+ * @param index
+ * position of the bit in the vector
+ * @return 1 if set, otherwise 0
+ */
+ public final int get(int index) {
+ int byteIndex = index >> 3;
+ byte b = data.getByte(byteIndex);
+ int bitIndex = index & 7;
+ return Long.bitCount(b & (1L << bitIndex));
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return false;
+ }
+
+ @Override
+ public final Boolean getObject(int index) {
+ return new Boolean(get(index) != 0);
+ }
+
+ @Override
+ public final int getValueCount() {
+ return valueCount;
+ }
+
+ public final void get(int index, BitHolder holder) {
+ holder.value = get(index);
+ }
+
+ public final void get(int index, NullableBitHolder holder) {
+ holder.isSet = 1;
+ holder.value = get(index);
+ }
+ }
+
+ /**
+ * MutableBit implements a vector of bit-width values. Elements in the vector are accessed by position from the
+ * logical start of the vector. Values should be pushed onto the vector sequentially, but may be randomly accessed.
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public class Mutator extends BaseMutator {
+
+ private Mutator() {
+ }
+
+ /**
+ * Set the bit at the given index to the specified value.
+ *
+ * @param index
+ * position of the bit to set
+ * @param value
+ * value to set (either 1 or 0)
+ */
+ public final void set(int index, int value) {
+ int byteIndex = index >> 3;
+ int bitIndex = index & 7;
+ byte currentByte = data.getByte(byteIndex);
+ byte bitMask = (byte) (1L << bitIndex);
+ if (value != 0) {
+ currentByte |= bitMask;
+ } else {
+ currentByte -= (bitMask & currentByte);
+ }
+
+ data.setByte(byteIndex, currentByte);
+ }
+
+ public final void set(int index, BitHolder holder) {
+ set(index, holder.value);
+ }
+
+ final void set(int index, NullableBitHolder holder) {
+ set(index, holder.value);
+ }
+
+ public void setSafe(int index, int value) {
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, value);
+ }
+
+ public void setSafe(int index, BitHolder holder) {
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, holder.value);
+ }
+
+ public void setSafe(int index, NullableBitHolder holder) {
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ set(index, holder.value);
+ }
+
+ @Override
+ public final void setValueCount(int valueCount) {
+ int currentValueCapacity = getValueCapacity();
+ BitVector.this.valueCount = valueCount;
+ int idx = getSizeFromCount(valueCount);
+ while(valueCount > getValueCapacity()) {
+ reAlloc();
+ }
+ if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
+ incrementAllocationMonitor();
+ } else if (allocationMonitor > 0) {
+ allocationMonitor = 0;
+ }
+ VectorTrimmer.trim(data, idx);
+ }
+
+ @Override
+ public final void generateTestData(int values) {
+ boolean even = true;
+ for(int i = 0; i < values; i++, even = !even) {
+ if (even) {
+ set(i, 1);
+ }
+ }
+ setValueCount(values);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
new file mode 100644
index 0000000..c2781eb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+
+public interface FixedWidthVector extends ValueVector{
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount Number of values in the vector.
+ */
+ void allocateNew(int valueCount);
+
+/**
+ * Zero out the underlying buffer backing this vector.
+ */
+ void zeroVector();
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
new file mode 100644
index 0000000..8091c4c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+public interface NullableVector extends ValueVector{
+
+ ValueVector getValuesVector();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
new file mode 100644
index 0000000..c418751
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+public interface NullableVectorDefinitionSetter {
+
+ public void setIndexDefined(int index);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
new file mode 100644
index 0000000..a7a3609
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.DrillBuf;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.expr.holders.ObjectHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class ObjectVector extends BaseValueVector {
+ private final Accessor accessor = new Accessor();
+ private final Mutator mutator = new Mutator();
+ private int maxCount = 0;
+ private int count = 0;
+ private int allocationSize = 4096;
+
+ private List<Object[]> objectArrayList = new ArrayList<>();
+
+ public ObjectVector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ public void addNewArray() {
+ objectArrayList.add(new Object[allocationSize]);
+ maxCount += allocationSize;
+ }
+
+ @Override
+ public FieldReader getReader() {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ public final class Mutator implements ValueVector.Mutator {
+
+ public void set(int index, Object obj) {
+ int listOffset = index / allocationSize;
+ if (listOffset >= objectArrayList.size()) {
+ addNewArray();
+ }
+ objectArrayList.get(listOffset)[index % allocationSize] = obj;
+ }
+
+ public boolean setSafe(int index, long value) {
+ set(index, value);
+ return true;
+ }
+
+ protected void set(int index, ObjectHolder holder) {
+ set(index, holder.obj);
+ }
+
+ public boolean setSafe(int index, ObjectHolder holder){
+ set(index, holder);
+ return true;
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ count = valueCount;
+ }
+
+ @Override
+ public void reset() {
+ count = 0;
+ maxCount = 0;
+ objectArrayList = new ArrayList<>();
+ addNewArray();
+ }
+
+ @Override
+ public void generateTestData(int values) {
+ }
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ // NoOp
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryRuntimeException {
+ addNewArray();
+ }
+
+ public void allocateNew(int valueCount) throws OutOfMemoryRuntimeException {
+ while (maxCount < valueCount) {
+ addNewArray();
+ }
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ allocateNew();
+ return true;
+ }
+
+ @Override
+ public int getBufferSize() {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ @Override
+ public void close() {
+ clear();
+ }
+
+ @Override
+ public void clear() {
+ objectArrayList.clear();
+ maxCount = 0;
+ count = 0;
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return maxCount;
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ @Override
+ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ @Override
+ public UserBitShared.SerializedField getMetadata() {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ throw new UnsupportedOperationException("ObjectVector does not support this");
+ }
+
+ public final class Accessor extends BaseAccessor {
+ @Override
+ public Object getObject(int index) {
+ int listOffset = index / allocationSize;
+ if (listOffset >= objectArrayList.size()) {
+ addNewArray();
+ }
+ return objectArrayList.get(listOffset)[index % allocationSize];
+ }
+
+ @Override
+ public int getValueCount() {
+ return count;
+ }
+
+ public Object get(int index) {
+ return getObject(index);
+ }
+
+ public void get(int index, ObjectHolder holder){
+ holder.obj = getObject(index);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
new file mode 100644
index 0000000..4c2491c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.vector;
+
+import org.apache.drill.exec.util.CallBack;
+
+
+public class SchemaChangeCallBack implements CallBack {
+ private boolean schemaChanged = false;
+
+ /**
+ * Constructs a schema-change callback with the schema-changed state set to
+ * {@code false}.
+ */
+ public SchemaChangeCallBack() {
+ }
+
+ /**
+ * Sets the schema-changed state to {@code true}.
+ */
+ @Override
+ public void doWork() {
+ schemaChanged = true;
+ }
+
+ /**
+ * Returns the value of schema-changed state, <strong>resetting</strong> the
+ * schema-changed state to {@code false}.
+ */
+ public boolean getSchemaChangedAndReset() {
+ final boolean current = schemaChanged;
+ schemaChanged = false;
+ return current;
+ }
+}
+
[40/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
deleted file mode 100644
index c29b9b0..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
+++ /dev/null
@@ -1,332 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.math.BigDecimal;
-
-import org.apache.drill.common.expression.IfExpression.IfCondition;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DateExpression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
-import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
-import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.joda.time.Period;
-
-import com.google.common.collect.ImmutableList;
-
-public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBuilder, RuntimeException>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionStringBuilder.class);
-
- static final ExpressionStringBuilder INSTANCE = new ExpressionStringBuilder();
-
- public static String toString(LogicalExpression expr) {
- StringBuilder sb = new StringBuilder();
- expr.accept(INSTANCE, sb);
- return sb.toString();
- }
-
- public static void toString(LogicalExpression expr, StringBuilder sb) {
- expr.accept(INSTANCE, sb);
- }
-
-
- @Override
- public Void visitFunctionCall(FunctionCall call, StringBuilder sb) throws RuntimeException {
- ImmutableList<LogicalExpression> args = call.args;
- sb.append(call.getName());
- sb.append("(");
- for (int i = 0; i < args.size(); i++) {
- if (i != 0) {
- sb.append(", ");
- }
- args.get(i).accept(this, sb);
- }
- sb.append(") ");
- return null;
- }
-
- @Override
- public Void visitBooleanOperator(BooleanOperator op, StringBuilder sb) throws RuntimeException {
- return visitFunctionCall(op, sb);
- }
-
- @Override
- public Void visitFunctionHolderExpression(FunctionHolderExpression holder, StringBuilder sb) throws RuntimeException {
- ImmutableList<LogicalExpression> args = holder.args;
- sb.append(holder.getName());
- sb.append("(");
- for (int i = 0; i < args.size(); i++) {
- if (i != 0) {
- sb.append(", ");
- }
- args.get(i).accept(this, sb);
- }
- sb.append(") ");
- return null;
- }
-
- @Override
- public Void visitIfExpression(IfExpression ifExpr, StringBuilder sb) throws RuntimeException {
-
- // serialize the if expression
- sb.append(" ( ");
- IfCondition c = ifExpr.ifCondition;
- sb.append("if (");
- c.condition.accept(this, sb);
- sb.append(" ) then (");
- c.expression.accept(this, sb);
- sb.append(" ) ");
-
- sb.append(" else (");
- ifExpr.elseExpression.accept(this, sb);
- sb.append(" ) ");
- sb.append(" end ");
- sb.append(" ) ");
- return null;
- }
-
- @Override
- public Void visitSchemaPath(SchemaPath path, StringBuilder sb) throws RuntimeException {
- PathSegment seg = path.getRootSegment();
- if (seg.isArray()) {
- throw new IllegalStateException("Drill doesn't currently support top level arrays");
- }
- sb.append('`');
- sb.append(seg.getNameSegment().getPath());
- sb.append('`');
-
- while ( (seg = seg.getChild()) != null) {
- if (seg.isNamed()) {
- sb.append('.');
- sb.append('`');
- sb.append(seg.getNameSegment().getPath());
- sb.append('`');
- } else {
- sb.append('[');
- sb.append(seg.getArraySegment().getIndex());
- sb.append(']');
- }
- }
- return null;
- }
-
- @Override
- public Void visitLongConstant(LongExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append(lExpr.getLong());
- return null;
- }
-
- @Override
- public Void visitDateConstant(DateExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append("cast( ");
- sb.append(lExpr.getDate());
- sb.append(" as DATE)");
- return null;
- }
-
- @Override
- public Void visitTimeConstant(TimeExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append("cast( ");
- sb.append(lExpr.getTime());
- sb.append(" as TIME)");
- return null;
- }
-
- @Override
- public Void visitTimeStampConstant(TimeStampExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append("cast( ");
- sb.append(lExpr.getTimeStamp());
- sb.append(" as TIMESTAMP)");
- return null;
- }
-
- @Override
- public Void visitIntervalYearConstant(IntervalYearExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append("cast( '");
- sb.append(Period.months(lExpr.getIntervalYear()).toString());
- sb.append("' as INTERVALYEAR)");
- return null;
- }
-
- @Override
- public Void visitIntervalDayConstant(IntervalDayExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append("cast( '");
- sb.append(Period.days(lExpr.getIntervalDay()).plusMillis(lExpr.getIntervalMillis()).toString());
- sb.append("' as INTERVALDAY)");
- return null;
- }
-
- @Override
- public Void visitDecimal9Constant(Decimal9Expression decExpr, StringBuilder sb) throws RuntimeException {
- BigDecimal value = new BigDecimal(decExpr.getIntFromDecimal());
- sb.append((value.setScale(decExpr.getScale())).toString());
- return null;
- }
-
- @Override
- public Void visitDecimal18Constant(Decimal18Expression decExpr, StringBuilder sb) throws RuntimeException {
- BigDecimal value = new BigDecimal(decExpr.getLongFromDecimal());
- sb.append((value.setScale(decExpr.getScale())).toString());
- return null;
- }
-
- @Override
- public Void visitDecimal28Constant(Decimal28Expression decExpr, StringBuilder sb) throws RuntimeException {
- sb.append(decExpr.toString());
- return null;
- }
-
- @Override
- public Void visitDecimal38Constant(Decimal38Expression decExpr, StringBuilder sb) throws RuntimeException {
- sb.append(decExpr.getBigDecimal().toString());
- return null;
- }
-
- @Override
- public Void visitDoubleConstant(DoubleExpression dExpr, StringBuilder sb) throws RuntimeException {
- sb.append(dExpr.getDouble());
- return null;
- }
-
- @Override
- public Void visitBooleanConstant(BooleanExpression e, StringBuilder sb) throws RuntimeException {
- sb.append(e.getBoolean());
- return null;
- }
-
- @Override
- public Void visitQuotedStringConstant(QuotedString e, StringBuilder sb) throws RuntimeException {
- sb.append("'");
- sb.append(e.value);
- sb.append("'");
- return null;
- }
-
- @Override
- public Void visitConvertExpression(ConvertExpression e, StringBuilder sb) throws RuntimeException {
- sb.append(e.getConvertFunction()).append("(");
- e.getInput().accept(this, sb);
- sb.append(", '").append(e.getEncodingType()).append("')");
- return null;
- }
-
- @Override
- public Void visitCastExpression(CastExpression e, StringBuilder sb) throws RuntimeException {
- MajorType mt = e.getMajorType();
-
- sb.append("cast( (");
- e.getInput().accept(this, sb);
- sb.append(" ) as ");
- sb.append(mt.getMinorType().name());
-
- switch(mt.getMinorType()) {
- case FLOAT4:
- case FLOAT8:
- case BIT:
- case INT:
- case TINYINT:
- case SMALLINT:
- case BIGINT:
- case UINT1:
- case UINT2:
- case UINT4:
- case UINT8:
- case DATE:
- case TIMESTAMP:
- case TIMESTAMPTZ:
- case TIME:
- case INTERVAL:
- case INTERVALDAY:
- case INTERVALYEAR:
- // do nothing else.
- break;
- case VAR16CHAR:
- case VARBINARY:
- case VARCHAR:
- case FIXED16CHAR:
- case FIXEDBINARY:
- case FIXEDCHAR:
-
- // add size in parens
- sb.append("(");
- sb.append(mt.getWidth());
- sb.append(")");
- break;
- case DECIMAL9:
- case DECIMAL18:
- case DECIMAL28DENSE:
- case DECIMAL28SPARSE:
- case DECIMAL38DENSE:
- case DECIMAL38SPARSE:
-
- // add scale and precision
- sb.append("(");
- sb.append(mt.getPrecision());
- sb.append(", ");
- sb.append(mt.getScale());
- sb.append(")");
- break;
- default:
- throw new UnsupportedOperationException(String.format("Unable to convert cast expression %s into string.", e));
- }
- sb.append(" )");
- return null;
- }
-
- @Override
- public Void visitFloatConstant(FloatExpression fExpr, StringBuilder sb) throws RuntimeException {
- sb.append(fExpr.getFloat());
- return null;
- }
-
- @Override
- public Void visitIntConstant(IntExpression intExpr, StringBuilder sb) throws RuntimeException {
- sb.append(intExpr.getInt());
- return null;
- }
-
- @Override
- public Void visitNullConstant(TypedNullConstant e, StringBuilder sb) throws RuntimeException {
- sb.append("NULL");
- return null;
- }
-
- @Override
- public Void visitNullExpression(NullExpression e, StringBuilder sb) throws RuntimeException {
- sb.append("NULL");
- return null;
- }
-
- @Override
- public Void visitUnknown(LogicalExpression e, StringBuilder sb) {
- sb.append(e.toString());
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
deleted file mode 100644
index 7d0e86f..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.io.IOException;
-
-import org.apache.drill.common.expression.FieldReference.De;
-import org.apache.drill.common.expression.FieldReference.Se;
-import org.apache.drill.common.expression.PathSegment.NameSegment;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-
-@JsonSerialize(using = Se.class)
-@JsonDeserialize(using = De.class)
-public class FieldReference extends SchemaPath {
- MajorType overrideType;
-
- public FieldReference(SchemaPath sp) {
- super(sp);
- checkData();
- }
-
- private void checkData() {
- if (getRootSegment().getChild() != null) {
- throw new UnsupportedOperationException("Field references must be singular names.");
- }
-
- }
-
-
- private void checkSimpleString(CharSequence value) {
- if (value.toString().contains(".")) {
- throw new UnsupportedOperationException(
- String.format(
- "Unhandled field reference \"%s\"; a field reference identifier"
- + " must not have the form of a qualified name (i.e., with \".\").",
- value));
- }
- }
-
- public FieldReference(CharSequence value) {
- this(value, ExpressionPosition.UNKNOWN);
- checkSimpleString(value);
- }
-
- public static FieldReference getWithQuotedRef(CharSequence safeString) {
- return new FieldReference(safeString, ExpressionPosition.UNKNOWN, false);
- }
-
-
- public FieldReference(CharSequence value, ExpressionPosition pos) {
- this(value, pos, true);
- }
-
- public FieldReference(CharSequence value, ExpressionPosition pos, boolean check) {
- super(new NameSegment(value), pos);
- if (check) {
- checkData();
- checkSimpleString(value);
- }
-
- }
-
- public FieldReference(String value, ExpressionPosition pos, MajorType dataType) {
- this(value, pos);
- this.overrideType = dataType;
- }
-
- @Override
- public MajorType getMajorType() {
- if (overrideType == null) {
- return super.getMajorType();
- } else {
- return overrideType;
- }
- }
-
- public static class De extends StdDeserializer<FieldReference> {
-
- public De() {
- super(FieldReference.class);
- }
-
- @Override
- public FieldReference deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
- JsonProcessingException {
- String ref = this._parseString(jp, ctxt);
- ref = ref.replace("`", "");
- return new FieldReference(ref, ExpressionPosition.UNKNOWN, false);
- }
-
- }
-
- public static class Se extends StdSerializer<FieldReference> {
-
- public Se() {
- super(FieldReference.class);
- }
-
- @Override
- public void serialize(FieldReference value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
- JsonGenerationException {
- jgen.writeString('`' + value.getRootSegment().getNameSegment().getPath() + '`');
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java b/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java
deleted file mode 100644
index 938ab0a..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-public class FunctionCall extends LogicalExpressionBase implements Iterable<LogicalExpression> {
- private final String name;
- public final ImmutableList<LogicalExpression> args;
- private final ExpressionPosition pos;
-
- public FunctionCall(String name, List<LogicalExpression> args, ExpressionPosition pos) {
- super(pos);
- this.name = name;
-
- if (args == null) {
- args = Lists.newArrayList();
- }
-
- if (!(args instanceof ImmutableList)) {
- args = ImmutableList.copyOf(args);
- }
- this.args = (ImmutableList<LogicalExpression>) args;
- this.pos = pos;
- }
-
- public String getName() {
- return name;
- }
-
- @Override
- public ExpressionPosition getPosition() {
- return pos;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E{
- return visitor.visitFunctionCall(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return args.iterator();
- }
-
- @Override
- public MajorType getMajorType() {
- throw new UnsupportedOperationException(
- "No return type as FunctionCall is not a materialized expression");
- }
-
- @Override
- public String toString() {
- final int maxLen = 10;
- return "FunctionCall [func=" + name + ", args="
- + (args != null ? args.subList(0, Math.min(args.size(), maxLen)) : null) + ", pos=" + pos + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java b/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
deleted file mode 100644
index 9128a78..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import com.google.common.collect.Lists;
-
-public class FunctionCallFactory {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionCallFactory.class);
-
- private static Map<String, String> opToFuncTable = new HashMap<>();
-
- static {
- opToFuncTable.put("+", "add");
- opToFuncTable.put("-", "subtract");
- opToFuncTable.put("/", "divide");
- opToFuncTable.put("*", "multiply");
- opToFuncTable.put("%", "modulo");
- opToFuncTable.put("^", "xor");
- opToFuncTable.put("||", "concatOperator");
- opToFuncTable.put("or", "booleanOr");
- opToFuncTable.put("and", "booleanAnd");
- opToFuncTable.put(">", "greater_than");
- opToFuncTable.put("<", "less_than");
- opToFuncTable.put("==", "equal");
- opToFuncTable.put("=", "equal");
- opToFuncTable.put("!=", "not_equal");
- opToFuncTable.put("<>", "not_equal");
- opToFuncTable.put(">=", "greater_than_or_equal_to");
- opToFuncTable.put("<=", "less_than_or_equal_to");
- opToFuncTable.put("is null", "isnull");
- opToFuncTable.put("is not null", "isnotnull");
- opToFuncTable.put("is true", "istrue");
- opToFuncTable.put("is not true", "isnottrue");
- opToFuncTable.put("is false", "isfalse");
- opToFuncTable.put("is not false", "isnotfalse");
- opToFuncTable.put("similar to", "similar_to");
-
- opToFuncTable.put("!", "not");
- opToFuncTable.put("u-", "negative");
- }
-
- private static String replaceOpWithFuncName(String op) {
- return (opToFuncTable.containsKey(op)) ? (opToFuncTable.get(op)) : op;
- }
-
- public static boolean isBooleanOperator(String funcName) {
- String opName = replaceOpWithFuncName(funcName);
- return opName.equals("booleanAnd") || opName.equals("booleanOr");
- }
-
- /*
- * create a cast function.
- * arguments : type -- targetType
- * ep -- input expression position
- * expr -- input expression
- */
- public static LogicalExpression createCast(MajorType type, ExpressionPosition ep, LogicalExpression expr){
- return new CastExpression(expr, type, ep);
- }
-
- public static LogicalExpression createConvert(String function, String conversionType, LogicalExpression expr, ExpressionPosition ep) {
- return new ConvertExpression(function, conversionType, expr, ep);
- }
-
- public static LogicalExpression createExpression(String functionName, List<LogicalExpression> args){
- return createExpression(functionName, ExpressionPosition.UNKNOWN, args);
- }
-
- public static LogicalExpression createExpression(String functionName, ExpressionPosition ep, List<LogicalExpression> args){
- String name = replaceOpWithFuncName(functionName);
- if (isBooleanOperator(name)) {
- return new BooleanOperator(name, args, ep);
- } else {
- return new FunctionCall(name, args, ep);
- }
- }
-
- public static LogicalExpression createExpression(String functionName, ExpressionPosition ep, LogicalExpression... e){
- return createExpression(functionName, ep, Lists.newArrayList(e));
- }
-
- public static LogicalExpression createBooleanOperator(String functionName, List<LogicalExpression> args){
- return createBooleanOperator(functionName, ExpressionPosition.UNKNOWN, args);
- }
-
- public static LogicalExpression createBooleanOperator(String functionName, ExpressionPosition ep, List<LogicalExpression> args){
- return new BooleanOperator(replaceOpWithFuncName(functionName), args, ep);
- }
-
- public static LogicalExpression createByOp(List<LogicalExpression> args, ExpressionPosition ep, List<String> opTypes) {
- if (args.size() == 1) {
- return args.get(0);
- }
-
- if (args.size() - 1 != opTypes.size()) {
- throw new DrillRuntimeException("Must receive one more expression then the provided number of operators.");
- }
-
- LogicalExpression first = args.get(0);
- for (int i = 0; i < opTypes.size(); i++) {
- List<LogicalExpression> l2 = new ArrayList<LogicalExpression>();
- l2.add(first);
- l2.add(args.get(i + 1));
- first = createExpression(opTypes.get(i), ep, l2);
- }
- return first;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java b/common/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
deleted file mode 100644
index d5497f7..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.fn.FuncHolder;
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-public abstract class FunctionHolderExpression extends LogicalExpressionBase {
- public final ImmutableList<LogicalExpression> args;
- public final String nameUsed;
-
- public FunctionHolderExpression(String nameUsed, ExpressionPosition pos, List<LogicalExpression> args) {
- super(pos);
- if (args == null) {
- args = Lists.newArrayList();
- }
-
- if (!(args instanceof ImmutableList)) {
- args = ImmutableList.copyOf(args);
- }
- this.args = (ImmutableList<LogicalExpression>) args;
- this.nameUsed = nameUsed;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitFunctionHolderExpression(this, value);
- }
-
- /**
- * A function can have multiple names, it returns the function name used in the query
- * @return
- */
- public String getName() {
- return nameUsed;
- }
-
- /**
- * constant input expected for i'th argument?
- * @param i
- * @return
- */
- public abstract boolean argConstantOnly(int i);
-
- /**
- * @return aggregating function or not
- */
- public abstract boolean isAggregating();
-
- /**
- * is the function output non-deterministic?
- */
- public abstract boolean isRandom();
-
- /**
- * @ return a copy of FunctionHolderExpression, with passed in argument list.
- */
- public abstract FunctionHolderExpression copy(List<LogicalExpression> args);
-
- /** Return the underlying function implementation holder. */
- public abstract FuncHolder getHolder();
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/FunctionName.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionName.java b/common/src/main/java/org/apache/drill/common/expression/FunctionName.java
deleted file mode 100644
index 2164cb2..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionName.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.TYPE})
-public @interface FunctionName {
- String value();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/IfExpression.java b/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
deleted file mode 100644
index e85caa0..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/IfExpression.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import com.google.common.collect.Sets;
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class IfExpression extends LogicalExpressionBase {
- static final Logger logger = LoggerFactory.getLogger(IfExpression.class);
-
- public final IfCondition ifCondition;
- public final LogicalExpression elseExpression;
- public final MajorType outputType;
-
- private IfExpression(ExpressionPosition pos, IfCondition conditions, LogicalExpression elseExpression, MajorType outputType) {
- super(pos);
- this.ifCondition = conditions;
- this.elseExpression = elseExpression;
- this.outputType = outputType;
- }
-
- public static class IfCondition{
- public final LogicalExpression condition;
- public final LogicalExpression expression;
-
- public IfCondition(LogicalExpression condition, LogicalExpression expression) {
- //logger.debug("Generating IfCondition {}, {}", condition, expression);
-
- this.condition = condition;
- this.expression = expression;
- }
-
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitIfExpression(this, value);
- }
-
- public static class Builder {
- IfCondition conditions;
- private LogicalExpression elseExpression;
- private ExpressionPosition pos = ExpressionPosition.UNKNOWN;
- private MajorType outputType;
-
- public Builder setPosition(ExpressionPosition pos) {
- this.pos = pos;
- return this;
- }
-
- public Builder setElse(LogicalExpression elseExpression) {
- this.elseExpression = elseExpression;
- return this;
- }
-
- public Builder setIfCondition(IfCondition conditions) {
- this.conditions = conditions;
- return this;
- }
-
- public Builder setOutputType(MajorType outputType) {
- this.outputType = outputType;
- return this;
- }
-
- public IfExpression build(){
- Preconditions.checkNotNull(pos);
- Preconditions.checkNotNull(conditions);
- return new IfExpression(pos, conditions, elseExpression, outputType);
- }
-
- }
-
- @Override
- public MajorType getMajorType() {
- if (outputType != null) {
- return outputType;
- }
-
- MajorType majorType = elseExpression.getMajorType();
- if (majorType.getMinorType() == MinorType.UNION) {
- Set<MinorType> subtypes = Sets.newHashSet();
- for (MinorType subtype : majorType.getSubTypeList()) {
- subtypes.add(subtype);
- }
- for (MinorType subtype : ifCondition.expression.getMajorType().getSubTypeList()) {
- subtypes.add(subtype);
- }
- MajorType.Builder builder = MajorType.newBuilder().setMinorType(MinorType.UNION).setMode(DataMode.OPTIONAL);
- for (MinorType subtype : subtypes) {
- builder.addSubType(subtype);
- }
- return builder.build();
- }
- if (majorType.getMode() == DataMode.OPTIONAL) {
- return majorType;
- }
-
- if (ifCondition.expression.getMajorType().getMode() == DataMode.OPTIONAL) {
- assert ifCondition.expression.getMajorType().getMinorType() == majorType.getMinorType();
-
- return ifCondition.expression.getMajorType();
- }
-
- return majorType;
- }
-
- public static Builder newBuilder() {
- return new Builder();
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- List<LogicalExpression> children = Lists.newLinkedList();
-
- children.add(ifCondition.condition);
- children.add(ifCondition.expression);
- children.add(this.elseExpression);
- return children.iterator();
- }
-
- @Override
- public int getCumulativeCost() {
- // return the average cost of operands for a boolean "and" | "or"
- int cost = this.getSelfCost();
-
- int i = 0;
- for (LogicalExpression e : this) {
- cost += e.getCumulativeCost();
- i++;
- }
-
- return (int) (cost / i) ;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java b/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
deleted file mode 100644
index a74ae29..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.io.IOException;
-
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.RecognitionException;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
-import org.apache.drill.common.expression.parser.ExprParser.parse_return;
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-
-//@JsonDeserialize(using = LogicalExpression.De.class) // Excluded as we need to register this with the DrillConfig.
-@JsonSerialize(using = LogicalExpression.Se.class)
-public interface LogicalExpression extends Iterable<LogicalExpression>{
- static final Logger logger = LoggerFactory.getLogger(LogicalExpression.class);
-
- public abstract MajorType getMajorType();
-
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E;
-
- public ExpressionPosition getPosition();
-
- public int getSelfCost();
- public int getCumulativeCost();
-
- public static class De extends StdDeserializer<LogicalExpression> {
- DrillConfig config;
-
- public De(DrillConfig config) {
- super(LogicalExpression.class);
- this.config = config;
- }
-
- @Override
- public LogicalExpression deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
- JsonProcessingException {
- String expr = jp.getText();
-
- if (expr == null || expr.isEmpty()) {
- return null;
- }
- try {
- // logger.debug("Parsing expression string '{}'", expr);
- ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
- CommonTokenStream tokens = new CommonTokenStream(lexer);
- ExprParser parser = new ExprParser(tokens);
-
- //TODO: move functionregistry and error collector to injectables.
- //ctxt.findInjectableValue(valueId, forProperty, beanInstance)
- parse_return ret = parser.parse();
-
- // ret.e.resolveAndValidate(expr, errorCollector);
- return ret.e;
- } catch (RecognitionException e) {
- throw new RuntimeException(e);
- }
- }
-
- }
-
- public static class Se extends StdSerializer<LogicalExpression> {
-
- protected Se() {
- super(LogicalExpression.class);
- }
-
- @Override
- public void serialize(LogicalExpression value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
- JsonGenerationException {
- StringBuilder sb = new StringBuilder();
- ExpressionStringBuilder esb = new ExpressionStringBuilder();
- value.accept(esb, sb);
- jgen.writeString(sb.toString());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java b/common/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java
deleted file mode 100644
index 7dfe4a2..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/LogicalExpressionBase.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-
-@JsonPropertyOrder({ "type" })
-public abstract class LogicalExpressionBase implements LogicalExpression {
-
- private final ExpressionPosition pos;
-
- protected LogicalExpressionBase(ExpressionPosition pos) {
- super();
- this.pos = pos;
- }
-
- @Override
- public ExpressionPosition getPosition() {
- return pos;
- }
-
- protected void i(StringBuilder sb, int indent) {
- for (int i = 0; i < indent; i++){
- sb.append(" ");
- }
- }
-
- @Override
- public MajorType getMajorType() {
- throw new UnsupportedOperationException(String.format("The type of %s doesn't currently support LogicalExpression.getDataType().", this.getClass().getCanonicalName()));
- }
-
- @JsonProperty("type")
- public String getDescription() {
- return this.getClass().getSimpleName();
- }
-
- @JsonIgnore
- public int getSelfCost() {
- return 0;
- }
-
- @JsonIgnore
- public int getCumulativeCost() {
- int cost = this.getSelfCost();
-
- for (LogicalExpression e : this) {
- cost += e.getCumulativeCost();
- }
-
- return cost;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/NullExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/NullExpression.java b/common/src/main/java/org/apache/drill/common/expression/NullExpression.java
deleted file mode 100644
index e00399f..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/NullExpression.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-
-import com.google.common.collect.Iterators;
-
-public class NullExpression implements LogicalExpression{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NullExpression.class);
-
- public static final NullExpression INSTANCE = new NullExpression();
-
- final MajorType t = Types.optional(MinorType.NULL);
-
- @Override
- public MajorType getMajorType() {
- return t;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitNullExpression(this, value);
- }
-
- @Override
- public ExpressionPosition getPosition() {
- return ExpressionPosition.UNKNOWN;
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- public int getSelfCost() { return 0 ; }
-
- public int getCumulativeCost() { return 0; }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
deleted file mode 100644
index a03524c..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-
-public interface OutputTypeDeterminer {
-
- public static OutputTypeDeterminer FIXED_BIT = new FixedType(Types.required(MinorType.BIT));
- public static OutputTypeDeterminer FIXED_INT = new FixedType(Types.required(MinorType.INT));
- public static OutputTypeDeterminer FIXED_BIGINT = new FixedType(Types.required(MinorType.BIGINT));
- public static OutputTypeDeterminer FIXED_FLOAT4 = new FixedType(Types.required(MinorType.FLOAT4));
- public static OutputTypeDeterminer FIXED_FLOAT8 = new FixedType(Types.required(MinorType.FLOAT8));
- public static OutputTypeDeterminer FIXED_VARCHAR = new FixedType(Types.required(MinorType.VARCHAR));
- public static OutputTypeDeterminer FIXED_VARBINARY = new FixedType(Types.required(MinorType.VARBINARY));
- public static OutputTypeDeterminer FIXED_VAR16CHAR = new FixedType(Types.required(MinorType.VAR16CHAR));
-
- public MajorType getOutputType(List<LogicalExpression> expressions);
-
-
- public static class FixedType implements OutputTypeDeterminer{
- public MajorType outputType;
-
-
- public FixedType(MajorType outputType) {
- super();
- this.outputType = outputType;
- }
-
-
- @Override
- public MajorType getOutputType(List<LogicalExpression> expressions) {
- return outputType;
- }
-
- }
-
- public static class SameAsFirstInput implements OutputTypeDeterminer{
- @Override
- public MajorType getOutputType(List<LogicalExpression> expressions) {
- return expressions.get(0).getMajorType();
- }
- }
-
- public static class SameAsAnySoft implements OutputTypeDeterminer{
- @Override
- public MajorType getOutputType(List<LogicalExpression> expressions) {
- for(LogicalExpression e : expressions){
- if(e.getMajorType().getMode() == DataMode.OPTIONAL){
- return e.getMajorType();
- }
- }
- return expressions.get(0).getMajorType();
- }
- }
-
- public static class NullIfNullType implements OutputTypeDeterminer{
- public MinorType outputMinorType;
-
-
- public NullIfNullType(MinorType outputType) {
- super();
- this.outputMinorType = outputType;
- }
-
- @Override
- public MajorType getOutputType(List<LogicalExpression> expressions) {
- for(LogicalExpression e : expressions){
- if(e.getMajorType().getMode() == DataMode.OPTIONAL){
- return Types.optional(outputMinorType);
- }
- }
- return Types.required(outputMinorType);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java b/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
deleted file mode 100644
index 744a07f..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-public abstract class PathSegment{
-
- PathSegment child;
-
- int hash;
-
- public abstract PathSegment cloneWithNewChild(PathSegment segment);
- @Override
- public abstract PathSegment clone();
-
- public static final class ArraySegment extends PathSegment {
- private final int index;
-
- public ArraySegment(String numberAsText, PathSegment child) {
- this(Integer.parseInt(numberAsText), child);
- }
-
- public ArraySegment(int index, PathSegment child) {
- this.child = child;
- this.index = index;
- assert index >=0;
- }
-
- public ArraySegment(PathSegment child) {
- this.child = child;
- this.index = -1;
- }
-
- public boolean hasIndex() {
- return index != -1;
- }
-
- public ArraySegment(int index) {
- if (index < 0 ) {
- throw new IllegalArgumentException();
- }
- this.index = index;
- }
-
- public int getIndex() {
- return index;
- }
-
- @Override
- public boolean isArray() {
- return true;
- }
-
- @Override
- public boolean isNamed() {
- return false;
- }
-
- @Override
- public ArraySegment getArraySegment() {
- return this;
- }
-
- @Override
- public String toString() {
- return "ArraySegment [index=" + index + ", getChild()=" + getChild() + "]";
- }
-
- @Override
- public int segmentHashCode() {
- return index;
- }
-
- @Override
- public boolean segmentEquals(PathSegment obj) {
- if (this == obj) {
- return true;
- } else if (obj == null) {
- return false;
- } else if (obj instanceof ArraySegment) {
- return index == ((ArraySegment)obj).getIndex();
- }
- return false;
- }
-
- @Override
- public PathSegment clone() {
- PathSegment seg = index < 0 ? new ArraySegment(null) : new ArraySegment(index);
- if (child != null) {
- seg.setChild(child.clone());
- }
- return seg;
- }
-
- @Override
- public ArraySegment cloneWithNewChild(PathSegment newChild) {
- ArraySegment seg = index < 0 ? new ArraySegment(null) : new ArraySegment(index);
- if (child != null) {
- seg.setChild(child.cloneWithNewChild(newChild));
- } else {
- seg.setChild(newChild);
- }
- return seg;
- }
- }
-
-
- public static final class NameSegment extends PathSegment {
- private final String path;
-
- public NameSegment(CharSequence n, PathSegment child) {
- this.child = child;
- this.path = n.toString();
- }
-
- public NameSegment(CharSequence n) {
- this.path = n.toString();
- }
-
- public String getPath() {
- return path;
- }
-
- @Override
- public boolean isArray() {
- return false;
- }
-
- @Override
- public boolean isNamed() {
- return true;
- }
-
- @Override
- public NameSegment getNameSegment() {
- return this;
- }
-
- @Override
- public String toString() {
- return "NameSegment [path=" + path + ", getChild()=" + getChild() + "]";
- }
-
- @Override
- public int segmentHashCode() {
- return ((path == null) ? 0 : path.toLowerCase().hashCode());
- }
-
- @Override
- public boolean segmentEquals(PathSegment obj) {
- if (this == obj) {
- return true;
- } else if (obj == null) {
- return false;
- } else if (getClass() != obj.getClass()) {
- return false;
- }
-
- NameSegment other = (NameSegment) obj;
- if (path == null) {
- return other.path == null;
- }
- return path.equalsIgnoreCase(other.path);
- }
-
- @Override
- public NameSegment clone() {
- NameSegment s = new NameSegment(this.path);
- if (child != null) {
- s.setChild(child.clone());
- }
- return s;
- }
-
- @Override
- public NameSegment cloneWithNewChild(PathSegment newChild) {
- NameSegment s = new NameSegment(this.path);
- if (child != null) {
- s.setChild(child.cloneWithNewChild(newChild));
- } else {
- s.setChild(newChild);
- }
- return s;
- }
-
- }
-
- public NameSegment getNameSegment() {
- throw new UnsupportedOperationException();
- }
-
- public ArraySegment getArraySegment() {
- throw new UnsupportedOperationException();
- }
-
- public abstract boolean isArray();
- public abstract boolean isNamed();
-
- public boolean isLastPath() {
- return child == null;
- }
-
- public PathSegment getChild() {
- return child;
- }
-
- void setChild(PathSegment child) {
- this.child = child;
- }
-
- protected abstract int segmentHashCode();
- protected abstract boolean segmentEquals(PathSegment other);
-
- @Override
- public int hashCode() {
- int h = hash;
- if (h == 0) {
- h = segmentHashCode();
- h = 31*h + ((child == null) ? 0 : child.hashCode());
- hash = h;
- }
- return h;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
-
- PathSegment other = (PathSegment) obj;
- if (!segmentEquals(other)) {
- return false;
- } else if (child == null) {
- return (other.child == null);
- } else {
- return child.equals(other.child);
- }
- }
-
- /**
- * Check if another path is contained in this one. This is useful for 2 cases. The first
- * is checking if the other is lower down in the tree, below this path. The other is if
- * a path is actually contained above the current one.
- *
- * Examples:
- * [a] . contains( [a.b.c] ) returns true
- * [a.b.c] . contains( [a] ) returns true
- *
- * This behavior is used for cases like scanning json in an event based fashion, when we arrive at
- * a node in a complex type, we will know the complete path back to the root. This method can
- * be used to determine if we need the data below. This is true in both the cases where the
- * column requested from the user is below the current node (in which case we may ignore other nodes
- * further down the tree, while keeping others). This is also the case if the requested path is further
- * up the tree, if we know we are at position a.b.c and a.b was a requested column, we need to scan
- * all of the data at and below the current a.b.c node.
- *
- * @param otherSeg - path segment to check if it is contained below this one.
- * @return - is this a match
- */
- public boolean contains(PathSegment otherSeg) {
- if (this == otherSeg) {
- return true;
- }
- if (otherSeg == null) {
- return false;
- }
- // TODO - fix this in the future to match array segments are part of the path
- // the current behavior to always return true when we hit an array may be useful in some cases,
- // but we can get better performance in the JSON reader if we avoid reading unwanted elements in arrays
- if (otherSeg.isArray() || this.isArray()) {
- return true;
- }
- if (getClass() != otherSeg.getClass()) {
- return false;
- }
-
- if (!segmentEquals(otherSeg)) {
- return false;
- }
- else if (child == null || otherSeg.child == null) {
- return true;
- } else {
- return child.contains(otherSeg.child);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
deleted file mode 100644
index f5fc687..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.RecognitionException;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.PathSegment.ArraySegment;
-import org.apache.drill.common.expression.PathSegment.NameSegment;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
-import org.apache.drill.common.expression.parser.ExprParser.parse_return;
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.proto.UserBitShared.NamePart;
-import org.apache.drill.exec.proto.UserBitShared.NamePart.Type;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-
-public class SchemaPath extends LogicalExpressionBase {
-
- private final NameSegment rootSegment;
-
- public static SchemaPath getSimplePath(String name) {
- return getCompoundPath(name);
- }
-
- public static SchemaPath getCompoundPath(String... strings) {
- NameSegment s = null;
- // loop through strings in reverse order
- for (int i = strings.length - 1; i >= 0; i--) {
- s = new NameSegment(strings[i], s);
- }
- return new SchemaPath(s);
- }
-
- @SuppressWarnings("unused")
- public PathSegment getLastSegment() {
- PathSegment s= rootSegment;
- while (s.getChild() != null) {
- s = s.getChild();
- }
- return s;
- }
-
- @Deprecated
- public SchemaPath(String simpleName, ExpressionPosition pos) {
- super(pos);
- this.rootSegment = new NameSegment(simpleName);
- if (simpleName.contains(".")) {
- throw new IllegalStateException("This is deprecated and only supports simpe paths.");
- }
- }
-
-
- public NamePart getAsNamePart() {
- return getNamePart(rootSegment);
- }
-
- private static NamePart getNamePart(PathSegment s) {
- if (s == null) {
- return null;
- }
- NamePart.Builder b = NamePart.newBuilder();
- if (s.getChild() != null) {
- b.setChild(getNamePart(s.getChild()));
- }
-
- if (s.isArray()) {
- if (s.getArraySegment().hasIndex()) {
- throw new IllegalStateException("You cannot convert a indexed schema path to a NamePart. NameParts can only reference Vectors, not individual records or values.");
- }
- b.setType(Type.ARRAY);
- } else {
- b.setType(Type.NAME);
- b.setName(s.getNameSegment().getPath());
- }
- return b.build();
- }
-
- private static PathSegment getPathSegment(NamePart n) {
- PathSegment child = n.hasChild() ? getPathSegment(n.getChild()) : null;
- if (n.getType() == Type.ARRAY) {
- return new ArraySegment(child);
- } else {
- return new NameSegment(n.getName(), child);
- }
- }
-
- public static SchemaPath create(NamePart namePart) {
- Preconditions.checkArgument(namePart.getType() == NamePart.Type.NAME);
- return new SchemaPath((NameSegment) getPathSegment(namePart));
- }
-
- /**
- * A simple is a path where there are no repeated elements outside the lowest level of the path.
- * @return Whether this path is a simple path.
- */
- public boolean isSimplePath() {
- PathSegment seg = rootSegment;
- while (seg != null) {
- if (seg.isArray() && !seg.isLastPath()) {
- return false;
- }
- seg = seg.getChild();
- }
- return true;
- }
-
-
- public SchemaPath(SchemaPath path) {
- super(path.getPosition());
- this.rootSegment = path.rootSegment;
- }
-
- public SchemaPath(NameSegment rootSegment) {
- super(ExpressionPosition.UNKNOWN);
- this.rootSegment = rootSegment;
- }
-
- public SchemaPath(NameSegment rootSegment, ExpressionPosition pos) {
- super(pos);
- this.rootSegment = rootSegment;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitSchemaPath(this, value);
- }
-
- public SchemaPath getChild(String childPath) {
- NameSegment newRoot = rootSegment.cloneWithNewChild(new NameSegment(childPath));
- return new SchemaPath(newRoot);
- }
-
- @SuppressWarnings("unused")
- public SchemaPath getUnindexedArrayChild() {
- NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(null));
- return new SchemaPath(newRoot);
- }
-
- public SchemaPath getChild(int index) {
- NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(index));
- return new SchemaPath(newRoot);
- }
-
- public NameSegment getRootSegment() {
- return rootSegment;
- }
-
- @Override
- public MajorType getMajorType() {
- return Types.LATE_BIND_TYPE;
- }
-
- @Override
- public int hashCode() {
- return ((rootSegment == null) ? 0 : rootSegment.hashCode());
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof SchemaPath)) {
- return false;
- }
-
- SchemaPath other = (SchemaPath) obj;
- if (rootSegment == null) {
- return (other.rootSegment == null);
- }
- return rootSegment.equals(other.rootSegment);
- }
-
- public boolean contains(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof SchemaPath)) {
- return false;
- }
-
- SchemaPath other = (SchemaPath) obj;
- return rootSegment == null || rootSegment.contains(other.rootSegment);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- @Override
- public String toString() {
- return ExpressionStringBuilder.toString(this);
- }
-
- public String toExpr() {
- return ExpressionStringBuilder.toString(this);
- }
-
- public String getAsUnescapedPath() {
- StringBuilder sb = new StringBuilder();
- PathSegment seg = getRootSegment();
- if (seg.isArray()) {
- throw new IllegalStateException("Drill doesn't currently support top level arrays");
- }
- sb.append(seg.getNameSegment().getPath());
-
- while ( (seg = seg.getChild()) != null) {
- if (seg.isNamed()) {
- sb.append('.');
- sb.append(seg.getNameSegment().getPath());
- } else {
- sb.append('[');
- sb.append(seg.getArraySegment().getIndex());
- sb.append(']');
- }
- }
- return sb.toString();
- }
-
- public static class De extends StdDeserializer<SchemaPath> {
-
- public De() {
- super(LogicalExpression.class);
- }
-
- @Override
- public SchemaPath deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
- String expr = jp.getText();
-
- if (expr == null || expr.isEmpty()) {
- return null;
- }
- try {
- // logger.debug("Parsing expression string '{}'", expr);
- ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
- CommonTokenStream tokens = new CommonTokenStream(lexer);
- ExprParser parser = new ExprParser(tokens);
-
- //TODO: move functionregistry and error collector to injectables.
- //ctxt.findInjectableValue(valueId, forProperty, beanInstance)
- parse_return ret = parser.parse();
-
- // ret.e.resolveAndValidate(expr, errorCollector);
- if (ret.e instanceof SchemaPath) {
- return (SchemaPath) ret.e;
- } else {
- throw new IllegalStateException("Schema path is not a valid format.");
- }
- } catch (RecognitionException e) {
- throw new RuntimeException(e);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java b/common/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java
deleted file mode 100644
index bd129d6..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-import com.google.common.collect.Iterators;
-
-public class TypedNullConstant extends LogicalExpressionBase {
-
- private final MajorType type;
-
- public TypedNullConstant(MajorType type) {
- super(null);
- this.type = TypeProtos.MajorType.newBuilder().mergeFrom(type).setMode(DataMode.OPTIONAL).build();
- }
-
- @Override
- public MajorType getMajorType() {
- return this.type;
- }
-
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitNullConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/ValidationError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ValidationError.java b/common/src/main/java/org/apache/drill/common/expression/ValidationError.java
deleted file mode 100644
index 195a6d8..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ValidationError.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-public class ValidationError {
-
-}
[23/45] drill git commit: DRILL-3987: (MOVE) Extract RPC,
memory-base and memory-impl as separate modules.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ListeningCommand.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ListeningCommand.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ListeningCommand.java
new file mode 100644
index 0000000..92022b0
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ListeningCommand.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+import com.google.protobuf.MessageLite;
+
+public abstract class ListeningCommand<T extends MessageLite, C extends RemoteConnection> implements RpcCommand<T, C> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ListeningCommand.class);
+
+ private final RpcOutcomeListener<T> listener;
+
+ public ListeningCommand(RpcOutcomeListener<T> listener) {
+ this.listener = listener;
+ }
+
+ public abstract void doRpcCall(RpcOutcomeListener<T> outcomeListener, C connection);
+
+ @Override
+ public void connectionAvailable(C connection) {
+
+ doRpcCall(new DeferredRpcOutcome(), connection);
+ }
+
+ @Override
+ public void connectionSucceeded(C connection) {
+ connectionAvailable(connection);
+ }
+
+ private class DeferredRpcOutcome implements RpcOutcomeListener<T> {
+
+ @Override
+ public void failed(RpcException ex) {
+ listener.failed(ex);
+ }
+
+ @Override
+ public void success(T value, ByteBuf buf) {
+ listener.success(value, buf);
+ }
+
+ @Override
+ public void interrupted(final InterruptedException e) {
+ listener.interrupted(e);
+ }
+ }
+
+ @Override
+ public void connectionFailed(FailureType type, Throwable t) {
+ listener.failed(RpcException.mapException(
+ String.format("Command failed while establishing connection. Failure type %s.", type), t));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/NamedThreadFactory.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/NamedThreadFactory.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/NamedThreadFactory.java
new file mode 100644
index 0000000..e7580b2
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/NamedThreadFactory.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * {@link ThreadFactory} for {@link ExecutorServices} that names threads sequentially.
+ * Creates Threads named with the prefix specified at construction time. Created threads
+ * have the daemon bit set and priority Thread.MAX_PRIORITY.
+ *
+ * <p>An instance creates names with an instance-specific prefix suffixed with sequential
+ * integers.</p>
+ *
+ * <p>Concurrency: See {@link newThread}.</p>
+ */
+public class NamedThreadFactory implements ThreadFactory {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NamedThreadFactory.class);
+ private final AtomicInteger nextId = new AtomicInteger(); // used to generate unique ids
+ private final String prefix;
+
+ /**
+ * Constructor.
+ *
+ * @param prefix the string prefix that will be used to name threads created by this factory
+ */
+ public NamedThreadFactory(final String prefix) {
+ this.prefix = prefix;
+ }
+
+ /**
+ * Creates a sequentially named thread running a given Runnable.
+ * <p>
+ * The thread's name will be this instance's prefix concatenated with
+ * this instance's next<sup><a href="#fn-1">*</a></sup> sequential integer.
+ * </p>
+ * <p>
+ * Concurrency: Thread-safe.
+ * </p>
+ * <p>
+ * (Concurrent calls get different numbers.
+ * Calls started after other calls complete get later/higher numbers than
+ * those other calls.
+ * </p>
+ * <p>
+ * <a name="fn-1" />*However, for concurrent calls, the order of numbers
+ * is not defined.)
+ */
+ @Override
+ public Thread newThread(final Runnable runnable) {
+ final Thread thread = new Thread(runnable, prefix + nextId.incrementAndGet());
+ thread.setDaemon(true);
+
+ try {
+ if (thread.getPriority() != Thread.MAX_PRIORITY) {
+ thread.setPriority(Thread.MAX_PRIORITY);
+ }
+ } catch (Exception ignored) {
+ // Doesn't matter even if failed to set.
+ logger.info("ignored exception " + ignored);
+ }
+ return thread;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/OutOfMemoryHandler.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/OutOfMemoryHandler.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/OutOfMemoryHandler.java
new file mode 100644
index 0000000..5d7db47
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/OutOfMemoryHandler.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+public interface OutOfMemoryHandler {
+
+ public static OutOfMemoryHandler DEFAULT_INSTANCE = new OutOfMemoryHandler() {
+ @Override
+ public void handle() {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ public void handle();
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/OutboundRpcMessage.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/OutboundRpcMessage.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/OutboundRpcMessage.java
new file mode 100644
index 0000000..5eda350
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/OutboundRpcMessage.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
+
+import com.google.common.collect.Lists;
+import com.google.protobuf.Internal.EnumLite;
+import com.google.protobuf.MessageLite;
+
+public class OutboundRpcMessage extends RpcMessage {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OutboundRpcMessage.class);
+
+ final MessageLite pBody;
+ public ByteBuf[] dBodies;
+
+
+
+ public OutboundRpcMessage(RpcMode mode, EnumLite rpcType, int coordinationId, MessageLite pBody, ByteBuf... dBodies) {
+ this(mode, rpcType.getNumber(), coordinationId, pBody, dBodies);
+ }
+
+
+ OutboundRpcMessage(RpcMode mode, int rpcTypeNumber, int coordinationId, MessageLite pBody, ByteBuf... dBodies) {
+ super(mode, rpcTypeNumber, coordinationId);
+ this.pBody = pBody;
+
+ // Netty doesn't traditionally release the reference on an unreadable buffer. However, we need to so that if we send a empty or unwritable buffer, we still release. otherwise we get weird memory leaks when sending empty vectors.
+ List<ByteBuf> bufs = Lists.newArrayList();
+ for (ByteBuf d : dBodies) {
+ if (d.readableBytes() == 0) {
+ d.release();
+ } else {
+ bufs.add(d);
+ }
+ }
+ this.dBodies = bufs.toArray(new ByteBuf[bufs.size()]);
+ }
+
+ @Override
+ public int getBodySize() {
+ int len = pBody.getSerializedSize();
+ len += RpcEncoder.getRawVarintSize(len);
+ len += getRawBodySize();
+ return len;
+ }
+
+ public int getRawBodySize() {
+ if (dBodies == null) {
+ return 0;
+ }
+ int len = 0;
+
+ for (int i = 0; i < dBodies.length; i++) {
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Reader Index {}, Writer Index {}", dBodies[i].readerIndex(), dBodies[i].writerIndex());
+ }
+ len += dBodies[i].readableBytes();
+ }
+ return len;
+ }
+
+ @Override
+ public String toString() {
+ return "OutboundRpcMessage [pBody=" + pBody + ", mode=" + mode + ", rpcType=" + rpcType + ", coordinationId="
+ + coordinationId + ", dBodies=" + Arrays.toString(dBodies) + "]";
+ }
+
+ @Override
+ void release() {
+ if (dBodies != null) {
+ for (ByteBuf b : dBodies) {
+ b.release();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/PositiveAtomicInteger.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/PositiveAtomicInteger.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/PositiveAtomicInteger.java
new file mode 100644
index 0000000..401663d
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/PositiveAtomicInteger.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/*
+ * An atomic integer that only ever returns 0 > MAX_INT and then starts over. Should never has a negative overflow.
+ */
+public class PositiveAtomicInteger {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PositiveAtomicInteger.class);
+
+ private final AtomicInteger internal = new AtomicInteger(0);
+
+ public int getNext(){
+ int i = internal.addAndGet(1);
+ if(i < 0){
+ return i + (-Integer.MIN_VALUE);
+ }else{
+ return i;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
new file mode 100644
index 0000000..3dfe03f
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.ByteToMessageDecoder;
+import io.netty.handler.codec.CorruptedFrameException;
+
+import java.util.List;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+
+import com.google.protobuf.CodedInputStream;
+import org.apache.drill.exec.exception.OutOfMemoryException;
+
+/**
+ * Modified version of {@link io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder} that avoids bytebuf copy.
+ * See the documentation there.
+ */
+public class ProtobufLengthDecoder extends ByteToMessageDecoder {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProtobufLengthDecoder.class);
+
+ private BufferAllocator allocator;
+ private OutOfMemoryHandler outOfMemoryHandler;
+
+ public ProtobufLengthDecoder(BufferAllocator allocator, OutOfMemoryHandler outOfMemoryHandler) {
+ super();
+ this.allocator = allocator;
+ this.outOfMemoryHandler = outOfMemoryHandler;
+ }
+
+
+ @Override
+ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
+ if (!ctx.channel().isOpen()) {
+ if (in.readableBytes() > 0) {
+ logger.info("Channel is closed, discarding remaining {} byte(s) in buffer.", in.readableBytes());
+ }
+ in.skipBytes(in.readableBytes());
+ return;
+ }
+
+ in.markReaderIndex();
+ final byte[] buf = new byte[5];
+ for (int i = 0; i < buf.length; i++) {
+ if (!in.isReadable()) {
+ in.resetReaderIndex();
+ return;
+ }
+
+ buf[i] = in.readByte();
+ if (buf[i] >= 0) {
+
+ int length = CodedInputStream.newInstance(buf, 0, i + 1).readRawVarint32();
+
+ if (length < 0) {
+ throw new CorruptedFrameException("negative length: " + length);
+ }
+ if (length == 0) {
+ throw new CorruptedFrameException("Received a message of length 0.");
+ }
+
+ if (in.readableBytes() < length) {
+ in.resetReaderIndex();
+ return;
+ } else {
+ // need to make buffer copy, otherwise netty will try to refill this buffer if we move the readerIndex forward...
+ // TODO: Can we avoid this copy?
+ ByteBuf outBuf;
+ try {
+ outBuf = allocator.buffer(length);
+ } catch (OutOfMemoryException e) {
+ logger.warn("Failure allocating buffer on incoming stream due to memory limits. Current Allocation: {}.", allocator.getAllocatedMemory());
+ in.resetReaderIndex();
+ outOfMemoryHandler.handle();
+ return;
+ }
+ outBuf.writeBytes(in, in.readerIndex(), length);
+
+ in.skipBytes(length);
+
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug(String.format(
+ "ReaderIndex is %d after length header of %d bytes and frame body of length %d bytes.",
+ in.readerIndex(), i + 1, length));
+ }
+
+ out.add(outBuf);
+ return;
+ }
+ }
+ }
+
+ // Couldn't find the byte whose MSB is off.
+ throw new CorruptedFrameException("length wider than 32-bit");
+
+ }
+
+ @Override
+ public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
+ ctx.fireChannelReadComplete();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java
new file mode 100644
index 0000000..d62b6f2
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java
@@ -0,0 +1,269 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.channel.ChannelFuture;
+import io.netty.util.concurrent.GenericFutureListener;
+
+import java.io.Closeable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.AbstractFuture;
+import com.google.protobuf.MessageLite;
+
+/**
+ * Manager all connections between two particular bits.
+ */
+public abstract class ReconnectingConnection<CONNECTION_TYPE extends RemoteConnection, OUTBOUND_HANDSHAKE extends MessageLite>
+ implements Closeable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReconnectingConnection.class);
+
+ private final AtomicReference<CONNECTION_TYPE> connectionHolder = new AtomicReference<CONNECTION_TYPE>();
+ private final String host;
+ private final int port;
+ private final OUTBOUND_HANDSHAKE handshake;
+
+ public ReconnectingConnection(OUTBOUND_HANDSHAKE handshake, String host, int port) {
+ Preconditions.checkNotNull(host);
+ Preconditions.checkArgument(port > 0);
+ this.host = host;
+ this.port = port;
+ this.handshake = handshake;
+ }
+
+ protected abstract BasicClient<?, CONNECTION_TYPE, OUTBOUND_HANDSHAKE, ?> getNewClient();
+
+ public <R extends MessageLite, C extends RpcCommand<R, CONNECTION_TYPE>> void runCommand(C cmd) {
+// if(logger.isDebugEnabled()) logger.debug(String.format("Running command %s sending to host %s:%d", cmd, host, port));
+ CONNECTION_TYPE connection = connectionHolder.get();
+ if (connection != null) {
+ if (connection.isActive()) {
+ cmd.connectionAvailable(connection);
+// logger.debug("Connection available and active, command run inline.");
+ return;
+ } else {
+ // remove the old connection. (don't worry if we fail since someone else should have done it.
+ connectionHolder.compareAndSet(connection, null);
+ }
+ }
+
+ /**
+ * We've arrived here without a connection, let's make sure only one of us makes a connection. (fyi, another
+ * endpoint could create a reverse connection
+ **/
+ synchronized (this) {
+ connection = connectionHolder.get();
+ if (connection != null) {
+ cmd.connectionAvailable(connection);
+
+ } else {
+// logger.debug("No connection active, opening client connection.");
+ BasicClient<?, CONNECTION_TYPE, OUTBOUND_HANDSHAKE, ?> client = getNewClient();
+ ConnectionListeningFuture<R, C> future = new ConnectionListeningFuture<R, C>(cmd);
+ client.connectAsClient(future, handshake, host, port);
+ future.waitAndRun();
+// logger.debug("Connection available and active, command now being run inline.");
+ }
+ return;
+
+ }
+ }
+
+ public class ConnectionListeningFuture<R extends MessageLite, C extends RpcCommand<R, CONNECTION_TYPE>> extends
+ AbstractFuture<CONNECTION_TYPE> implements RpcConnectionHandler<CONNECTION_TYPE> {
+
+ private C cmd;
+
+ public ConnectionListeningFuture(C cmd) {
+ super();
+ this.cmd = cmd;
+ }
+
+ /**
+ * Called by
+ */
+ public void waitAndRun() {
+ boolean isInterrupted = false;
+
+ // We want to wait for at least 120 secs when interrupts occur. Establishing a connection fails/succeeds quickly,
+ // So there is no point propagating the interruption as failure immediately.
+ long remainingWaitTimeMills = 120000;
+ long startTime = System.currentTimeMillis();
+
+ while(true) {
+ try {
+ // logger.debug("Waiting for connection.");
+ CONNECTION_TYPE connection = this.get(remainingWaitTimeMills, TimeUnit.MILLISECONDS);
+
+ if (connection == null) {
+ // logger.debug("Connection failed.");
+ } else {
+ // logger.debug("Connection received. {}", connection);
+ cmd.connectionSucceeded(connection);
+ // logger.debug("Finished connection succeeded activity.");
+ }
+ break;
+ } catch (final InterruptedException interruptEx) {
+ remainingWaitTimeMills -= (System.currentTimeMillis() - startTime);
+ startTime = System.currentTimeMillis();
+ isInterrupted = true;
+ if (remainingWaitTimeMills < 1) {
+ cmd.connectionFailed(FailureType.CONNECTION, interruptEx);
+ break;
+ }
+ // Ignore the interrupt and continue to wait until we elapse remainingWaitTimeMills.
+ } catch (final ExecutionException | TimeoutException ex) {
+ logger.error("Failed to establish connection", ex);
+ cmd.connectionFailed(FailureType.CONNECTION, ex);
+ break;
+ }
+ }
+
+ if (isInterrupted) {
+ // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
+ // interruption and respond to it if it wants to.
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ public void connectionFailed(org.apache.drill.exec.rpc.RpcConnectionHandler.FailureType type, Throwable t) {
+ set(null);
+ cmd.connectionFailed(type, t);
+ }
+
+ @Override
+ public void connectionSucceeded(CONNECTION_TYPE incoming) {
+ CONNECTION_TYPE connection = connectionHolder.get();
+ while (true) {
+ boolean setted = connectionHolder.compareAndSet(null, incoming);
+ if (setted) {
+ connection = incoming;
+ break;
+ }
+ connection = connectionHolder.get();
+ if (connection != null) {
+ break;
+ }
+ }
+
+ if (connection != incoming) {
+ // close the incoming because another channel was created in the mean time (unless this is a self connection).
+ logger.debug("Closing incoming connection because a connection was already set.");
+ incoming.getChannel().close();
+ }
+ set(connection);
+
+ }
+
+ }
+
+ /** Factory for close handlers **/
+ public class CloseHandlerCreator {
+ public GenericFutureListener<ChannelFuture> getHandler(CONNECTION_TYPE connection,
+ GenericFutureListener<ChannelFuture> parent) {
+ return new CloseHandler(connection, parent);
+ }
+ }
+
+ /**
+ * Listens for connection closes and clears connection holder.
+ */
+ protected class CloseHandler implements GenericFutureListener<ChannelFuture> {
+ private CONNECTION_TYPE connection;
+ private GenericFutureListener<ChannelFuture> parent;
+
+ public CloseHandler(CONNECTION_TYPE connection, GenericFutureListener<ChannelFuture> parent) {
+ super();
+ this.connection = connection;
+ this.parent = parent;
+ }
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ connectionHolder.compareAndSet(connection, null);
+ parent.operationComplete(future);
+ }
+
+ }
+
+ public CloseHandlerCreator getCloseHandlerCreator() {
+ return new CloseHandlerCreator();
+ }
+
+ public void addExternalConnection(CONNECTION_TYPE connection) {
+ // if the connection holder is not set, set it to this incoming connection. We'll simply ignore if already set.
+ this.connectionHolder.compareAndSet(null, connection);
+ }
+
+ @Override
+ public void close() {
+ CONNECTION_TYPE c = connectionHolder.getAndSet(null);
+ if (c != null) {
+ c.getChannel().close();
+ }
+ }
+
+ /**
+ * Decorate a connection creation so that we capture a success and keep it available for future requests. If we have
+ * raced and another is already available... we return that one and close things down on this one.
+ */
+ private class ConnectionListeningDecorator implements RpcConnectionHandler<CONNECTION_TYPE> {
+
+ private final RpcConnectionHandler<CONNECTION_TYPE> delegate;
+
+ public ConnectionListeningDecorator(RpcConnectionHandler<CONNECTION_TYPE> delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void connectionSucceeded(CONNECTION_TYPE incoming) {
+ CONNECTION_TYPE connection = connectionHolder.get();
+ while (true) {
+ boolean setted = connectionHolder.compareAndSet(null, incoming);
+ if (setted) {
+ connection = incoming;
+ break;
+ }
+ connection = connectionHolder.get();
+ if (connection != null) {
+ break;
+ }
+ }
+
+ if (connection == incoming) {
+ delegate.connectionSucceeded(connection);
+ } else {
+ // close the incoming because another channel was created in the mean time (unless this is a self connection).
+ logger.debug("Closing incoming connection because a connection was already set.");
+ incoming.getChannel().close();
+ delegate.connectionSucceeded(connection);
+ }
+ }
+
+ @Override
+ public void connectionFailed(org.apache.drill.exec.rpc.RpcConnectionHandler.FailureType type, Throwable t) {
+ delegate.connectionFailed(type, t);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
new file mode 100644
index 0000000..561f0a4
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+
+import java.util.concurrent.ExecutionException;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+
+public abstract class RemoteConnection implements ConnectionThrottle, AutoCloseable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemoteConnection.class);
+ private final Channel channel;
+ private final WriteManager writeManager;
+ private String name;
+ private final String clientName;
+
+ public boolean inEventLoop(){
+ return channel.eventLoop().inEventLoop();
+ }
+
+ public RemoteConnection(SocketChannel channel, String name) {
+ super();
+ this.channel = channel;
+ this.clientName = name;
+ this.writeManager = new WriteManager();
+ channel.pipeline().addLast(new BackPressureHandler());
+ channel.closeFuture().addListener(new GenericFutureListener<Future<? super Void>>() {
+ public void operationComplete(Future<? super Void> future) throws Exception {
+ // this could possibly overrelease but it doesn't matter since we're only going to do this to ensure that we
+ // fail out any pending messages
+ writeManager.disable();
+ writeManager.setWritable(true);
+ }
+ });
+
+ }
+
+ public String getName() {
+ if(name == null){
+ name = String.format("%s <--> %s (%s)", channel.localAddress(), channel.remoteAddress(), clientName);
+ }
+ return name;
+ }
+
+ public abstract BufferAllocator getAllocator();
+
+ public final Channel getChannel() {
+ return channel;
+ }
+
+ public boolean blockOnNotWritable(RpcOutcomeListener<?> listener){
+ try{
+ writeManager.waitForWritable();
+ return true;
+ }catch(final InterruptedException e){
+ listener.interrupted(e);
+
+ // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
+ // interruption and respond to it if it wants to.
+ Thread.currentThread().interrupt();
+
+ return false;
+ }
+ }
+
+ public void setAutoRead(boolean enableAutoRead){
+ channel.config().setAutoRead(enableAutoRead);
+ }
+
+ public boolean isActive(){
+ return channel.isActive();
+ }
+
+ /**
+ * The write manager is responsible for controlling whether or not a write can be sent. It controls whether or not to block a sender if we have tcp backpressure on the receive side.
+ */
+ private static class WriteManager{
+ private final ResettableBarrier barrier = new ResettableBarrier();
+ private volatile boolean disabled = false;
+
+ public WriteManager(){
+ barrier.openBarrier();
+ }
+
+ public void waitForWritable() throws InterruptedException{
+ barrier.await();
+ }
+
+ public void setWritable(boolean isWritable){
+ if(isWritable){
+ barrier.openBarrier();
+ } else if (!disabled) {
+ barrier.closeBarrier();
+ }
+
+ }
+
+ public void disable() {
+ disabled = true;
+ }
+ }
+
+ private class BackPressureHandler extends ChannelInboundHandlerAdapter{
+
+ @Override
+ public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
+// logger.debug("Channel writability changed.", ctx.channel().isWritable());
+ writeManager.setWritable(ctx.channel().isWritable());
+ ctx.fireChannelWritabilityChanged();
+ }
+
+ }
+
+ @Override
+ public void close() {
+ try {
+ if (channel.isActive()) {
+ channel.close().get();
+ }
+ } catch (final InterruptedException | ExecutionException e) {
+ logger.warn("Caught exception while closing channel.", e);
+
+ // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the
+ // interruption and respond to it if it wants to.
+ Thread.currentThread().interrupt();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java
new file mode 100644
index 0000000..a2a6d2a
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.AbstractQueuedSynchronizer;
+
+/**
+ * Modified implementation of countdown latch that allows a barrier to be unilaterally opened and closed. All others simply wait when it is closed. Is initialized in a closed state.
+ */
+public class ResettableBarrier {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ResettableBarrier.class);
+
+ private final InternalSynchronizer sync = new InternalSynchronizer();
+
+ public ResettableBarrier() {
+ }
+
+ private static final class InternalSynchronizer extends AbstractQueuedSynchronizer {
+
+ private InternalSynchronizer() {
+ setState(1);
+ }
+
+ @Override
+ protected int tryAcquireShared(int acquires) {
+ assert acquires == 1;
+ return (getState() == 0) ? 1 : -1;
+ }
+
+ @Override
+ protected boolean tryReleaseShared(int releases) {
+ assert releases == 1;
+
+ while(true) {
+ int c = getState();
+ if (c == 0) {
+ return false;
+ }
+ int nextc = c - 1;
+ if (compareAndSetState(c, nextc)) {
+ return nextc == 0;
+ }
+ }
+ }
+
+ protected void reset() {
+ setState(1);
+ }
+
+ }
+
+ public void await() throws InterruptedException {
+// logger.debug("awaiting barrier interruptibly.");
+ sync.acquireSharedInterruptibly(1);
+ }
+
+ public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
+// logger.debug("awaiting barrier with timeout {}.", timeout);
+ return sync.tryAcquireSharedNanos(1, unit.toNanos(timeout));
+ }
+
+ public void openBarrier() {
+// logger.debug("opening barrier.");
+ sync.releaseShared(1);
+ }
+
+ public void closeBarrier() {
+// logger.debug("closing barrier.");
+ sync.reset();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/Response.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/Response.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/Response.java
new file mode 100644
index 0000000..b48adec
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/Response.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+import com.google.protobuf.Internal.EnumLite;
+import com.google.protobuf.MessageLite;
+
+public class Response {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Response.class);
+
+ public EnumLite rpcType;
+ public MessageLite pBody;
+ public ByteBuf[] dBodies;
+
+ public Response(EnumLite rpcType, MessageLite pBody, ByteBuf... dBodies) {
+ super();
+ this.rpcType = rpcType;
+ this.pBody = pBody;
+ this.dBodies = dBodies;
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
new file mode 100644
index 0000000..6dc9ae1
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/ResponseSender.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+
+public interface ResponseSender {
+ public void send(Response r);
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
new file mode 100644
index 0000000..61922a1
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
@@ -0,0 +1,478 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.MessageToMessageDecoder;
+import io.netty.util.Recycler;
+import io.netty.util.Recycler.Handle;
+import io.netty.util.concurrent.GenericFutureListener;
+
+import java.io.Closeable;
+import java.net.SocketAddress;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.drill.common.SerializedExecutor;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode;
+import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.protobuf.Internal.EnumLite;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.MessageLite;
+import com.google.protobuf.Parser;
+
+/**
+ * The Rpc Bus deals with incoming and outgoing communication and is used on both the server and the client side of a
+ * system.
+ *
+ * @param <T>
+ */
+public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> implements Closeable {
+ final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(this.getClass());
+
+ private static final OutboundRpcMessage PONG = new OutboundRpcMessage(RpcMode.PONG, 0, 0, Acks.OK);
+
+ protected final CoordinationQueue queue = new CoordinationQueue(16, 16);
+
+ protected abstract MessageLite getResponseDefaultInstance(int rpcType) throws RpcException;
+
+ protected void handle(C connection, int rpcType, ByteBuf pBody, ByteBuf dBody, ResponseSender sender) throws RpcException{
+ sender.send(handle(connection, rpcType, pBody, dBody));
+ }
+
+ protected abstract Response handle(C connection, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException;
+
+ public abstract boolean isClient();
+
+ protected final RpcConfig rpcConfig;
+
+ protected volatile SocketAddress local;
+ protected volatile SocketAddress remote;
+
+
+ public RpcBus(RpcConfig rpcConfig) {
+ this.rpcConfig = rpcConfig;
+ }
+
+ protected void setAddresses(SocketAddress remote, SocketAddress local){
+ this.remote = remote;
+ this.local = local;
+ }
+
+ <SEND extends MessageLite, RECEIVE extends MessageLite> DrillRpcFuture<RECEIVE> send(C connection, T rpcType,
+ SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
+ DrillRpcFutureImpl<RECEIVE> rpcFuture = new DrillRpcFutureImpl<RECEIVE>();
+ this.send(rpcFuture, connection, rpcType, protobufBody, clazz, dataBodies);
+ return rpcFuture;
+ }
+
+ public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener, C connection, T rpcType,
+ SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
+ send(listener, connection, rpcType, protobufBody, clazz, false, dataBodies);
+ }
+
+ public <SEND extends MessageLite, RECEIVE extends MessageLite> void send(RpcOutcomeListener<RECEIVE> listener, C connection, T rpcType,
+ SEND protobufBody, Class<RECEIVE> clazz, boolean allowInEventLoop, ByteBuf... dataBodies) {
+
+ Preconditions
+ .checkArgument(
+ allowInEventLoop || !connection.inEventLoop(),
+ "You attempted to send while inside the rpc event thread. This isn't allowed because sending will block if the channel is backed up.");
+
+ ByteBuf pBuffer = null;
+ boolean completed = false;
+
+ try {
+
+ if (!allowInEventLoop && !connection.blockOnNotWritable(listener)) {
+ // if we're in not in the event loop and we're interrupted while blocking, skip sending this message.
+ return;
+ }
+
+ assert !Arrays.asList(dataBodies).contains(null);
+ assert rpcConfig.checkSend(rpcType, protobufBody.getClass(), clazz);
+
+ Preconditions.checkNotNull(protobufBody);
+ ChannelListenerWithCoordinationId futureListener = queue.get(listener, clazz, connection);
+ OutboundRpcMessage m = new OutboundRpcMessage(RpcMode.REQUEST, rpcType, futureListener.getCoordinationId(), protobufBody, dataBodies);
+ ChannelFuture channelFuture = connection.getChannel().writeAndFlush(m);
+ channelFuture.addListener(futureListener);
+ channelFuture.addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
+ completed = true;
+ } catch (Exception | AssertionError e) {
+ listener.failed(new RpcException("Failure sending message.", e));
+ } finally {
+ if (!completed) {
+ if (pBuffer != null) {
+ pBuffer.release();
+ }
+ if (dataBodies != null) {
+ for (ByteBuf b : dataBodies) {
+ b.release();
+ }
+
+ }
+ }
+ ;
+ }
+ }
+
+ public abstract C initRemoteConnection(SocketChannel channel);
+
+ public class ChannelClosedHandler implements GenericFutureListener<ChannelFuture> {
+
+ final C clientConnection;
+ private final Channel channel;
+
+ public ChannelClosedHandler(C clientConnection, Channel channel) {
+ this.channel = channel;
+ this.clientConnection = clientConnection;
+ }
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ String msg;
+ if(local!=null) {
+ msg = String.format("Channel closed %s <--> %s.", local, remote);
+ }else{
+ msg = String.format("Channel closed %s <--> %s.", future.channel().localAddress(), future.channel().remoteAddress());
+ }
+
+ if (RpcBus.this.isClient()) {
+ if(local != null) {
+ logger.info(String.format(msg));
+ }
+ } else {
+ queue.channelClosed(new ChannelClosedException(msg));
+ }
+
+ clientConnection.close();
+ }
+
+ }
+
+ protected GenericFutureListener<ChannelFuture> getCloseHandler(SocketChannel channel, C clientConnection) {
+ return new ChannelClosedHandler(clientConnection, channel);
+ }
+
+ private interface Recyclable {
+ public void recycle();
+ }
+
+ private class ResponseSenderImpl implements ResponseSender {
+
+ private RemoteConnection connection;
+ private int coordinationId;
+ private final AtomicBoolean sent = new AtomicBoolean(false);
+ private final Recyclable recyclable;
+
+ public ResponseSenderImpl(Recyclable recyclable) {
+ this.recyclable = recyclable;
+ }
+
+ void set(RemoteConnection connection, int coordinationId){
+ this.connection = connection;
+ this.coordinationId = coordinationId;
+ sent.set(false);
+ }
+
+ public void send(Response r) {
+ try {
+ assert rpcConfig.checkResponseSend(r.rpcType, r.pBody.getClass());
+ sendOnce();
+ OutboundRpcMessage outMessage = new OutboundRpcMessage(RpcMode.RESPONSE, r.rpcType, coordinationId,
+ r.pBody, r.dBodies);
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Adding message to outbound buffer. {}", outMessage);
+ }
+ logger.debug("Sending response with Sender {}", System.identityHashCode(this));
+ connection.getChannel().writeAndFlush(outMessage);
+ } finally {
+ recyclable.recycle();
+ }
+ }
+
+ /**
+ * Ensures that each sender is only used once.
+ */
+ private void sendOnce() {
+ if (!sent.compareAndSet(false, true)) {
+ throw new IllegalStateException("Attempted to utilize a sender multiple times.");
+ }
+ }
+
+ void sendFailure(UserRpcException e){
+ try {
+ sendOnce();
+ UserException uex = UserException.systemError(e)
+ .addIdentity(e.getEndpoint())
+ .build(logger);
+
+ logger.error("Unexpected Error while handling request message", e);
+
+ OutboundRpcMessage outMessage = new OutboundRpcMessage(
+ RpcMode.RESPONSE_FAILURE,
+ 0,
+ coordinationId,
+ uex.getOrCreatePBError(false)
+ );
+
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Adding message to outbound buffer. {}", outMessage);
+ }
+ connection.getChannel().writeAndFlush(outMessage);
+ } finally {
+ recyclable.recycle();
+ }
+ }
+
+ }
+
+
+ protected class InboundHandler extends MessageToMessageDecoder<InboundRpcMessage> {
+
+ private final RpcEventHandler exec;
+ private final C connection;
+
+ public InboundHandler(C connection) {
+ super();
+ this.connection = connection;
+ this.exec = new RpcEventHandler(rpcConfig.getExecutor());
+ }
+
+ @Override
+ protected void decode(final ChannelHandlerContext ctx, final InboundRpcMessage msg, final List<Object> output) throws Exception {
+ if (!ctx.channel().isOpen()) {
+ return;
+ }
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Received message {}", msg);
+ }
+ final Channel channel = connection.getChannel();
+ final Stopwatch watch = new Stopwatch().start();
+
+ try{
+
+ switch (msg.mode) {
+ case REQUEST:
+ RequestEvent reqEvent = requestRecycler.get();
+ reqEvent.set(msg.coordinationId, connection, msg.rpcType, msg.pBody, msg.dBody);
+ exec.execute(reqEvent);
+ break;
+
+ case RESPONSE:
+ ResponseEvent respEvent = responseRecycler.get();
+ respEvent.set(msg.rpcType, msg.coordinationId, msg.pBody, msg.dBody);
+ exec.execute(respEvent);
+ break;
+
+ case RESPONSE_FAILURE:
+ DrillPBError failure = DrillPBError.parseFrom(new ByteBufInputStream(msg.pBody, msg.pBody.readableBytes()));
+ queue.updateFailedFuture(msg.coordinationId, failure);
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Updated rpc future with coordinationId {} with failure ", msg.coordinationId, failure);
+ }
+ break;
+
+ case PING:
+ channel.writeAndFlush(PONG);
+ break;
+
+ case PONG:
+ // noop.
+ break;
+
+ default:
+ throw new UnsupportedOperationException();
+ }
+ } finally {
+ long time = watch.elapsed(TimeUnit.MILLISECONDS);
+ long delayThreshold = Integer.parseInt(System.getProperty("drill.exec.rpcDelayWarning", "500"));
+ if (time > delayThreshold) {
+ logger.warn(String.format(
+ "Message of mode %s of rpc type %d took longer than %dms. Actual duration was %dms.",
+ msg.mode, msg.rpcType, delayThreshold, time));
+ }
+ msg.release();
+ }
+ }
+ }
+
+ public static <T> T get(ByteBuf pBody, Parser<T> parser) throws RpcException {
+ try {
+ ByteBufInputStream is = new ByteBufInputStream(pBody);
+ return parser.parseFrom(is);
+ } catch (InvalidProtocolBufferException e) {
+ throw new RpcException(String.format("Failure while decoding message with parser of type. %s", parser.getClass().getCanonicalName()), e);
+ }
+ }
+
+ class RpcEventHandler extends SerializedExecutor {
+
+ public RpcEventHandler(Executor underlyingExecutor) {
+ super(rpcConfig.getName() + "-rpc-event-queue", underlyingExecutor);
+ }
+
+ @Override
+ protected void runException(Runnable command, Throwable t) {
+ logger.error("Failure while running rpc command.", t);
+ }
+
+ }
+
+ private final Recycler<RequestEvent> requestRecycler = new Recycler<RequestEvent>() {
+ @Override
+ protected RequestEvent newObject(Handle handle) {
+ return new RequestEvent(handle);
+ }
+ };
+
+ private class RequestEvent implements Runnable, Recyclable {
+ private final ResponseSenderImpl sender;
+ private final Handle handle;
+ private C connection;
+ private int rpcType;
+ private ByteBuf pBody;
+ private ByteBuf dBody;
+
+ RequestEvent(Handle handle){
+ this.handle = handle;
+ sender = new ResponseSenderImpl(this);
+ }
+
+ public void set(int coordinationId, C connection, int rpcType, ByteBuf pBody, ByteBuf dBody) {
+ this.connection = connection;
+ this.rpcType = rpcType;
+ this.pBody = pBody;
+ this.dBody = dBody;
+ sender.set(connection, coordinationId);
+
+ if(pBody != null){
+ pBody.retain();
+ }
+
+ if(dBody != null){
+ dBody.retain();
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ handle(connection, rpcType, pBody, dBody, sender);
+ } catch (UserRpcException e) {
+ sender.sendFailure(e);
+ } catch (Exception e) {
+ logger.error("Failure while handling message.", e);
+ }finally{
+ if(pBody != null){
+ pBody.release();
+ }
+
+ if(dBody != null){
+ dBody.release();
+ }
+ }
+
+ }
+
+ @Override
+ public void recycle() {
+ // We must defer recycling until the sender has been used.
+ requestRecycler.recycle(this, handle);
+ }
+
+
+ }
+
+ private final Recycler<ResponseEvent> responseRecycler = new Recycler<ResponseEvent>() {
+ @Override
+ protected ResponseEvent newObject(Handle handle) {
+ return new ResponseEvent(handle);
+ }
+ };
+
+ private class ResponseEvent implements Runnable {
+ private final Handle handle;
+
+ private int rpcType;
+ private int coordinationId;
+ private ByteBuf pBody;
+ private ByteBuf dBody;
+
+ public ResponseEvent(Handle handle){
+ this.handle = handle;
+ }
+
+ public void set(int rpcType, int coordinationId, ByteBuf pBody, ByteBuf dBody) {
+ this.rpcType = rpcType;
+ this.coordinationId = coordinationId;
+ this.pBody = pBody;
+ this.dBody = dBody;
+
+ if(pBody != null){
+ pBody.retain();
+ }
+
+ if(dBody != null){
+ dBody.retain();
+ }
+ }
+
+ public void run(){
+ try {
+ MessageLite m = getResponseDefaultInstance(rpcType);
+ assert rpcConfig.checkReceive(rpcType, m.getClass());
+ RpcOutcome<?> rpcFuture = queue.getFuture(rpcType, coordinationId, m.getClass());
+ Parser<?> parser = m.getParserForType();
+ Object value = parser.parseFrom(new ByteBufInputStream(pBody, pBody.readableBytes()));
+ rpcFuture.set(value, dBody);
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Updated rpc future {} with value {}", rpcFuture, value);
+ }
+ } catch (Exception ex) {
+ logger.error("Failure while handling response.", ex);
+ }finally{
+ if(pBody != null){
+ pBody.release();
+ }
+
+ if(dBody != null){
+ dBody.release();
+ }
+
+ responseRecycler.recycle(this, handle);
+ }
+
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcCheckedFuture.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcCheckedFuture.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcCheckedFuture.java
new file mode 100644
index 0000000..11b07ad
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcCheckedFuture.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+
+import com.google.common.util.concurrent.AbstractCheckedFuture;
+import com.google.common.util.concurrent.ListenableFuture;
+
+public class RpcCheckedFuture<T> extends AbstractCheckedFuture<T, RpcException> implements DrillRpcFuture<T>{
+
+ volatile ByteBuf buffer;
+
+ public RpcCheckedFuture(ListenableFuture<T> delegate) {
+ super(delegate);
+ }
+
+ public void set(T obj, ByteBuf buffer){
+ this.buffer = buffer;
+ }
+
+ @Override
+ protected RpcException mapException(Exception e) {
+ return RpcException.mapException(e);
+ }
+
+ @Override
+ public ByteBuf getBuffer() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcCommand.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcCommand.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcCommand.java
new file mode 100644
index 0000000..93b901c
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcCommand.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import com.google.protobuf.MessageLite;
+
+public interface RpcCommand<T extends MessageLite, C extends RemoteConnection> extends RpcConnectionHandler<C>{
+
+ public abstract void connectionAvailable(C connection);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConfig.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConfig.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConfig.java
new file mode 100644
index 0000000..22b253a
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConfig.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import java.util.Map;
+import java.util.concurrent.Executor;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.protobuf.Internal.EnumLite;
+import com.google.protobuf.MessageLite;
+
+public class RpcConfig {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcConfig.class);
+
+ private final String name;
+ private final int timeout;
+ private final Map<EnumLite, RpcMessageType<?, ?, ?>> sendMap;
+ private final Map<Integer, RpcMessageType<?, ?, ?>> receiveMap;
+ private final Executor executor;
+
+ private RpcConfig(String name, Map<EnumLite, RpcMessageType<?, ?, ?>> sendMap,
+ Map<Integer, RpcMessageType<?, ?, ?>> receiveMap, int timeout, Executor executor) {
+ Preconditions.checkNotNull(executor, "Executor must be defined.");
+ this.name = name;
+ this.timeout = timeout;
+ this.sendMap = ImmutableMap.copyOf(sendMap);
+ this.receiveMap = ImmutableMap.copyOf(receiveMap);
+ this.executor = executor;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getTimeout() {
+ return timeout;
+ }
+
+ public boolean hasTimeout() {
+ return timeout > 0;
+ }
+
+ public Executor getExecutor() {
+ return executor;
+ }
+
+ public boolean checkReceive(int rpcType, Class<?> receiveClass) {
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug(String.format("Checking reception for rpcType %d and receive class %s.", rpcType, receiveClass));
+ }
+ RpcMessageType<?,?,?> type = receiveMap.get(rpcType);
+ if (type == null) {
+ throw new IllegalStateException(String.format("%s: There is no defined RpcMessage type for a Rpc receive type number of %s.", name, rpcType));
+ }
+
+ if (receiveClass != type.getRet()) {
+ throw new IllegalStateException(String.format("%s: The definition for receive doesn't match implementation code. The definition is %s however the current receive for this type was of type %s.", name, type, receiveClass.getCanonicalName()));
+ }
+ return true;
+ }
+
+ public boolean checkSend(EnumLite send, Class<?> sendClass, Class<?> receiveClass) {
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug(String.format("Checking send classes for send RpcType %s. Send Class is %s and Receive class is %s.", send, sendClass, receiveClass));
+ }
+ RpcMessageType<?,?,?> type = sendMap.get(send);
+ if (type == null) {
+ throw new IllegalStateException(String.format("%s: There is no defined RpcMessage type for a Rpc send type of %s.", name, send));
+ }
+
+ if (type.getSend() != sendClass) {
+ throw new IllegalStateException(String.format("%s: The definition for send doesn't match implementation code. The definition is %s however the current send is trying to send an object of type %s.", name, type, sendClass.getCanonicalName()));
+ }
+ if (type.getRet() != receiveClass) {
+ throw new IllegalStateException(String.format("%s: The definition for send doesn't match implementation code. The definition is %s however the current send is trying to setup an expected reception of an object of type %s.", name, type, receiveClass.getCanonicalName()));
+ }
+
+ return true;
+ }
+
+ public boolean checkResponseSend(EnumLite responseType, Class<?> responseClass) {
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug(String.format("Checking responce send of type %s with response class of %s.", responseType, responseClass));
+ }
+ RpcMessageType<?,?,?> type = receiveMap.get(responseType.getNumber());
+ if (type == null) {
+ throw new IllegalStateException(String.format("%s: There is no defined RpcMessage type for a Rpc response of type %s.", name, responseType));
+ }
+ if (type.getRet() != responseClass) {
+ throw new IllegalStateException(String.format("%s: The definition for the response doesn't match implementation code. The definition is %s however the current response is trying to response with an object of type %s.", name, type, responseClass.getCanonicalName()));
+ }
+
+ return true;
+ }
+
+ public static class RpcMessageType<SEND extends MessageLite, RECEIVE extends MessageLite, T extends EnumLite>{
+ private T sendEnum;
+ private Class<SEND> send;
+ private T receiveEnum;
+ private Class<RECEIVE> ret;
+ public RpcMessageType(T sendEnum, Class<SEND> send, T receiveEnum, Class<RECEIVE> ret) {
+ super();
+ this.sendEnum = sendEnum;
+ this.send = send;
+ this.receiveEnum = receiveEnum;
+ this.ret = ret;
+ }
+ public Class<SEND> getSend() {
+ return send;
+ }
+ public void setSend(Class<SEND> send) {
+ this.send = send;
+ }
+ public T getSendEnum() {
+ return sendEnum;
+ }
+ public void setSendEnum(T sendEnum) {
+ this.sendEnum = sendEnum;
+ }
+ public Class<RECEIVE> getRet() {
+ return ret;
+ }
+ public void setRet(Class<RECEIVE> ret) {
+ this.ret = ret;
+ }
+ public T getReceiveEnum() {
+ return receiveEnum;
+ }
+ public void setReceiveEnum(T receiveEnum) {
+ this.receiveEnum = receiveEnum;
+ }
+ @Override
+ public String toString() {
+ return "RpcMessageType [sendEnum=" + sendEnum + ", send=" + send + ", receiveEnum=" + receiveEnum + ", ret="
+ + ret + "]";
+ }
+
+ }
+
+ public static RpcConfigBuilder newBuilder() {
+ return new RpcConfigBuilder();
+ }
+
+ public static class RpcConfigBuilder {
+ private String name;
+ private int timeout = -1;
+ private Executor executor;
+ private Map<EnumLite, RpcMessageType<?, ?, ?>> sendMap = Maps.newHashMap();
+ private Map<Integer, RpcMessageType<?, ?, ?>> receiveMap = Maps.newHashMap();
+
+ private RpcConfigBuilder() {
+ }
+
+ public RpcConfigBuilder name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public RpcConfigBuilder timeout(int timeoutInSeconds) {
+ this.timeout = timeoutInSeconds;
+ return this;
+ }
+
+ public <SEND extends MessageLite, RECEIVE extends MessageLite, T extends EnumLite> RpcConfigBuilder add(T sendEnum, Class<SEND> send, T receiveEnum, Class<RECEIVE> rec) {
+ RpcMessageType<SEND, RECEIVE, T> type = new RpcMessageType<SEND, RECEIVE, T>(sendEnum, send, receiveEnum, rec);
+ this.sendMap.put(sendEnum, type);
+ this.receiveMap.put(receiveEnum.getNumber(), type);
+ return this;
+ }
+
+ public RpcConfigBuilder executor(Executor executor) {
+ this.executor = executor;
+ return this;
+ }
+
+ public RpcConfig build() {
+ Preconditions.checkArgument(timeout > -1, "Timeout must be a positive number or zero for disabled.");
+ Preconditions.checkArgument(name != null, "RpcConfig name must be set.");
+ return new RpcConfig(name, sendMap, receiveMap, timeout, executor);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConnectionHandler.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConnectionHandler.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConnectionHandler.java
new file mode 100644
index 0000000..7618231
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConnectionHandler.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+public interface RpcConnectionHandler<T extends RemoteConnection> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcConnectionHandler.class);
+
+ public static enum FailureType{CONNECTION, HANDSHAKE_COMMUNICATION, HANDSHAKE_VALIDATION}
+
+ public void connectionSucceeded(T connection);
+ public void connectionFailed(FailureType type, Throwable t);
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConstants.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConstants.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConstants.java
new file mode 100644
index 0000000..4be365c
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcConstants.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+public class RpcConstants {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcConstants.class);
+
+ private RpcConstants(){}
+
+ public static final boolean SOME_DEBUGGING = false;
+ public static final boolean EXTRA_DEBUGGING = false;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcDecoder.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcDecoder.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcDecoder.java
new file mode 100644
index 0000000..ac48187
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcDecoder.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.CorruptedFrameException;
+import io.netty.handler.codec.MessageToMessageDecoder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.drill.exec.proto.GeneralRPCProtos.RpcHeader;
+
+/**
+ * Converts a previously length adjusted buffer into an RpcMessage.
+ */
+class RpcDecoder extends MessageToMessageDecoder<ByteBuf> {
+ final org.slf4j.Logger logger;
+
+ private final AtomicLong messageCounter = new AtomicLong();
+
+ public RpcDecoder(String name) {
+ this.logger = org.slf4j.LoggerFactory.getLogger(RpcDecoder.class.getCanonicalName() + "-" + name);
+ }
+
+
+ @Override
+ protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> out) throws Exception {
+ if (!ctx.channel().isOpen()) {
+ return;
+ }
+
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Inbound rpc message received.");
+ }
+
+ // now, we know the entire message is in the buffer and the buffer is constrained to this message. Additionally,
+ // this process should avoid reading beyond the end of this buffer so we inform the ByteBufInputStream to throw an
+ // exception if be go beyond readable bytes (as opposed to blocking).
+ final ByteBufInputStream is = new ByteBufInputStream(buffer, buffer.readableBytes());
+
+ // read the rpc header, saved in delimited format.
+ checkTag(is, RpcEncoder.HEADER_TAG);
+ final RpcHeader header = RpcHeader.parseDelimitedFrom(is);
+
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug(" post header read index {}", buffer.readerIndex());
+ }
+
+ // read the protobuf body into a buffer.
+ checkTag(is, RpcEncoder.PROTOBUF_BODY_TAG);
+ final int pBodyLength = readRawVarint32(is);
+ final ByteBuf pBody = buffer.slice(buffer.readerIndex(), pBodyLength);
+ buffer.skipBytes(pBodyLength);
+ pBody.retain(1);
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Read protobuf body of length {} into buffer {}.", pBodyLength, pBody);
+ }
+
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("post protobufbody read index {}", buffer.readerIndex());
+ }
+
+ ByteBuf dBody = null;
+ int dBodyLength = 0;
+
+ // read the data body.
+ if (buffer.readableBytes() > 0) {
+
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Reading raw body, buffer has {} bytes available, is available {}.", buffer.readableBytes(), is.available());
+ }
+ checkTag(is, RpcEncoder.RAW_BODY_TAG);
+ dBodyLength = readRawVarint32(is);
+ if (buffer.readableBytes() != dBodyLength) {
+ throw new CorruptedFrameException(String.format("Expected to receive a raw body of %d bytes but received a buffer with %d bytes.", dBodyLength, buffer.readableBytes()));
+ }
+ dBody = buffer.slice();
+ dBody.retain(1);
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Read raw body of {}", dBody);
+ }
+
+ }else{
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("No need to read raw body, no readable bytes left.");
+ }
+ }
+
+
+ // return the rpc message.
+ InboundRpcMessage m = new InboundRpcMessage(header.getMode(), header.getRpcType(), header.getCoordinationId(),
+ pBody, dBody);
+
+ // move the reader index forward so the next rpc call won't try to work with it.
+ buffer.skipBytes(dBodyLength);
+ messageCounter.incrementAndGet();
+ if (RpcConstants.SOME_DEBUGGING) {
+ logger.debug("Inbound Rpc Message Decoded {}.", m);
+ }
+ out.add(m);
+
+ }
+
+ private void checkTag(ByteBufInputStream is, int expectedTag) throws IOException {
+ int actualTag = readRawVarint32(is);
+ if (actualTag != expectedTag) {
+ throw new CorruptedFrameException(String.format("Expected to read a tag of %d but actually received a value of %d. Happened after reading %d message.", expectedTag, actualTag, messageCounter.get()));
+ }
+ }
+
+ // Taken from CodedInputStream and modified to enable ByteBufInterface.
+ public static int readRawVarint32(ByteBufInputStream is) throws IOException {
+ byte tmp = is.readByte();
+ if (tmp >= 0) {
+ return tmp;
+ }
+ int result = tmp & 0x7f;
+ if ((tmp = is.readByte()) >= 0) {
+ result |= tmp << 7;
+ } else {
+ result |= (tmp & 0x7f) << 7;
+ if ((tmp = is.readByte()) >= 0) {
+ result |= tmp << 14;
+ } else {
+ result |= (tmp & 0x7f) << 14;
+ if ((tmp = is.readByte()) >= 0) {
+ result |= tmp << 21;
+ } else {
+ result |= (tmp & 0x7f) << 21;
+ result |= (tmp = is.readByte()) << 28;
+ if (tmp < 0) {
+ // Discard upper 32 bits.
+ for (int i = 0; i < 5; i++) {
+ if (is.readByte() >= 0) {
+ return result;
+ }
+ }
+ throw new CorruptedFrameException("Encountered a malformed varint.");
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcEncoder.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcEncoder.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcEncoder.java
new file mode 100644
index 0000000..f9da6f1
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcEncoder.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufOutputStream;
+import io.netty.buffer.CompositeByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.MessageToMessageEncoder;
+
+import java.io.OutputStream;
+import java.util.List;
+
+import org.apache.drill.exec.proto.GeneralRPCProtos.CompleteRpcMessage;
+import org.apache.drill.exec.proto.GeneralRPCProtos.RpcHeader;
+
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.WireFormat;
+
+/**
+ * Converts an RPCMessage into wire format.
+ */
+class RpcEncoder extends MessageToMessageEncoder<OutboundRpcMessage>{
+ final org.slf4j.Logger logger;
+
+ static final int HEADER_TAG = makeTag(CompleteRpcMessage.HEADER_FIELD_NUMBER, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+ static final int PROTOBUF_BODY_TAG = makeTag(CompleteRpcMessage.PROTOBUF_BODY_FIELD_NUMBER, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+ static final int RAW_BODY_TAG = makeTag(CompleteRpcMessage.RAW_BODY_FIELD_NUMBER, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+ static final int HEADER_TAG_LENGTH = getRawVarintSize(HEADER_TAG);
+ static final int PROTOBUF_BODY_TAG_LENGTH = getRawVarintSize(PROTOBUF_BODY_TAG);
+ static final int RAW_BODY_TAG_LENGTH = getRawVarintSize(RAW_BODY_TAG);
+
+ public RpcEncoder(String name) {
+ this.logger = org.slf4j.LoggerFactory.getLogger(RpcEncoder.class.getCanonicalName() + "-" + name);
+ }
+
+ @Override
+ protected void encode(ChannelHandlerContext ctx, OutboundRpcMessage msg, List<Object> out) throws Exception {
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Rpc Encoder called with msg {}", msg);
+ }
+
+ if (!ctx.channel().isOpen()) {
+ //output.add(ctx.alloc().buffer(0));
+ logger.debug("Channel closed, skipping encode.");
+ msg.release();
+ return;
+ }
+
+ try{
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Encoding outbound message {}", msg);
+ }
+ // first we build the RpcHeader
+ RpcHeader header = RpcHeader.newBuilder() //
+ .setMode(msg.mode) //
+ .setCoordinationId(msg.coordinationId) //
+ .setRpcType(msg.rpcType).build();
+
+ // figure out the full length
+ int headerLength = header.getSerializedSize();
+ int protoBodyLength = msg.pBody.getSerializedSize();
+ int rawBodyLength = msg.getRawBodySize();
+ int fullLength = //
+ HEADER_TAG_LENGTH + getRawVarintSize(headerLength) + headerLength + //
+ PROTOBUF_BODY_TAG_LENGTH + getRawVarintSize(protoBodyLength) + protoBodyLength; //
+
+ if (rawBodyLength > 0) {
+ fullLength += (RAW_BODY_TAG_LENGTH + getRawVarintSize(rawBodyLength) + rawBodyLength);
+ }
+
+ ByteBuf buf = ctx.alloc().buffer();
+ OutputStream os = new ByteBufOutputStream(buf);
+ CodedOutputStream cos = CodedOutputStream.newInstance(os);
+
+ // write full length first (this is length delimited stream).
+ cos.writeRawVarint32(fullLength);
+
+ // write header
+ cos.writeRawVarint32(HEADER_TAG);
+ cos.writeRawVarint32(headerLength);
+ header.writeTo(cos);
+
+ // write protobuf body length and body
+ cos.writeRawVarint32(PROTOBUF_BODY_TAG);
+ cos.writeRawVarint32(protoBodyLength);
+ msg.pBody.writeTo(cos);
+
+ // if exists, write data body and tag.
+ if (msg.getRawBodySize() > 0) {
+ if(RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Writing raw body of size {}", msg.getRawBodySize());
+ }
+
+ cos.writeRawVarint32(RAW_BODY_TAG);
+ cos.writeRawVarint32(rawBodyLength);
+ cos.flush(); // need to flush so that dbody goes after if cos is caching.
+
+ CompositeByteBuf cbb = new CompositeByteBuf(buf.alloc(), true, msg.dBodies.length + 1);
+ cbb.addComponent(buf);
+ int bufLength = buf.readableBytes();
+ for (ByteBuf b : msg.dBodies) {
+ cbb.addComponent(b);
+ bufLength += b.readableBytes();
+ }
+ cbb.writerIndex(bufLength);
+ out.add(cbb);
+ } else {
+ cos.flush();
+ out.add(buf);
+ }
+
+ if (RpcConstants.SOME_DEBUGGING) {
+ logger.debug("Wrote message length {}:{} bytes (head:body). Message: " + msg, getRawVarintSize(fullLength), fullLength);
+ }
+ if (RpcConstants.EXTRA_DEBUGGING) {
+ logger.debug("Sent message. Ending writer index was {}.", buf.writerIndex());
+ }
+ } finally {
+ // make sure to release Rpc Messages underlying byte buffers.
+ //msg.release();
+ }
+ }
+
+ /** Makes a tag value given a field number and wire type, copied from WireFormat since it isn't public. */
+ static int makeTag(final int fieldNumber, final int wireType) {
+ return (fieldNumber << 3) | wireType;
+ }
+
+ public static int getRawVarintSize(int value) {
+ int count = 0;
+ while (true) {
+ if ((value & ~0x7F) == 0) {
+ count++;
+ return count;
+ } else {
+ count++;
+ value >>>= 7;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcException.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcException.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcException.java
new file mode 100644
index 0000000..a6d0e8e
--- /dev/null
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcException.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc;
+
+import java.util.concurrent.ExecutionException;
+
+import org.apache.drill.common.exceptions.DrillIOException;
+import org.apache.drill.common.util.DrillStringUtils;
+import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
+
+/**
+ * Parent class for all rpc exceptions.
+ */
+public class RpcException extends DrillIOException{
+ private static final long serialVersionUID = -5964230316010502319L;
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcException.class);
+
+ public RpcException() {
+ super();
+ }
+
+ public RpcException(String message, Throwable cause) {
+ super(format(message), cause);
+ }
+
+ private static String format(String message) {
+ return DrillStringUtils.unescapeJava(message);
+ }
+
+ public RpcException(String message) {
+ super(format(message));
+ }
+
+ public RpcException(Throwable cause) {
+ super(cause);
+ }
+
+ public static RpcException mapException(Throwable t) {
+ while (t instanceof ExecutionException) {
+ t = ((ExecutionException)t).getCause();
+ }
+ if (t instanceof RpcException) {
+ return ((RpcException) t);
+ }
+ return new RpcException(t);
+ }
+
+ public static RpcException mapException(String message, Throwable t) {
+ while (t instanceof ExecutionException) {
+ t = ((ExecutionException)t).getCause();
+ }
+ return new RpcException(message, t);
+ }
+
+ public boolean isRemote(){
+ return false;
+ }
+
+ public DrillPBError getRemoteError(){
+ throw new UnsupportedOperationException();
+ }
+}
[02/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
new file mode 100644
index 0000000..a6967f3
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedFixedWidthVectorLike.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+/**
+ * A {@link org.apache.drill.exec.vector.ValueVector} mix-in that can be used in conjunction with
+ * {@link RepeatedValueVector} subtypes.
+ */
+public interface RepeatedFixedWidthVectorLike {
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount Number of separate repeating groupings.
+ * @param innerValueCount Number of supported values in the vector.
+ */
+ void allocateNew(int valueCount, int innerValueCount);
+
+ /**
+ * Load the records in the provided buffer based on the given number of values.
+ * @param valueCount Number of separate repeating groupings.
+ * @param innerValueCount Number atomic values the buffer contains.
+ * @param buf Incoming buffer.
+ * @return The number of bytes of the buffer that were consumed.
+ */
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
new file mode 100644
index 0000000..330eed4
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -0,0 +1,429 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import io.netty.buffer.DrillBuf;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.holders.ComplexHolder;
+import org.apache.drill.exec.expr.holders.RepeatedListHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.vector.AddOrGetResult;
+import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+import org.apache.drill.exec.vector.complex.impl.NullReader;
+import org.apache.drill.exec.vector.complex.impl.RepeatedListReaderImpl;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class RepeatedListVector extends AbstractContainerVector
+ implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
+
+ public final static MajorType TYPE = Types.repeated(MinorType.LIST);
+ private final RepeatedListReaderImpl reader = new RepeatedListReaderImpl(null, this);
+ private final DelegateRepeatedVector delegate;
+
+ protected static class DelegateRepeatedVector extends BaseRepeatedValueVector {
+
+ private final RepeatedListAccessor accessor = new RepeatedListAccessor();
+ private final RepeatedListMutator mutator = new RepeatedListMutator();
+ private final EmptyValuePopulator emptyPopulator;
+ private transient DelegateTransferPair ephPair;
+
+ public class RepeatedListAccessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
+
+ @Override
+ public Object getObject(int index) {
+ final List<Object> list = new JsonStringArrayList();
+ final int start = offsets.getAccessor().get(index);
+ final int until = offsets.getAccessor().get(index+1);
+ for (int i = start; i < until; i++) {
+ list.add(vector.getAccessor().getObject(i));
+ }
+ return list;
+ }
+
+ public void get(int index, RepeatedListHolder holder) {
+ assert index <= getValueCapacity();
+ holder.start = getOffsetVector().getAccessor().get(index);
+ holder.end = getOffsetVector().getAccessor().get(index+1);
+ }
+
+ public void get(int index, ComplexHolder holder) {
+ final FieldReader reader = getReader();
+ reader.setPosition(index);
+ holder.reader = reader;
+ }
+
+ public void get(int index, int arrayIndex, ComplexHolder holder) {
+ final RepeatedListHolder listHolder = new RepeatedListHolder();
+ get(index, listHolder);
+ int offset = listHolder.start + arrayIndex;
+ if (offset >= listHolder.end) {
+ holder.reader = NullReader.INSTANCE;
+ } else {
+ FieldReader r = getDataVector().getReader();
+ r.setPosition(offset);
+ holder.reader = r;
+ }
+ }
+ }
+
+ public class RepeatedListMutator extends BaseRepeatedValueVector.BaseRepeatedMutator {
+
+ public int add(int index) {
+ final int curEnd = getOffsetVector().getAccessor().get(index+1);
+ getOffsetVector().getMutator().setSafe(index + 1, curEnd + 1);
+ return curEnd;
+ }
+
+ @Override
+ public void startNewValue(int index) {
+ emptyPopulator.populate(index+1);
+ super.startNewValue(index);
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ emptyPopulator.populate(valueCount);
+ super.setValueCount(valueCount);
+ }
+ }
+
+
+ public class DelegateTransferPair implements TransferPair {
+ private final DelegateRepeatedVector target;
+ private final TransferPair[] children;
+
+ public DelegateTransferPair(DelegateRepeatedVector target) {
+ this.target = Preconditions.checkNotNull(target);
+ if (target.getDataVector() == DEFAULT_DATA_VECTOR) {
+ target.addOrGetVector(VectorDescriptor.create(getDataVector().getField()));
+ target.getDataVector().allocateNew();
+ }
+ this.children = new TransferPair[] {
+ getOffsetVector().makeTransferPair(target.getOffsetVector()),
+ getDataVector().makeTransferPair(target.getDataVector())
+ };
+ }
+
+ @Override
+ public void transfer() {
+ for (TransferPair child:children) {
+ child.transfer();
+ }
+ }
+
+ @Override
+ public ValueVector getTo() {
+ return target;
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ target.allocateNew();
+ for (int i = 0; i < length; i++) {
+ copyValueSafe(startIndex + i, i);
+ }
+ }
+
+ @Override
+ public void copyValueSafe(int srcIndex, int destIndex) {
+ final RepeatedListHolder holder = new RepeatedListHolder();
+ getAccessor().get(srcIndex, holder);
+ target.emptyPopulator.populate(destIndex+1);
+ final TransferPair vectorTransfer = children[1];
+ int newIndex = target.getOffsetVector().getAccessor().get(destIndex);
+ //todo: make this a bulk copy.
+ for (int i = holder.start; i < holder.end; i++, newIndex++) {
+ vectorTransfer.copyValueSafe(i, newIndex);
+ }
+ target.getOffsetVector().getMutator().setSafe(destIndex + 1, newIndex);
+ }
+ }
+
+ public DelegateRepeatedVector(SchemaPath path, BufferAllocator allocator) {
+ this(MaterializedField.create(path, TYPE), allocator);
+ }
+
+ public DelegateRepeatedVector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ emptyPopulator = new EmptyValuePopulator(getOffsetVector());
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryRuntimeException {
+ if (!allocateNewSafe()) {
+ throw new OutOfMemoryRuntimeException();
+ }
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ return makeTransferPair(new DelegateRepeatedVector(ref, allocator));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector target) {
+ return new DelegateTransferPair(DelegateRepeatedVector.class.cast(target));
+ }
+
+ @Override
+ public RepeatedListAccessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public RepeatedListMutator getMutator() {
+ return mutator;
+ }
+
+ @Override
+ public FieldReader getReader() {
+ throw new UnsupportedOperationException();
+ }
+
+ public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) {
+ if(ephPair == null || ephPair.target != from) {
+ ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this));
+ }
+ ephPair.copyValueSafe(fromIndex, thisIndex);
+ }
+
+ }
+
+ protected class RepeatedListTransferPair implements TransferPair {
+ private final TransferPair delegate;
+
+ public RepeatedListTransferPair(TransferPair delegate) {
+ this.delegate = delegate;
+ }
+
+ public void transfer() {
+ delegate.transfer();
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ delegate.splitAndTransfer(startIndex, length);
+ }
+
+ @Override
+ public ValueVector getTo() {
+ final DelegateRepeatedVector delegateVector = DelegateRepeatedVector.class.cast(delegate.getTo());
+ return new RepeatedListVector(getField(), allocator, callBack, delegateVector);
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) {
+ delegate.copyValueSafe(from, to);
+ }
+ }
+
+ public RepeatedListVector(SchemaPath path, BufferAllocator allocator, CallBack callBack) {
+ this(MaterializedField.create(path, TYPE), allocator, callBack);
+ }
+
+ public RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+ this(field, allocator, callBack, new DelegateRepeatedVector(field, allocator));
+ }
+
+ protected RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack, DelegateRepeatedVector delegate) {
+ super(field, allocator, callBack);
+ this.delegate = Preconditions.checkNotNull(delegate);
+
+ final List<MaterializedField> children = Lists.newArrayList(field.getChildren());
+ final int childSize = children.size();
+ assert childSize < 3;
+ final boolean hasChild = childSize > 0;
+ if (hasChild) {
+ // the last field is data field
+ final MaterializedField child = children.get(childSize-1);
+ addOrGetVector(VectorDescriptor.create(child));
+ }
+ }
+
+
+ @Override
+ public RepeatedListReaderImpl getReader() {
+ return reader;
+ }
+
+ @Override
+ public DelegateRepeatedVector.RepeatedListAccessor getAccessor() {
+ return delegate.getAccessor();
+ }
+
+ @Override
+ public DelegateRepeatedVector.RepeatedListMutator getMutator() {
+ return delegate.getMutator();
+ }
+
+ @Override
+ public UInt4Vector getOffsetVector() {
+ return delegate.getOffsetVector();
+ }
+
+ @Override
+ public ValueVector getDataVector() {
+ return delegate.getDataVector();
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryRuntimeException {
+ delegate.allocateNew();
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ return delegate.allocateNewSafe();
+ }
+
+ @Override
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ final AddOrGetResult<T> result = delegate.addOrGetVector(descriptor);
+ if (result.isCreated() && callBack != null) {
+ callBack.doWork();
+ }
+ return result;
+ }
+
+ @Override
+ public int size() {
+ return delegate.size();
+ }
+
+ @Override
+ public int getBufferSize() {
+ return delegate.getBufferSize();
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ return delegate.getBufferSizeFor(valueCount);
+ }
+
+ @Override
+ public void close() {
+ delegate.close();
+ }
+
+ @Override
+ public void clear() {
+ delegate.clear();
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ return new RepeatedListTransferPair(delegate.getTransferPair());
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ return new RepeatedListTransferPair(delegate.getTransferPair(ref));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ final RepeatedListVector target = RepeatedListVector.class.cast(to);
+ return new RepeatedListTransferPair(delegate.makeTransferPair(target.delegate));
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return delegate.getValueCapacity();
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ return delegate.getBuffers(clear);
+ }
+
+
+ @Override
+ public void load(SerializedField metadata, DrillBuf buf) {
+ delegate.load(metadata, buf);
+ }
+
+ @Override
+ public SerializedField getMetadata() {
+ return delegate.getMetadata();
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ return delegate.iterator();
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ delegate.setInitialCapacity(numRecords);
+ }
+
+ /**
+ * @deprecated
+ * prefer using {@link #addOrGetVector(org.apache.drill.exec.vector.VectorDescriptor)} instead.
+ */
+ @Override
+ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+ final AddOrGetResult<T> result = addOrGetVector(VectorDescriptor.create(type));
+ return result.getVector();
+ }
+
+ @Override
+ public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+ if (name != null) {
+ return null;
+ }
+ return typeify(delegate.getDataVector(), clazz);
+ }
+
+ @Override
+ public void allocateNew(int valueCount, int innerValueCount) {
+ clear();
+ getOffsetVector().allocateNew(valueCount + 1);
+ getMutator().reset();
+ }
+
+ @Override
+ public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
+ if (name != null) {
+ return null;
+ }
+ return new VectorWithOrdinal(delegate.getDataVector(), 0);
+ }
+
+ public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) {
+ delegate.copyFromSafe(fromIndex, thisIndex, from.delegate);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
new file mode 100644
index 0000000..618e3a3
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -0,0 +1,611 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import io.netty.buffer.DrillBuf;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.holders.ComplexHolder;
+import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.vector.AddOrGetResult;
+import org.apache.drill.exec.vector.AllocationHelper;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+import org.apache.drill.exec.vector.complex.impl.NullReader;
+import org.apache.drill.exec.vector.complex.impl.RepeatedMapReaderImpl;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+public class RepeatedMapVector extends AbstractMapVector
+ implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
+ //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
+
+ public final static MajorType TYPE = MajorType.newBuilder().setMinorType(MinorType.MAP).setMode(DataMode.REPEATED).build();
+
+ private final UInt4Vector offsets; // offsets to start of each record (considering record indices are 0-indexed)
+ private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
+ private final RepeatedMapAccessor accessor = new RepeatedMapAccessor();
+ private final Mutator mutator = new Mutator();
+ private final EmptyValuePopulator emptyPopulator;
+
+ public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+ super(field, allocator, callBack);
+ this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator);
+ this.emptyPopulator = new EmptyValuePopulator(offsets);
+ }
+
+ @Override
+ public UInt4Vector getOffsetVector() {
+ return offsets;
+ }
+
+ @Override
+ public ValueVector getDataVector() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ offsets.setInitialCapacity(numRecords + 1);
+ for(final ValueVector v : (Iterable<ValueVector>) this) {
+ v.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+ }
+ }
+
+ @Override
+ public RepeatedMapReaderImpl getReader() {
+ return reader;
+ }
+
+ @Override
+ public void allocateNew(int groupCount, int innerValueCount) {
+ clear();
+ try {
+ offsets.allocateNew(groupCount + 1);
+ for (ValueVector v : getChildren()) {
+ AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
+ }
+ } catch (OutOfMemoryRuntimeException e){
+ clear();
+ throw e;
+ }
+ offsets.zeroVector();
+ mutator.reset();
+ }
+
+ public Iterator<String> fieldNameIterator() {
+ return getChildFieldNames().iterator();
+ }
+
+ @Override
+ public List<ValueVector> getPrimitiveVectors() {
+ final List<ValueVector> primitiveVectors = super.getPrimitiveVectors();
+ primitiveVectors.add(offsets);
+ return primitiveVectors;
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (getAccessor().getValueCount() == 0) {
+ return 0;
+ }
+ long bufferSize = offsets.getBufferSize();
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ bufferSize += v.getBufferSize();
+ }
+ return (int) bufferSize;
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ long bufferSize = 0;
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ bufferSize += v.getBufferSizeFor(valueCount);
+ }
+
+ return (int) bufferSize;
+ }
+
+ @Override
+ public void close() {
+ offsets.close();
+ super.close();
+ }
+
+ @Override
+ public TransferPair getTransferPair() {
+ return new RepeatedMapTransferPair(this, getField().getPath());
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new RepeatedMapTransferPair(this, (RepeatedMapVector)to);
+ }
+
+ MapSingleCopier makeSingularCopier(MapVector to) {
+ return new MapSingleCopier(this, to);
+ }
+
+ protected static class MapSingleCopier {
+ private final TransferPair[] pairs;
+ public final RepeatedMapVector from;
+
+ public MapSingleCopier(RepeatedMapVector from, MapVector to) {
+ this.from = from;
+ this.pairs = new TransferPair[from.size()];
+
+ int i = 0;
+ ValueVector vector;
+ for (final String child:from.getChildFieldNames()) {
+ int preSize = to.size();
+ vector = from.getChild(child);
+ if (vector == null) {
+ continue;
+ }
+ final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+ if (to.size() != preSize) {
+ newVector.allocateNew();
+ }
+ pairs[i++] = vector.makeTransferPair(newVector);
+ }
+ }
+
+ public void copySafe(int fromSubIndex, int toIndex) {
+ for (TransferPair p : pairs) {
+ p.copyValueSafe(fromSubIndex, toIndex);
+ }
+ }
+ }
+
+ @Override
+ public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
+ if (seg != null && seg.isArray() && !seg.isLastPath()) {
+ if (addToBreadCrumb) {
+ addToBreadCrumb = false;
+ builder.remainder(seg);
+ }
+ // skip the first array segment as there is no corresponding child vector.
+ seg = seg.getChild();
+
+ // multi-level numbered access to a repeated map is not possible so return if the next part is also an array
+ // segment.
+ if (seg.isArray()) {
+ return null;
+ }
+ }
+
+ return super.getFieldIdIfMatches(builder, addToBreadCrumb, seg);
+ }
+
+ public TransferPair getTransferPairToSingleMap(FieldReference reference) {
+ return new SingleMapTransferPair(this, reference);
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref) {
+ return new RepeatedMapTransferPair(this, ref);
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if (!offsets.allocateNewSafe()) {
+ return false;
+ }
+ success = super.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ offsets.zeroVector();
+ return success;
+ }
+
+ protected static class SingleMapTransferPair implements TransferPair {
+ private final TransferPair[] pairs;
+ private final RepeatedMapVector from;
+ private final MapVector to;
+ private static final MajorType MAP_TYPE = Types.required(MinorType.MAP);
+
+ public SingleMapTransferPair(RepeatedMapVector from, SchemaPath path) {
+ this(from, new MapVector(MaterializedField.create(path, MAP_TYPE), from.allocator, from.callBack), false);
+ }
+
+ public SingleMapTransferPair(RepeatedMapVector from, MapVector to) {
+ this(from, to, true);
+ }
+
+ public SingleMapTransferPair(RepeatedMapVector from, MapVector to, boolean allocate) {
+ this.from = from;
+ this.to = to;
+ this.pairs = new TransferPair[from.size()];
+ int i = 0;
+ ValueVector vector;
+ for (final String child : from.getChildFieldNames()) {
+ int preSize = to.size();
+ vector = from.getChild(child);
+ if (vector == null) {
+ continue;
+ }
+ final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+ if (allocate && to.size() != preSize) {
+ newVector.allocateNew();
+ }
+ pairs[i++] = vector.makeTransferPair(newVector);
+ }
+ }
+
+
+ @Override
+ public void transfer() {
+ for (TransferPair p : pairs) {
+ p.transfer();
+ }
+ to.getMutator().setValueCount(from.getAccessor().getValueCount());
+ from.clear();
+ }
+
+ @Override
+ public ValueVector getTo() {
+ return to;
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) {
+ for (TransferPair p : pairs) {
+ p.copyValueSafe(from, to);
+ }
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ for (TransferPair p : pairs) {
+ p.splitAndTransfer(startIndex, length);
+ }
+ to.getMutator().setValueCount(length);
+ }
+ }
+
+ private static class RepeatedMapTransferPair implements TransferPair{
+
+ private final TransferPair[] pairs;
+ private final RepeatedMapVector to;
+ private final RepeatedMapVector from;
+
+ public RepeatedMapTransferPair(RepeatedMapVector from, SchemaPath path) {
+ this(from, new RepeatedMapVector(MaterializedField.create(path, TYPE), from.allocator, from.callBack), false);
+ }
+
+ public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to) {
+ this(from, to, true);
+ }
+
+ public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to, boolean allocate) {
+ this.from = from;
+ this.to = to;
+ this.pairs = new TransferPair[from.size()];
+ this.to.ephPair = null;
+
+ int i = 0;
+ ValueVector vector;
+ for (final String child : from.getChildFieldNames()) {
+ final int preSize = to.size();
+ vector = from.getChild(child);
+ if (vector == null) {
+ continue;
+ }
+
+ final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+ if (to.size() != preSize) {
+ newVector.allocateNew();
+ }
+
+ pairs[i++] = vector.makeTransferPair(newVector);
+ }
+ }
+
+ @Override
+ public void transfer() {
+ from.offsets.transferTo(to.offsets);
+ for (TransferPair p : pairs) {
+ p.transfer();
+ }
+ from.clear();
+ }
+
+ @Override
+ public ValueVector getTo() {
+ return to;
+ }
+
+ @Override
+ public void copyValueSafe(int srcIndex, int destIndex) {
+ RepeatedMapHolder holder = new RepeatedMapHolder();
+ from.getAccessor().get(srcIndex, holder);
+ to.emptyPopulator.populate(destIndex + 1);
+ int newIndex = to.offsets.getAccessor().get(destIndex);
+ //todo: make these bulk copies
+ for (int i = holder.start; i < holder.end; i++, newIndex++) {
+ for (TransferPair p : pairs) {
+ p.copyValueSafe(i, newIndex);
+ }
+ }
+ to.offsets.getMutator().setSafe(destIndex + 1, newIndex);
+ }
+
+ @Override
+ public void splitAndTransfer(final int groupStart, final int groups) {
+ final UInt4Vector.Accessor a = from.offsets.getAccessor();
+ final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+ final int startPos = a.get(groupStart);
+ final int endPos = a.get(groupStart + groups);
+ final int valuesToCopy = endPos - startPos;
+
+ to.offsets.clear();
+ to.offsets.allocateNew(groups + 1);
+
+ int normalizedPos;
+ for (int i = 0; i < groups + 1; i++) {
+ normalizedPos = a.get(groupStart + i) - startPos;
+ m.set(i, normalizedPos);
+ }
+
+ m.setValueCount(groups + 1);
+ to.emptyPopulator.populate(groups);
+
+ for (final TransferPair p : pairs) {
+ p.splitAndTransfer(startPos, valuesToCopy);
+ }
+ }
+ }
+
+
+ transient private RepeatedMapTransferPair ephPair;
+
+ public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) {
+ if (ephPair == null || ephPair.from != from) {
+ ephPair = (RepeatedMapTransferPair) from.makeTransferPair(this);
+ }
+ ephPair.copyValueSafe(fromIndex, thisIndex);
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return Math.max(offsets.getValueCapacity() - 1, 0);
+ }
+
+ @Override
+ public RepeatedMapAccessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ final int expectedBufferSize = getBufferSize();
+ final int actualBufferSize = super.getBufferSize();
+
+ Preconditions.checkArgument(expectedBufferSize == actualBufferSize + offsets.getBufferSize());
+ return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
+ }
+
+
+ @Override
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ final List<SerializedField> children = metadata.getChildList();
+
+ final SerializedField offsetField = children.get(0);
+ offsets.load(offsetField, buffer);
+ int bufOffset = offsetField.getBufferLength();
+
+ for (int i = 1; i < children.size(); i++) {
+ final SerializedField child = children.get(i);
+ final MaterializedField fieldDef = MaterializedField.create(child);
+ ValueVector vector = getChild(fieldDef.getLastName());
+ if (vector == null) {
+ // if we arrive here, we didn't have a matching vector.
+ vector = TypeHelper.getNewVector(fieldDef, allocator);
+ putChild(fieldDef.getLastName(), vector);
+ }
+ final int vectorLength = child.getBufferLength();
+ vector.load(child, buffer.slice(bufOffset, vectorLength));
+ bufOffset += vectorLength;
+ }
+
+ assert bufOffset == buffer.capacity();
+ }
+
+
+ @Override
+ public SerializedField getMetadata() {
+ SerializedField.Builder builder = getField() //
+ .getAsBuilder() //
+ .setBufferLength(getBufferSize()) //
+ // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
+ .setValueCount(accessor.getValueCount());
+ builder.addChild(offsets.getMetadata());
+ for (final ValueVector child : getChildren()) {
+ builder.addChild(child.getMetadata());
+ }
+ return builder.build();
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ public class RepeatedMapAccessor implements RepeatedAccessor {
+ @Override
+ public Object getObject(int index) {
+ final List<Object> list = new JsonStringArrayList<>();
+ final int end = offsets.getAccessor().get(index+1);
+ String fieldName;
+ for (int i = offsets.getAccessor().get(index); i < end; i++) {
+ final Map<String, Object> vv = Maps.newLinkedHashMap();
+ for (final MaterializedField field : getField().getChildren()) {
+ if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
+ fieldName = field.getLastName();
+ final Object value = getChild(fieldName).getAccessor().getObject(i);
+ if (value != null) {
+ vv.put(fieldName, value);
+ }
+ }
+ }
+ list.add(vv);
+ }
+ return list;
+ }
+
+ @Override
+ public int getValueCount() {
+ return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
+ }
+
+ @Override
+ public int getInnerValueCount() {
+ final int valueCount = getValueCount();
+ if (valueCount == 0) {
+ return 0;
+ }
+ return offsets.getAccessor().get(valueCount);
+ }
+
+ @Override
+ public int getInnerValueCountAt(int index) {
+ return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+ }
+
+ @Override
+ public boolean isEmpty(int index) {
+ return false;
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return false;
+ }
+
+ public void get(int index, RepeatedMapHolder holder) {
+ assert index < getValueCapacity() :
+ String.format("Attempted to access index %d when value capacity is %d",
+ index, getValueCapacity());
+ final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
+ holder.start = offsetsAccessor.get(index);
+ holder.end = offsetsAccessor.get(index + 1);
+ }
+
+ public void get(int index, ComplexHolder holder) {
+ final FieldReader reader = getReader();
+ reader.setPosition(index);
+ holder.reader = reader;
+ }
+
+ public void get(int index, int arrayIndex, ComplexHolder holder) {
+ final RepeatedMapHolder h = new RepeatedMapHolder();
+ get(index, h);
+ final int offset = h.start + arrayIndex;
+
+ if (offset >= h.end) {
+ holder.reader = NullReader.INSTANCE;
+ } else {
+ reader.setSinglePosition(index, arrayIndex);
+ holder.reader = reader;
+ }
+ }
+ }
+
+ public class Mutator implements RepeatedMutator {
+ @Override
+ public void startNewValue(int index) {
+ emptyPopulator.populate(index + 1);
+ offsets.getMutator().setSafe(index + 1, offsets.getAccessor().get(index));
+ }
+
+ @Override
+ public void setValueCount(int topLevelValueCount) {
+ emptyPopulator.populate(topLevelValueCount);
+ offsets.getMutator().setValueCount(topLevelValueCount == 0 ? 0 : topLevelValueCount + 1);
+ int childValueCount = offsets.getAccessor().get(topLevelValueCount);
+ for (final ValueVector v : getChildren()) {
+ v.getMutator().setValueCount(childValueCount);
+ }
+ }
+
+ @Override
+ public void reset() {}
+
+ @Override
+ public void generateTestData(int values) {}
+
+ public int add(int index) {
+ final int prevEnd = offsets.getAccessor().get(index + 1);
+ offsets.getMutator().setSafe(index + 1, prevEnd + 1);
+ return prevEnd;
+ }
+ }
+
+ @Override
+ public void clear() {
+ getMutator().reset();
+
+ offsets.clear();
+ for(final ValueVector vector : getChildren()) {
+ vector.clear();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
new file mode 100644
index 0000000..0fba292
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedValueVector.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.ContainerVectorLike;
+
+/**
+ * An abstraction representing repeated value vectors.
+ *
+ * A repeated vector contains values that may either be flat or nested. A value consists of zero or more cells(inner values).
+ * Current design maintains data and offsets vectors. Each cell is stored in the data vector. Repeated vector
+ * uses the offset vector to determine the sequence of cells pertaining to an individual value.
+ *
+ */
+public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
+
+ final static int DEFAULT_REPEAT_PER_RECORD = 5;
+
+ /**
+ * Returns the underlying offset vector or null if none exists.
+ *
+ * TODO(DRILL-2995): eliminate exposing low-level interfaces.
+ */
+ UInt4Vector getOffsetVector();
+
+ /**
+ * Returns the underlying data vector or null if none exists.
+ */
+ ValueVector getDataVector();
+
+ @Override
+ RepeatedAccessor getAccessor();
+
+ @Override
+ RepeatedMutator getMutator();
+
+ interface RepeatedAccessor extends ValueVector.Accessor {
+ /**
+ * Returns total number of cells that vector contains.
+ *
+ * The result includes empty, null valued cells.
+ */
+ int getInnerValueCount();
+
+
+ /**
+ * Returns number of cells that the value at the given index contains.
+ */
+ int getInnerValueCountAt(int index);
+
+ /**
+ * Returns true if the value at the given index is empty, false otherwise.
+ *
+ * @param index value index
+ */
+ boolean isEmpty(int index);
+ }
+
+ interface RepeatedMutator extends ValueVector.Mutator {
+ /**
+ * Starts a new value that is a container of cells.
+ *
+ * @param index index of new value to start
+ */
+ void startNewValue(int index);
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
new file mode 100644
index 0000000..67954e7
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedVariableWidthVectorLike.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+public interface RepeatedVariableWidthVectorLike {
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Desired size of the underlying data buffer.
+ * @param parentValueCount Number of separate repeating groupings.
+ * @param childValueCount Number of supported values in the vector.
+ */
+ void allocateNew(int totalBytes, int parentValueCount, int childValueCount);
+
+ /**
+ * Provide the maximum amount of variable width bytes that can be stored int his vector.
+ * @return
+ */
+ int getByteCapacity();
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java
new file mode 100644
index 0000000..f5ed3a0
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/StateTool.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import java.util.Arrays;
+
+public class StateTool {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StateTool.class);
+
+ public static <T extends Enum<?>> void check(T currentState, T... expectedStates) {
+ for (T s : expectedStates) {
+ if (s == currentState) {
+ return;
+ }
+ }
+ throw new IllegalArgumentException(String.format("Expected to be in one of these states %s but was actuall in state %s", Arrays.toString(expectedStates), currentState));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/VectorWithOrdinal.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/VectorWithOrdinal.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/VectorWithOrdinal.java
new file mode 100644
index 0000000..a0c8dd8
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/VectorWithOrdinal.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import org.apache.drill.exec.vector.ValueVector;
+
+public class VectorWithOrdinal {
+ public final ValueVector vector;
+ public final int ordinal;
+
+ public VectorWithOrdinal(ValueVector v, int ordinal) {
+ this.vector = v;
+ this.ordinal = ordinal;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
new file mode 100644
index 0000000..05fe63f
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.impl;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.holders.UnionHolder;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
+
+
+abstract class AbstractBaseReader implements FieldReader{
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseReader.class);
+
+ private int index;
+
+ public AbstractBaseReader() {
+ super();
+ }
+
+ public void setPosition(int index){
+ this.index = index;
+ }
+
+ int idx(){
+ return index;
+ }
+
+ @Override
+ public void reset() {
+ index = 0;
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ throw new IllegalStateException("The current reader doesn't support reading as a map.");
+ }
+
+ public MajorType getType(){
+ throw new IllegalStateException("The current reader doesn't support getting type information.");
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return MaterializedField.create("unknown", Types.LATE_BIND_TYPE);
+ }
+
+ @Override
+ public boolean next() {
+ throw new IllegalStateException("The current reader doesn't support getting next information.");
+ }
+
+ @Override
+ public int size() {
+ throw new IllegalStateException("The current reader doesn't support getting size information.");
+ }
+
+ @Override
+ public void read(UnionHolder holder) {
+ holder.reader = this;
+ holder.isSet = this.isSet() ? 1 : 0;
+ }
+
+ @Override
+ public void read(int index, UnionHolder holder) {
+ throw new IllegalStateException("The current reader doesn't support reading union type");
+ }
+
+ @Override
+ public void copyAsValue(UnionWriter writer) {
+ throw new IllegalStateException("The current reader doesn't support reading union type");
+ }
+
+ @Override
+ public void copyAsValue(ListWriter writer) {
+ ComplexCopier.copy(this, (FieldWriter)writer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseWriter.java
new file mode 100644
index 0000000..bd16223
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseWriter.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.impl;
+
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
+
+
+abstract class AbstractBaseWriter implements FieldWriter {
+ //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseWriter.class);
+
+ final FieldWriter parent;
+ private int index;
+
+ public AbstractBaseWriter(FieldWriter parent) {
+ this.parent = parent;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + "[index = " + index + ", parent = " + parent + "]";
+ }
+
+ @Override
+ public FieldWriter getParent() {
+ return parent;
+ }
+
+ public boolean isRoot() {
+ return parent == null;
+ }
+
+ int idx() {
+ return index;
+ }
+
+ @Override
+ public void setPosition(int index) {
+ this.index = index;
+ }
+
+ @Override
+ public void end() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
new file mode 100644
index 0000000..5575b5e
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.impl;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.MapVector;
+import org.apache.drill.exec.vector.complex.StateTool;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+
+import com.google.common.base.Preconditions;
+
+public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
+// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
+
+ private SingleMapWriter mapRoot;
+ private SingleListWriter listRoot;
+ private final MapVector container;
+
+ Mode mode = Mode.INIT;
+ private final String name;
+ private final boolean unionEnabled;
+
+ private enum Mode { INIT, MAP, LIST };
+
+ public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){
+ super(null);
+ this.name = name;
+ this.container = container;
+ this.unionEnabled = unionEnabled;
+ }
+
+ public ComplexWriterImpl(String name, MapVector container){
+ this(name, container, false);
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return container.getField();
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return container.getValueCapacity();
+ }
+
+ private void check(Mode... modes){
+ StateTool.check(mode, modes);
+ }
+
+ @Override
+ public void reset(){
+ setPosition(0);
+ }
+
+ @Override
+ public void close() throws Exception {
+ clear();
+ mapRoot.close();
+ if (listRoot != null) {
+ listRoot.close();
+ }
+ }
+
+ @Override
+ public void clear(){
+ switch(mode){
+ case MAP:
+ mapRoot.clear();
+ break;
+ case LIST:
+ listRoot.clear();
+ break;
+ }
+ }
+
+ @Override
+ public void setValueCount(int count){
+ switch(mode){
+ case MAP:
+ mapRoot.setValueCount(count);
+ break;
+ case LIST:
+ listRoot.setValueCount(count);
+ break;
+ }
+ }
+
+ @Override
+ public void setPosition(int index){
+ super.setPosition(index);
+ switch(mode){
+ case MAP:
+ mapRoot.setPosition(index);
+ break;
+ case LIST:
+ listRoot.setPosition(index);
+ break;
+ }
+ }
+
+
+ public MapWriter directMap(){
+ Preconditions.checkArgument(name == null);
+
+ switch(mode){
+
+ case INIT:
+ MapVector map = (MapVector) container;
+ mapRoot = new SingleMapWriter(map, this, unionEnabled);
+ mapRoot.setPosition(idx());
+ mode = Mode.MAP;
+ break;
+
+ case MAP:
+ break;
+
+ default:
+ check(Mode.INIT, Mode.MAP);
+ }
+
+ return mapRoot;
+ }
+
+ @Override
+ public MapWriter rootAsMap() {
+ switch(mode){
+
+ case INIT:
+ MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
+ mapRoot = new SingleMapWriter(map, this, unionEnabled);
+ mapRoot.setPosition(idx());
+ mode = Mode.MAP;
+ break;
+
+ case MAP:
+ break;
+
+ default:
+ check(Mode.INIT, Mode.MAP);
+ }
+
+ return mapRoot;
+ }
+
+
+ @Override
+ public void allocate() {
+ if(mapRoot != null) {
+ mapRoot.allocate();
+ } else if(listRoot != null) {
+ listRoot.allocate();
+ }
+ }
+
+ @Override
+ public ListWriter rootAsList() {
+ switch(mode){
+
+ case INIT:
+ listRoot = new SingleListWriter(name, container, this);
+ listRoot.setPosition(idx());
+ mode = Mode.LIST;
+ break;
+
+ case LIST:
+ break;
+
+ default:
+ check(Mode.INIT, Mode.MAP);
+ }
+
+ return listRoot;
+ }
+
+ private static class VectorAccessibleFacade extends MapVector {
+
+ private final VectorContainer vc;
+
+ public VectorAccessibleFacade(VectorContainer vc) {
+ super("", null, null);
+ this.vc = vc;
+ }
+
+ @Override
+ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+ final ValueVector v = vc.addOrGet(name, type, clazz);
+ putChild(name, v);
+ return this.typeify(v, clazz);
+
+ }
+
+ }
+
+ public static ComplexWriter getWriter(String name, VectorContainer container){
+ VectorAccessibleFacade vc = new VectorAccessibleFacade(container);
+ return new ComplexWriterImpl(name, vc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
new file mode 100644
index 0000000..5f57c01
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.impl;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VectorDescriptor;
+import org.apache.drill.exec.vector.ZeroVector;
+import org.apache.drill.exec.vector.complex.AbstractMapVector;
+import org.apache.drill.exec.vector.complex.ListVector;
+import org.apache.drill.exec.vector.complex.UnionVector;
+import org.apache.drill.exec.vector.complex.writer.FieldWriter;
+
+import java.lang.reflect.Constructor;
+
+/**
+ * This FieldWriter implementation delegates all FieldWriter API calls to an inner FieldWriter. This inner field writer
+ * can start as a specific type, and this class will promote the writer to a UnionWriter if a call is made that the specifically
+ * typed writer cannot handle. A new UnionVector is created, wrapping the original vector, and replaces the original vector
+ * in the parent vector, which can be either an AbstractMapVector or a ListVector.
+ */
+public class PromotableWriter extends AbstractPromotableFieldWriter {
+
+ private final AbstractMapVector parentContainer;
+ private final ListVector listVector;
+ private int position;
+
+ private enum State {
+ UNTYPED, SINGLE, UNION
+ }
+
+ private MinorType type;
+ private ValueVector vector;
+ private UnionVector unionVector;
+ private State state;
+ private FieldWriter writer;
+
+ public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
+ super(null);
+ this.parentContainer = parentContainer;
+ this.listVector = null;
+ init(v);
+ }
+
+ public PromotableWriter(ValueVector v, ListVector listVector) {
+ super(null);
+ this.listVector = listVector;
+ this.parentContainer = null;
+ init(v);
+ }
+
+ private void init(ValueVector v) {
+ if (v instanceof UnionVector) {
+ state = State.UNION;
+ unionVector = (UnionVector) v;
+ writer = new UnionWriter(unionVector);
+ } else if (v instanceof ZeroVector) {
+ state = State.UNTYPED;
+ } else {
+ setWriter(v);
+ }
+ }
+
+ private void setWriter(ValueVector v) {
+ state = State.SINGLE;
+ vector = v;
+ type = v.getField().getType().getMinorType();
+ Class writerClass = TypeHelper.getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
+ if (writerClass.equals(SingleListWriter.class)) {
+ writerClass = UnionListWriter.class;
+ }
+ Class vectorClass = TypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField().getDataMode());
+ try {
+ Constructor constructor = null;
+ for (Constructor c : writerClass.getConstructors()) {
+ if (c.getParameterTypes().length == 3) {
+ constructor = c;
+ }
+ }
+ if (constructor == null) {
+ constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class);
+ writer = (FieldWriter) constructor.newInstance(vector, null);
+ } else {
+ writer = (FieldWriter) constructor.newInstance(vector, null, true);
+ }
+ } catch (ReflectiveOperationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void setPosition(int index) {
+ super.setPosition(index);
+ FieldWriter w = getWriter();
+ if (w == null) {
+ position = index;
+ } else {
+ w.setPosition(index);
+ }
+ }
+
+ protected FieldWriter getWriter(MinorType type) {
+ if (state == State.UNION) {
+ return writer;
+ }
+ if (state == State.UNTYPED) {
+ if (type == null) {
+ return null;
+ }
+ ValueVector v = listVector.addOrGetVector(new VectorDescriptor(Types.optional(type))).getVector();
+ v.allocateNew();
+ setWriter(v);
+ writer.setPosition(position);
+ }
+ if (type != this.type) {
+ return promoteToUnion();
+ }
+ return writer;
+ }
+
+ @Override
+ public boolean isEmptyMap() {
+ return writer.isEmptyMap();
+ }
+
+ protected FieldWriter getWriter() {
+ return getWriter(type);
+ }
+
+ private FieldWriter promoteToUnion() {
+ String name = vector.getField().getLastName();
+ TransferPair tp = vector.getTransferPair(new FieldReference(vector.getField().getType().getMinorType().name().toLowerCase()));
+ tp.transfer();
+ if (parentContainer != null) {
+ unionVector = parentContainer.addOrGet(name, Types.optional(MinorType.UNION), UnionVector.class);
+ } else if (listVector != null) {
+ unionVector = listVector.promoteToUnion();
+ }
+ unionVector.addVector(tp.getTo());
+ writer = new UnionWriter(unionVector);
+ writer.setPosition(idx());
+ for (int i = 0; i < idx(); i++) {
+ unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
+ }
+ vector = null;
+ state = State.UNION;
+ return writer;
+ }
+
+ @Override
+ public void allocate() {
+ getWriter().allocate();
+ }
+
+ @Override
+ public void clear() {
+ getWriter().clear();
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return getWriter().getField();
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return getWriter().getValueCapacity();
+ }
+
+ @Override
+ public void close() throws Exception {
+ getWriter().close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java
new file mode 100644
index 0000000..36e9bee
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedListReaderImpl.java
@@ -0,0 +1,145 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector.complex.impl;
+
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.holders.RepeatedListHolder;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedListVector;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+
+public class RepeatedListReaderImpl extends AbstractFieldReader{
+ private static final int NO_VALUES = Integer.MAX_VALUE - 1;
+ private static final MajorType TYPE = Types.repeated(MinorType.LIST);
+ private final String name;
+ private final RepeatedListVector container;
+ private FieldReader reader;
+
+ public RepeatedListReaderImpl(String name, RepeatedListVector container) {
+ super();
+ this.name = name;
+ this.container = container;
+ }
+
+ @Override
+ public MajorType getType() {
+ return TYPE;
+ }
+
+ @Override
+ public void copyAsValue(ListWriter writer) {
+ if (currentOffset == NO_VALUES) {
+ return;
+ }
+ RepeatedListWriter impl = (RepeatedListWriter) writer;
+ impl.container.copyFromSafe(idx(), impl.idx(), container);
+ }
+
+ @Override
+ public void copyAsField(String name, MapWriter writer) {
+ if (currentOffset == NO_VALUES) {
+ return;
+ }
+ RepeatedListWriter impl = (RepeatedListWriter) writer.list(name);
+ impl.container.copyFromSafe(idx(), impl.idx(), container);
+ }
+
+ private int currentOffset;
+ private int maxOffset;
+
+ @Override
+ public void reset() {
+ super.reset();
+ currentOffset = 0;
+ maxOffset = 0;
+ if (reader != null) {
+ reader.reset();
+ }
+ reader = null;
+ }
+
+ @Override
+ public int size() {
+ return maxOffset - currentOffset;
+ }
+
+ @Override
+ public void setPosition(int index) {
+ if (index < 0 || index == NO_VALUES) {
+ currentOffset = NO_VALUES;
+ return;
+ }
+
+ super.setPosition(index);
+ RepeatedListHolder h = new RepeatedListHolder();
+ container.getAccessor().get(index, h);
+ if (h.start == h.end) {
+ currentOffset = NO_VALUES;
+ } else {
+ currentOffset = h.start-1;
+ maxOffset = h.end;
+ if(reader != null) {
+ reader.setPosition(currentOffset);
+ }
+ }
+ }
+
+ @Override
+ public boolean next() {
+ if (currentOffset +1 < maxOffset) {
+ currentOffset++;
+ if (reader != null) {
+ reader.setPosition(currentOffset);
+ }
+ return true;
+ } else {
+ currentOffset = NO_VALUES;
+ return false;
+ }
+ }
+
+ @Override
+ public Object readObject() {
+ return container.getAccessor().getObject(idx());
+ }
+
+ @Override
+ public FieldReader reader() {
+ if (reader == null) {
+ ValueVector child = container.getChild(name);
+ if (child == null) {
+ reader = NullReader.INSTANCE;
+ } else {
+ reader = child.getReader();
+ }
+ reader.setPosition(currentOffset);
+ }
+ return reader;
+ }
+
+ public boolean isSet() {
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
new file mode 100644
index 0000000..b2fe7b7
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/RepeatedMapReaderImpl.java
@@ -0,0 +1,192 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector.complex.impl;
+
+import java.util.Map;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.RepeatedMapVector;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+
+import com.google.common.collect.Maps;
+
+@SuppressWarnings("unused")
+public class RepeatedMapReaderImpl extends AbstractFieldReader{
+ private static final int NO_VALUES = Integer.MAX_VALUE - 1;
+
+ private final RepeatedMapVector vector;
+ private final Map<String, FieldReader> fields = Maps.newHashMap();
+
+ public RepeatedMapReaderImpl(RepeatedMapVector vector) {
+ this.vector = vector;
+ }
+
+ private void setChildrenPosition(int index) {
+ for (FieldReader r : fields.values()) {
+ r.setPosition(index);
+ }
+ }
+
+ @Override
+ public FieldReader reader(String name) {
+ FieldReader reader = fields.get(name);
+ if (reader == null) {
+ ValueVector child = vector.getChild(name);
+ if (child == null) {
+ reader = NullReader.INSTANCE;
+ } else {
+ reader = child.getReader();
+ }
+ fields.put(name, reader);
+ reader.setPosition(currentOffset);
+ }
+ return reader;
+ }
+
+ @Override
+ public FieldReader reader() {
+ if (currentOffset == NO_VALUES) {
+ return NullReader.INSTANCE;
+ }
+
+ setChildrenPosition(currentOffset);
+ return new SingleLikeRepeatedMapReaderImpl(vector, this);
+ }
+
+ private int currentOffset;
+ private int maxOffset;
+
+ @Override
+ public void reset() {
+ super.reset();
+ currentOffset = 0;
+ maxOffset = 0;
+ for (FieldReader reader:fields.values()) {
+ reader.reset();
+ }
+ fields.clear();
+ }
+
+ @Override
+ public int size() {
+ if (isNull()) {
+ return 0;
+ }
+ return maxOffset - (currentOffset < 0 ? 0 : currentOffset);
+ }
+
+ @Override
+ public void setPosition(int index) {
+ if (index < 0 || index == NO_VALUES) {
+ currentOffset = NO_VALUES;
+ return;
+ }
+
+ super.setPosition(index);
+ RepeatedMapHolder h = new RepeatedMapHolder();
+ vector.getAccessor().get(index, h);
+ if (h.start == h.end) {
+ currentOffset = NO_VALUES;
+ } else {
+ currentOffset = h.start-1;
+ maxOffset = h.end;
+ setChildrenPosition(currentOffset);
+ }
+ }
+
+ public void setSinglePosition(int index, int childIndex) {
+ super.setPosition(index);
+ RepeatedMapHolder h = new RepeatedMapHolder();
+ vector.getAccessor().get(index, h);
+ if (h.start == h.end) {
+ currentOffset = NO_VALUES;
+ } else {
+ int singleOffset = h.start + childIndex;
+ assert singleOffset < h.end;
+ currentOffset = singleOffset;
+ maxOffset = singleOffset + 1;
+ setChildrenPosition(singleOffset);
+ }
+ }
+
+ @Override
+ public boolean next() {
+ if (currentOffset +1 < maxOffset) {
+ setChildrenPosition(++currentOffset);
+ return true;
+ } else {
+ currentOffset = NO_VALUES;
+ return false;
+ }
+ }
+
+ public boolean isNull() {
+ return currentOffset == NO_VALUES;
+ }
+
+ @Override
+ public Object readObject() {
+ return vector.getAccessor().getObject(idx());
+ }
+
+ @Override
+ public MajorType getType() {
+ return vector.getField().getType();
+ }
+
+ @Override
+ public java.util.Iterator<String> iterator() {
+ return vector.fieldNameIterator();
+ }
+
+ @Override
+ public boolean isSet() {
+ return true;
+ }
+
+ @Override
+ public void copyAsValue(MapWriter writer) {
+ if (currentOffset == NO_VALUES) {
+ return;
+ }
+ RepeatedMapWriter impl = (RepeatedMapWriter) writer;
+ impl.container.copyFromSafe(idx(), impl.idx(), vector);
+ }
+
+ public void copyAsValueSingle(MapWriter writer) {
+ if (currentOffset == NO_VALUES) {
+ return;
+ }
+ SingleMapWriter impl = (SingleMapWriter) writer;
+ impl.container.copyFromSafe(currentOffset, impl.idx(), vector);
+ }
+
+ @Override
+ public void copyAsField(String name, MapWriter writer) {
+ if (currentOffset == NO_VALUES) {
+ return;
+ }
+ RepeatedMapWriter impl = (RepeatedMapWriter) writer.map(name);
+ impl.container.copyFromSafe(idx(), impl.idx(), vector);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
new file mode 100644
index 0000000..3f89a9f
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.vector.complex.impl;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.vector.complex.RepeatedMapVector;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+
+public class SingleLikeRepeatedMapReaderImpl extends AbstractFieldReader{
+
+ private RepeatedMapReaderImpl delegate;
+
+ public SingleLikeRepeatedMapReaderImpl(RepeatedMapVector vector, FieldReader delegate) {
+ this.delegate = (RepeatedMapReaderImpl) delegate;
+ }
+
+ @Override
+ public int size() {
+ throw new UnsupportedOperationException("You can't call size on a single map reader.");
+ }
+
+ @Override
+ public boolean next() {
+ throw new UnsupportedOperationException("You can't call next on a single map reader.");
+ }
+
+ @Override
+ public MajorType getType() {
+ return Types.required(MinorType.MAP);
+ }
+
+
+ @Override
+ public void copyAsValue(MapWriter writer) {
+ delegate.copyAsValueSingle(writer);
+ }
+
+ public void copyAsValueSingle(MapWriter writer){
+ delegate.copyAsValueSingle(writer);
+ }
+
+ @Override
+ public FieldReader reader(String name) {
+ return delegate.reader(name);
+ }
+
+ @Override
+ public void setPosition(int index) {
+ delegate.setPosition(index);
+ }
+
+ @Override
+ public Object readObject() {
+ return delegate.readObject();
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return delegate.iterator();
+ }
+
+ @Override
+ public boolean isSet() {
+ return ! delegate.isNull();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleListReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleListReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleListReaderImpl.java
new file mode 100644
index 0000000..60d8c4e
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleListReaderImpl.java
@@ -0,0 +1,88 @@
+
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector.complex.impl;
+
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.vector.complex.AbstractContainerVector;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+
+@SuppressWarnings("unused")
+public class SingleListReaderImpl extends AbstractFieldReader{
+
+ private static final MajorType TYPE = Types.optional(MinorType.LIST);
+ private final String name;
+ private final AbstractContainerVector container;
+ private FieldReader reader;
+
+ public SingleListReaderImpl(String name, AbstractContainerVector container) {
+ super();
+ this.name = name;
+ this.container = container;
+ }
+
+ @Override
+ public MajorType getType() {
+ return TYPE;
+ }
+
+
+ @Override
+ public void setPosition(int index) {
+ super.setPosition(index);
+ if (reader != null) {
+ reader.setPosition(index);
+ }
+ }
+
+ @Override
+ public Object readObject() {
+ return reader.readObject();
+ }
+
+ @Override
+ public FieldReader reader() {
+ if (reader == null) {
+ reader = container.getChild(name).getReader();
+ setPosition(idx());
+ }
+ return reader;
+ }
+
+ @Override
+ public boolean isSet() {
+ return false;
+ }
+
+ @Override
+ public void copyAsValue(ListWriter writer) {
+ throw new UnsupportedOperationException("Generic list copying not yet supported. Please resolve to typed list.");
+ }
+
+ @Override
+ public void copyAsField(String name, MapWriter writer) {
+ throw new UnsupportedOperationException("Generic list copying not yet supported. Please resolve to typed list.");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
new file mode 100644
index 0000000..1b39775
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleMapReaderImpl.java
@@ -0,0 +1,108 @@
+
+
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector.complex.impl;
+
+
+import java.util.Map;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.MapVector;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+
+import com.google.common.collect.Maps;
+
+@SuppressWarnings("unused")
+public class SingleMapReaderImpl extends AbstractFieldReader{
+
+ private final MapVector vector;
+ private final Map<String, FieldReader> fields = Maps.newHashMap();
+
+ public SingleMapReaderImpl(MapVector vector) {
+ this.vector = vector;
+ }
+
+ private void setChildrenPosition(int index){
+ for(FieldReader r : fields.values()){
+ r.setPosition(index);
+ }
+ }
+
+ @Override
+ public FieldReader reader(String name){
+ FieldReader reader = fields.get(name);
+ if(reader == null){
+ ValueVector child = vector.getChild(name);
+ if(child == null){
+ reader = NullReader.INSTANCE;
+ }else{
+ reader = child.getReader();
+ }
+ fields.put(name, reader);
+ reader.setPosition(idx());
+ }
+ return reader;
+ }
+
+ @Override
+ public void setPosition(int index){
+ super.setPosition(index);
+ for(FieldReader r : fields.values()){
+ r.setPosition(index);
+ }
+ }
+
+ @Override
+ public Object readObject() {
+ return vector.getAccessor().getObject(idx());
+ }
+
+ @Override
+ public boolean isSet() {
+ return true;
+ }
+
+ @Override
+ public MajorType getType(){
+ return vector.getField().getType();
+ }
+
+ @Override
+ public java.util.Iterator<String> iterator(){
+ return vector.fieldNameIterator();
+ }
+
+ @Override
+ public void copyAsValue(MapWriter writer){
+ SingleMapWriter impl = (SingleMapWriter) writer;
+ impl.container.copyFromSafe(idx(), impl.idx(), vector);
+ }
+
+ @Override
+ public void copyAsField(String name, MapWriter writer){
+ SingleMapWriter impl = (SingleMapWriter) writer.map(name);
+ impl.container.copyFromSafe(idx(), impl.idx(), vector);
+ }
+
+
+}
+
[28/45] drill git commit: DRILL-3987: (MOVE) Extract RPC,
memory-base and memory-impl as separate modules.
Posted by ja...@apache.org.
DRILL-3987: (MOVE) Extract RPC, memory-base and memory-impl as separate modules.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1fde9bb1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1fde9bb1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1fde9bb1
Branch: refs/heads/master
Commit: 1fde9bb1505f04e0b0a1afb542a1aa5dfd20ed1b
Parents: a8297df
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Nov 7 19:28:54 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 17:00:47 2015 -0800
----------------------------------------------------------------------
common/pom.xml | 20 +-
.../exceptions/LogicalPlanParsingException.java | 27 -
.../apache/drill/exec/metrics/DrillMetrics.java | 115 +++
.../apache/drill/exec/util/AssertionUtil.java | 39 +
exec/java-exec/pom.xml | 26 +-
.../org/apache/drill/exec/ExecConstants.java | 12 +-
.../apache/drill/exec/client/DrillClient.java | 2 +-
.../exec/client/PrintingResultsListener.java | 2 +-
.../apache/drill/exec/memory/AccountorImpl.java | 434 -----------
.../drill/exec/memory/AtomicRemainder.java | 215 ------
.../drill/exec/memory/RootAllocatorFactory.java | 67 --
.../drill/exec/memory/TopLevelAllocator.java | 389 ----------
.../apache/drill/exec/metrics/DrillMetrics.java | 111 ---
.../exec/metrics/SingleThreadNestedCounter.java | 54 --
.../exec/rpc/AbstractHandshakeHandler.java | 69 --
.../java/org/apache/drill/exec/rpc/Acks.java | 27 -
.../drill/exec/rpc/BaseRpcOutcomeListener.java | 41 -
.../org/apache/drill/exec/rpc/BasicClient.java | 316 --------
.../exec/rpc/BasicClientWithConnection.java | 77 --
.../org/apache/drill/exec/rpc/BasicServer.java | 243 ------
.../drill/exec/rpc/ChannelClosedException.java | 39 -
.../rpc/ChannelListenerWithCoordinationId.java | 25 -
.../drill/exec/rpc/CoordinationQueue.java | 160 ----
.../apache/drill/exec/rpc/DrillRpcFuture.java | 28 -
.../drill/exec/rpc/DrillRpcFutureImpl.java | 80 --
.../apache/drill/exec/rpc/FutureBitCommand.java | 79 --
.../drill/exec/rpc/InboundRpcMessage.java | 66 --
.../apache/drill/exec/rpc/ListeningCommand.java | 70 --
.../drill/exec/rpc/NamedThreadFactory.java | 80 --
.../drill/exec/rpc/OutOfMemoryHandler.java | 31 -
.../drill/exec/rpc/OutboundRpcMessage.java | 98 ---
.../drill/exec/rpc/PositiveAtomicInteger.java | 39 -
.../drill/exec/rpc/ProtobufLengthDecoder.java | 120 ---
.../drill/exec/rpc/ReconnectingConnection.java | 269 -------
.../apache/drill/exec/rpc/RemoteConnection.java | 151 ----
.../drill/exec/rpc/ResettableBarrier.java | 88 ---
.../org/apache/drill/exec/rpc/Response.java | 42 --
.../apache/drill/exec/rpc/ResponseSender.java | 24 -
.../java/org/apache/drill/exec/rpc/RpcBus.java | 478 ------------
.../apache/drill/exec/rpc/RpcCheckedFuture.java | 47 --
.../org/apache/drill/exec/rpc/RpcCommand.java | 26 -
.../org/apache/drill/exec/rpc/RpcConfig.java | 201 -----
.../drill/exec/rpc/RpcConnectionHandler.java | 28 -
.../org/apache/drill/exec/rpc/RpcConstants.java | 27 -
.../org/apache/drill/exec/rpc/RpcDecoder.java | 165 ----
.../org/apache/drill/exec/rpc/RpcEncoder.java | 158 ----
.../org/apache/drill/exec/rpc/RpcException.java | 77 --
.../drill/exec/rpc/RpcExceptionHandler.java | 53 --
.../org/apache/drill/exec/rpc/RpcMessage.java | 38 -
.../org/apache/drill/exec/rpc/RpcOutcome.java | 28 -
.../drill/exec/rpc/RpcOutcomeListener.java | 39 -
.../apache/drill/exec/rpc/TransportCheck.java | 77 --
.../apache/drill/exec/rpc/UserRpcException.java | 49 --
.../rpc/user/AwaitableUserResultsListener.java | 1 +
.../drill/exec/rpc/user/ConnectionThrottle.java | 22 -
.../drill/exec/rpc/user/QueryResultHandler.java | 1 +
.../apache/drill/exec/rpc/user/UserClient.java | 1 +
.../exec/rpc/user/UserResultsListener.java | 1 +
.../drill/exec/server/rest/QueryWrapper.java | 3 +-
.../apache/drill/exec/work/foreman/Foreman.java | 3 +-
.../java/org/apache/drill/BaseTestQuery.java | 2 +-
.../org/apache/drill/SingleRowListener.java | 2 +-
.../drill/exec/memory/TestAllocators.java | 14 +-
.../apache/drill/exec/memory/TestEndianess.java | 43 --
.../exec/server/TestDrillbitResilience.java | 2 +-
.../store/parquet/ParquetResultListener.java | 2 +-
.../store/parquet/TestParquetPhysicalPlan.java | 2 +-
exec/memory/base/pom.xml | 63 ++
.../src/main/java/io/netty/buffer/DrillBuf.java | 749 +++++++++++++++++++
.../java/io/netty/buffer/FakeAllocator.java | 187 +++++
.../main/java/io/netty/buffer/LargeBuffer.java | 350 +++++++++
.../netty/buffer/PooledByteBufAllocatorL.java | 257 +++++++
.../netty/buffer/UnsafeDirectLittleEndian.java | 247 ++++++
.../exec/exception/OutOfMemoryException.java | 51 ++
.../org/apache/drill/exec/memory/Accountor.java | 43 ++
.../drill/exec/memory/BufferAllocator.java | 140 ++++
.../apache/drill/exec/memory/LimitConsumer.java | 28 +
.../apache/drill/exec/ops/BufferManager.java | 66 ++
.../org/apache/drill/exec/util/Pointer.java | 28 +
exec/memory/impl/pom.xml | 68 ++
.../apache/drill/exec/memory/AccountorImpl.java | 437 +++++++++++
.../drill/exec/memory/AtomicRemainder.java | 215 ++++++
.../drill/exec/memory/RootAllocatorFactory.java | 67 ++
.../drill/exec/memory/TopLevelAllocator.java | 388 ++++++++++
.../apache/drill/exec/memory/TestEndianess.java | 43 ++
exec/memory/pom.xml | 35 +
exec/pom.xml | 2 +
exec/rpc/pom.xml | 70 ++
.../exec/rpc/AbstractHandshakeHandler.java | 69 ++
.../java/org/apache/drill/exec/rpc/Acks.java | 27 +
.../drill/exec/rpc/BaseRpcOutcomeListener.java | 41 +
.../org/apache/drill/exec/rpc/BasicClient.java | 316 ++++++++
.../exec/rpc/BasicClientWithConnection.java | 76 ++
.../org/apache/drill/exec/rpc/BasicServer.java | 237 ++++++
.../drill/exec/rpc/ChannelClosedException.java | 39 +
.../rpc/ChannelListenerWithCoordinationId.java | 25 +
.../drill/exec/rpc/ConnectionThrottle.java | 22 +
.../drill/exec/rpc/CoordinationQueue.java | 160 ++++
.../apache/drill/exec/rpc/DrillRpcFuture.java | 28 +
.../drill/exec/rpc/DrillRpcFutureImpl.java | 80 ++
.../apache/drill/exec/rpc/FutureBitCommand.java | 79 ++
.../drill/exec/rpc/InboundRpcMessage.java | 66 ++
.../apache/drill/exec/rpc/ListeningCommand.java | 70 ++
.../drill/exec/rpc/NamedThreadFactory.java | 80 ++
.../drill/exec/rpc/OutOfMemoryHandler.java | 31 +
.../drill/exec/rpc/OutboundRpcMessage.java | 98 +++
.../drill/exec/rpc/PositiveAtomicInteger.java | 39 +
.../drill/exec/rpc/ProtobufLengthDecoder.java | 120 +++
.../drill/exec/rpc/ReconnectingConnection.java | 269 +++++++
.../apache/drill/exec/rpc/RemoteConnection.java | 150 ++++
.../drill/exec/rpc/ResettableBarrier.java | 88 +++
.../org/apache/drill/exec/rpc/Response.java | 42 ++
.../apache/drill/exec/rpc/ResponseSender.java | 24 +
.../java/org/apache/drill/exec/rpc/RpcBus.java | 478 ++++++++++++
.../apache/drill/exec/rpc/RpcCheckedFuture.java | 47 ++
.../org/apache/drill/exec/rpc/RpcCommand.java | 26 +
.../org/apache/drill/exec/rpc/RpcConfig.java | 201 +++++
.../drill/exec/rpc/RpcConnectionHandler.java | 28 +
.../org/apache/drill/exec/rpc/RpcConstants.java | 27 +
.../org/apache/drill/exec/rpc/RpcDecoder.java | 165 ++++
.../org/apache/drill/exec/rpc/RpcEncoder.java | 158 ++++
.../org/apache/drill/exec/rpc/RpcException.java | 77 ++
.../drill/exec/rpc/RpcExceptionHandler.java | 52 ++
.../org/apache/drill/exec/rpc/RpcMessage.java | 38 +
.../org/apache/drill/exec/rpc/RpcOutcome.java | 28 +
.../drill/exec/rpc/RpcOutcomeListener.java | 39 +
.../apache/drill/exec/rpc/TransportCheck.java | 78 ++
.../apache/drill/exec/rpc/UserRpcException.java | 49 ++
exec/vector/pom.xml | 10 +-
.../src/main/java/io/netty/buffer/DrillBuf.java | 749 -------------------
.../java/io/netty/buffer/FakeAllocator.java | 187 -----
.../main/java/io/netty/buffer/LargeBuffer.java | 350 ---------
.../netty/buffer/PooledByteBufAllocatorL.java | 257 -------
.../netty/buffer/UnsafeDirectLittleEndian.java | 247 ------
.../exec/exception/OutOfMemoryException.java | 51 --
.../org/apache/drill/exec/memory/Accountor.java | 43 --
.../drill/exec/memory/BufferAllocator.java | 140 ----
.../apache/drill/exec/memory/LimitConsumer.java | 28 -
.../apache/drill/exec/ops/BufferManager.java | 66 --
.../apache/drill/exec/util/AssertionUtil.java | 39 -
.../org/apache/drill/exec/util/Pointer.java | 28 -
.../vector/complex/AbstractContainerVector.java | 15 +-
142 files changed, 7510 insertions(+), 7366 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index fa19e21..db9c9a8 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -83,17 +83,17 @@
</dependency>
<dependency>
- <groupId>org.hibernate</groupId>
- <artifactId>hibernate-validator</artifactId>
- <version>4.3.1.Final</version>
- <exclusions>
- <exclusion>
- <groupId>org.jboss.logging</groupId>
- <artifactId>jboss-logging</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.1</version>
</dependency>
<dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-servlets</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
+ <dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr-runtime</artifactId>
<version>3.4</version>
@@ -102,7 +102,7 @@
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
- <version>2.3</version>
+ <version>2.9</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java b/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
index 1f437b2..c8c0afa 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/LogicalPlanParsingException.java
@@ -16,29 +16,17 @@
* limitations under the License.
*/
package org.apache.drill.common.exceptions;
-import java.util.Set;
-
-import javax.validation.ConstraintViolation;
-
import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.common.logical.data.LogicalOperatorBase;
public class LogicalPlanParsingException extends DrillRuntimeException{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalPlanParsingException.class);
- private Set<ConstraintViolation<LogicalOperatorBase>> violations;
private LogicalOperator operator;
public LogicalPlanParsingException() {
super();
}
- public LogicalPlanParsingException(LogicalOperator operator, Set<ConstraintViolation<LogicalOperatorBase>> violations){
- super(getErrorMessage(operator, violations));
- this.operator = operator;
- this.violations = violations;
- }
-
public LogicalPlanParsingException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
@@ -57,19 +45,4 @@ public class LogicalPlanParsingException extends DrillRuntimeException{
}
- private static String getErrorMessage(LogicalOperator operator, Set<ConstraintViolation<LogicalOperatorBase>> violations){
- StringBuffer sb = new StringBuffer();
- sb.append("Failure while attempting to set up operator of type ");
- sb.append(operator.getClass().getCanonicalName());
- sb.append(". Constraints violated:\n");
- for(ConstraintViolation<LogicalOperatorBase> v : violations){
- sb.append("\t");
- sb.append(v.getMessage());
- sb.append(", Invalid value: ");
- sb.append(v.getInvalidValue());
- sb.append("\n");
- }
- return sb.toString();
- }
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java b/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
new file mode 100644
index 0000000..6fd3dd4
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.metrics;
+
+import java.lang.management.ManagementFactory;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.common.config.DrillConfig;
+
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
+import com.codahale.metrics.Slf4jReporter;
+import com.codahale.metrics.jvm.BufferPoolMetricSet;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+
+public class DrillMetrics {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMetrics.class);
+
+ public static final String METRICS_JMX_OUTPUT_ENABLED = "drill.exec.metrics.jmx.enabled";
+ public static final String METRICS_LOG_OUTPUT_ENABLED = "drill.exec.metrics.log.enabled";
+ public static final String METRICS_LOG_OUTPUT_INTERVAL = "drill.exec.metrics.log.interval";
+
+ static final DrillConfig config = DrillConfig.create();
+
+ private DrillMetrics() {
+ }
+
+ private static class RegistryHolder {
+ public static final MetricRegistry REGISTRY;
+ // private static final JmxReporter JMX_REPORTER;
+ private static final Slf4jReporter LOG_REPORTER;
+
+ static {
+ REGISTRY = new MetricRegistry();
+ registerSysStats();
+ // JMX_REPORTER = getJmxReporter();
+ LOG_REPORTER = getLogReporter();
+ }
+
+ private static void registerSysStats(){
+ REGISTRY.registerAll(new GarbageCollectorMetricSet());
+ REGISTRY.registerAll(new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()));
+ REGISTRY.registerAll(new MemoryUsageGaugeSet());
+ REGISTRY.registerAll(new ThreadStatesGaugeSet());
+ }
+
+ private static JmxReporter getJmxReporter() {
+ if (config.getBoolean(METRICS_JMX_OUTPUT_ENABLED)) {
+ JmxReporter reporter = JmxReporter.forRegistry(getInstance()).build();
+ reporter.start();
+
+ return reporter;
+ } else {
+ return null;
+ }
+ }
+
+ private static Slf4jReporter getLogReporter() {
+ if (config.getBoolean(METRICS_LOG_OUTPUT_ENABLED)) {
+ Slf4jReporter reporter = Slf4jReporter.forRegistry(getInstance()).outputTo(logger)
+ .convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build();
+ reporter.start(config.getInt(METRICS_LOG_OUTPUT_INTERVAL), TimeUnit.SECONDS);
+
+ return reporter;
+ } else {
+ return null;
+ }
+ }
+ }
+
+ private static void registerAll(String prefix, MetricSet metricSet, MetricRegistry registry) {
+ for (Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
+ if (entry.getValue() instanceof MetricSet) {
+ registerAll(prefix + "." + entry.getKey(), (MetricSet) entry.getValue(), registry);
+ } else {
+ registry.register(prefix + "." + entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ public static MetricRegistry getInstance() {
+ return RegistryHolder.REGISTRY;
+ }
+
+ public static void resetMetrics(){
+ RegistryHolder.REGISTRY.removeMatching(new MetricFilter(){
+ @Override
+ public boolean matches(String name, Metric metric) {
+ return true;
+ }});
+ RegistryHolder.registerSysStats();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java b/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
new file mode 100644
index 0000000..2348e62
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+public class AssertionUtil {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
+
+
+ public static final boolean ASSERT_ENABLED;
+ public static final boolean BOUNDS_CHECKING_ENABLED;
+
+ static{
+ boolean isAssertEnabled = false;
+ assert isAssertEnabled = true;
+ ASSERT_ENABLED = isAssertEnabled;
+ BOUNDS_CHECKING_ENABLED = ASSERT_ENABLED || !"true".equals(System.getProperty("drill.enable_unsafe_memory_access"));
+ }
+
+ public static boolean isAssertionsEnabled(){
+ return ASSERT_ENABLED;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index a0398b7..7299b3e 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -119,12 +119,6 @@
<version>2.4.3</version>
</dependency>
<dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-epoll</artifactId>
- <classifier>linux-x86_64</classifier>
- <version>4.0.27.Final</version>
- </dependency>
- <dependency>
<groupId>org.glassfish.jersey.ext</groupId>
<artifactId>jersey-mvc-freemarker</artifactId>
<version>2.8</version>
@@ -264,16 +258,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>3.0.1</version>
- </dependency>
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-servlets</artifactId>
- <version>3.0.1</version>
- </dependency>
- <dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
<version>1</version>
@@ -290,6 +274,16 @@
</dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
+ <artifactId>drill-rpc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.memory</groupId>
+ <artifactId>drill-memory-impl</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.exec</groupId>
<artifactId>vector</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index f85a776..4f278f0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -26,8 +26,8 @@ import org.apache.drill.exec.server.options.TypeValidators.EnumeratedStringValid
import org.apache.drill.exec.server.options.TypeValidators.LongValidator;
import org.apache.drill.exec.server.options.TypeValidators.PositiveLongValidator;
import org.apache.drill.exec.server.options.TypeValidators.PowerOfTwoLongValidator;
-import org.apache.drill.exec.server.options.TypeValidators.RangeLongValidator;
import org.apache.drill.exec.server.options.TypeValidators.RangeDoubleValidator;
+import org.apache.drill.exec.server.options.TypeValidators.RangeLongValidator;
import org.apache.drill.exec.server.options.TypeValidators.StringValidator;
import org.apache.drill.exec.testing.ExecutionControls;
import org.apache.drill.exec.util.ImpersonationUtil;
@@ -49,9 +49,6 @@ public interface ExecConstants {
public static final String USER_RPC_TIMEOUT = "drill.exec.rpc.user.timeout";
public static final String METRICS_CONTEXT_NAME = "drill.exec.metrics.context";
public static final String USE_IP_ADDRESS = "drill.exec.rpc.use.ip";
- public static final String METRICS_JMX_OUTPUT_ENABLED = "drill.exec.metrics.jmx.enabled";
- public static final String METRICS_LOG_OUTPUT_ENABLED = "drill.exec.metrics.log.enabled";
- public static final String METRICS_LOG_OUTPUT_INTERVAL = "drill.exec.metrics.log.interval";
public static final String CLIENT_RPC_THREADS = "drill.exec.rpc.user.client.threads";
public static final String BIT_SERVER_RPC_THREADS = "drill.exec.rpc.bit.server.threads";
public static final String USER_SERVER_RPC_THREADS = "drill.exec.rpc.user.server.threads";
@@ -75,7 +72,6 @@ public interface ExecConstants {
public static final String TEXT_LINE_READER_BATCH_SIZE = "drill.exec.storage.file.text.batch.size";
public static final String TEXT_LINE_READER_BUFFER_SIZE = "drill.exec.storage.file.text.buffer.size";
public static final String HAZELCAST_SUBNETS = "drill.exec.cache.hazel.subnets";
- public static final String TOP_LEVEL_MAX_ALLOC = "drill.exec.memory.top.max";
public static final String HTTP_ENABLE = "drill.exec.http.enabled";
public static final String HTTP_PORT = "drill.exec.http.port";
public static final String HTTP_ENABLE_SSL = "drill.exec.http.ssl_enabled";
@@ -91,7 +87,6 @@ public interface ExecConstants {
public static final String USER_AUTHENTICATION_ENABLED = "drill.exec.security.user.auth.enabled";
public static final String USER_AUTHENTICATOR_IMPL = "drill.exec.security.user.auth.impl";
public static final String PAM_AUTHENTICATOR_PROFILES = "drill.exec.security.user.auth.pam_profiles";
- public static final String ERROR_ON_MEMORY_LEAK = "drill.exec.debug.error_on_leak";
/** Size of JDBC batch queue (in batches) above which throttling begins. */
public static final String JDBC_BATCH_QUEUE_THROTTLING_THRESHOLD =
"drill.jdbc.batch_queue_throttling_threshold";
@@ -104,11 +99,8 @@ public interface ExecConstants {
public static final String RETURN_ERROR_FOR_FAILURE_IN_CANCELLED_FRAGMENTS =
"drill.exec.debug.return_error_for_failure_in_cancelled_fragments";
- /** Fragment memory planning */
- public static final String ENABLE_FRAGMENT_MEMORY_LIMIT = "drill.exec.memory.enable_frag_limit";
- public static final String FRAGMENT_MEM_OVERCOMMIT_FACTOR = "drill.exec.memory.frag_mem_overcommit_factor";
- public static final String USE_LINUX_EPOLL = "drill.exec.enable-epoll";
+
public static final String CLIENT_SUPPORT_COMPLEX_TYPES = "drill.client.supports-complex-types";
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index e476fff..902b35b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -57,12 +57,12 @@ import org.apache.drill.exec.proto.UserProtos.UserProperties;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.rpc.BasicClientWithConnection.ServerConnection;
import org.apache.drill.exec.rpc.ChannelClosedException;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.DrillRpcFuture;
import org.apache.drill.exec.rpc.NamedThreadFactory;
import org.apache.drill.exec.rpc.RpcConnectionHandler;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.TransportCheck;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserClient;
import org.apache.drill.exec.rpc.user.UserResultsListener;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
index d8932ee..6bc8593 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
@@ -31,7 +31,7 @@ import org.apache.drill.exec.proto.UserBitShared.QueryData;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.util.VectorUtil;
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
deleted file mode 100644
index fd98f59..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
+++ /dev/null
@@ -1,434 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.DrillBuf;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.util.AssertionUtil;
-
-import com.google.common.collect.LinkedListMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-
-public class AccountorImpl implements Accountor {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AccountorImpl.class);
-
- private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
- private final AtomicRemainder remainder;
- private final long total;
- private ConcurrentMap<ByteBuf, DebugStackTrace> buffers = Maps.newConcurrentMap();
- private AccountorImpl parent;
-
- private final boolean errorOnLeak;
- // some operators are no subject to the fragment limit. They set the applyFragmentLimit to false
-
- private final boolean enableFragmentLimit;
- private final double fragmentMemOvercommitFactor;
-
- private final boolean DEFAULT_ENABLE_FRAGMENT_LIMIT=false;
- private final double DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR=1.5;
-
- private final boolean applyFragmentLimit;
-
- private final LimitConsumer limitConsumer;
- long fragmentLimit;
-
- private long peakMemoryAllocation = 0;
-
- // The top level Allocator has an accountor that keeps track of all the LimitConsumers currently executing.
- // This enables the top level accountor to calculate a new fragment limit whenever necessary.
- private final List<LimitConsumer> limitConsumers;
-
- public AccountorImpl(DrillConfig config, boolean errorOnLeak, LimitConsumer context, AccountorImpl parent, long max,
- long preAllocated, boolean applyFragLimit) {
- // TODO: fix preallocation stuff
- this.errorOnLeak = errorOnLeak;
- AtomicRemainder parentRemainder = parent != null ? parent.remainder : null;
- this.parent = parent;
-
- boolean enableFragmentLimit;
- double fragmentMemOvercommitFactor;
-
- try {
- enableFragmentLimit = config.getBoolean(ExecConstants.ENABLE_FRAGMENT_MEMORY_LIMIT);
- fragmentMemOvercommitFactor = config.getDouble(ExecConstants.FRAGMENT_MEM_OVERCOMMIT_FACTOR);
- }catch(Exception e){
- enableFragmentLimit = DEFAULT_ENABLE_FRAGMENT_LIMIT;
- fragmentMemOvercommitFactor = DEFAULT_FRAGMENT_MEM_OVERCOMMIT_FACTOR;
- }
- this.enableFragmentLimit = enableFragmentLimit;
- this.fragmentMemOvercommitFactor = fragmentMemOvercommitFactor;
-
-
- this.applyFragmentLimit=applyFragLimit;
-
- this.remainder = new AtomicRemainder(errorOnLeak, parentRemainder, max, preAllocated, applyFragmentLimit);
- this.total = max;
- this.limitConsumer = context;
- this.fragmentLimit=this.total; // Allow as much as possible to start with;
- if (ENABLE_ACCOUNTING) {
- buffers = Maps.newConcurrentMap();
- } else {
- buffers = null;
- }
- this.limitConsumers = new ArrayList<LimitConsumer>();
- if(parent!=null && parent.parent==null){ // Only add the fragment context to the fragment level accountor
- synchronized(this) {
- addLimitConsumer(this.limitConsumer);
- }
- }
- }
-
- public boolean transferTo(Accountor target, DrillBuf buf, long size) {
- return transfer(target, buf, size, true);
- }
-
- public boolean transferIn(DrillBuf buf, long size) {
- return transfer(this, buf, size, false);
- }
-
- private boolean transfer(Accountor target, DrillBuf buf, long size, boolean release) {
- boolean withinLimit = target.forceAdditionalReservation(size);
- if(release){
- release(buf, size);
- }
-
- if (ENABLE_ACCOUNTING) {
- if (target instanceof AccountorImpl) {
- ((AccountorImpl) target).buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread()
- .getStackTrace()));
- }
- }
- return withinLimit;
- }
-
- public long getAvailable() {
- if (parent != null) {
- return Math.min(parent.getAvailable(), getCapacity() - getAllocation());
- }
- return getCapacity() - getAllocation();
- }
-
- public long getCapacity() {
- return fragmentLimit;
- }
-
- public long getAllocation() {
- return remainder.getUsed();
- }
-
- public long getPeakMemoryAllocation() {
- return peakMemoryAllocation;
- }
-
- public boolean reserve(long size) {
- boolean status = remainder.get(size, this.applyFragmentLimit);
- peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
- return status;
- }
-
- public boolean forceAdditionalReservation(long size) {
- if (size > 0) {
- boolean status = remainder.forceGet(size);
- peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
- return status;
- } else {
- return true;
- }
- }
-
- public void reserved(long expected, DrillBuf buf) {
- // make sure to take away the additional memory that happened due to rounding.
-
- long additional = buf.capacity() - expected;
- if (additional > 0) {
- remainder.forceGet(additional);
- }
-
- if (ENABLE_ACCOUNTING) {
- buffers.put(buf, new DebugStackTrace(buf.capacity(), Thread.currentThread().getStackTrace()));
- }
-
- peakMemoryAllocation = Math.max(peakMemoryAllocation, getAllocation());
- }
-
-
- public void releasePartial(DrillBuf buf, long size) {
- remainder.returnAllocation(size);
- if (ENABLE_ACCOUNTING) {
- if (buf != null) {
- DebugStackTrace dst = buffers.get(buf);
- if (dst == null) {
- throw new IllegalStateException("Partially releasing a buffer that has already been released. Buffer: " + buf);
- }
- dst.size -= size;
- if (dst.size < 0) {
- throw new IllegalStateException("Partially releasing a buffer that has already been released. Buffer: " + buf);
- }
- }
- }
- }
-
- void release(long size) {
- remainder.returnAllocation(size);
- }
-
- public void release(DrillBuf buf, long size) {
- remainder.returnAllocation(size);
- if (ENABLE_ACCOUNTING) {
- if (buf != null && buffers.remove(buf) == null) {
- throw new IllegalStateException("Releasing a buffer that has already been released. Buffer: " + buf);
- }
- }
- }
-
- private void addLimitConsumer(LimitConsumer c) {
- if (c == null) {
- return;
- }
-
- if (parent != null){
- parent.addLimitConsumer(c);
- }else {
- if(logger.isTraceEnabled()) {
- String fragStr = c == null ? "[Null Context]" : c.getIdentifier();
- fragStr+=" (Object Id: "+System.identityHashCode(c)+")";
- StackTraceElement[] ste = (new Throwable()).getStackTrace();
- StringBuffer sb = new StringBuffer();
- for (StackTraceElement s : ste) {
- sb.append(s.toString());
- sb.append("\n");
- }
-
- logger.trace("Fragment " + fragStr + " added to root accountor.\n"+sb.toString());
- }
- synchronized(this) {
- limitConsumers.add(c);
- }
- }
- }
-
- private void removeLimitConsumer(LimitConsumer c) {
- if (c == null) {
- return;
- }
-
- if (parent != null){
- if (parent.parent==null){
- // only fragment level allocators will have the fragment context saved
- parent.removeLimitConsumer(c);
- }
- }else{
- if(logger.isDebugEnabled()) {
- String fragStr = c == null ? "[Null Context]" : c.getIdentifier();
- fragStr += " (Object Id: " + System.identityHashCode(c) + ")";
- logger.trace("Fragment " + fragStr + " removed from root accountor");
- }
- synchronized(this) {
- limitConsumers.remove(c);
- }
- }
- }
-
- public long resetFragmentLimits(){
- // returns the new capacity
- if(!this.enableFragmentLimit){
- return getCapacity();
- }
-
- if(parent!=null){
- parent.resetFragmentLimits();
- }else {
- //Get remaining memory available per fragment and distribute it EQUALLY among all the fragments.
- //Fragments get the memory limit added to the amount already allocated.
- //This favours fragments that are already running which will get a limit greater than newly started fragments.
- //If the already running fragments end quickly, their limits will be assigned back to the remaining fragments
- //quickly. If they are long running, then we want to favour them with larger limits anyway.
- synchronized (this) {
- int nFragments = limitConsumers.size();
- long allocatedMemory=0;
- for (LimitConsumer fragment : limitConsumers) {
- allocatedMemory += fragment.getAllocated();
- }
- if(logger.isTraceEnabled()) {
- logger.trace("Resetting Fragment Memory Limit: total Available memory== "+total
- +" Total Allocated Memory :"+allocatedMemory
- +" Number of fragments: "+nFragments
- + " fragmentMemOvercommitFactor: "+fragmentMemOvercommitFactor
- + " Root fragment limit: "+this.fragmentLimit + "(Root obj: "+System.identityHashCode(this)+")"
- );
- }
- if(nFragments>0) {
- long rem = (total - allocatedMemory) / nFragments;
- for (LimitConsumer fragment : limitConsumers) {
- fragment.setLimit((long) (rem * fragmentMemOvercommitFactor));
- }
- }
- if(logger.isTraceEnabled() && false){
- StringBuffer sb= new StringBuffer();
- sb.append("[root](0:0)");
- sb.append("Allocated memory: ");
- sb.append(this.getAllocation());
- sb.append(" Fragment Limit: ");
- sb.append(this.getFragmentLimit());
- logger.trace(sb.toString());
- for (LimitConsumer fragment : limitConsumers) {
- sb= new StringBuffer();
- sb.append('[');
- sb.append(fragment.getIdentifier());
- sb.append(']');
- sb.append("Allocated memory: ");
- sb.append(fragment.getAllocated());
- sb.append(" Fragment Limit: ");
- sb.append(fragment.getLimit());
- logger.trace(sb.toString());
- }
- logger.trace("Resetting Complete");
- }
- }
- }
- return getCapacity();
- }
-
- public void close() {
- // remove the fragment context and reset fragment limits whenever an allocator closes
- if (parent != null && parent.parent == null && limitConsumer != null) {
- logger.debug("Fragment " + limitConsumer.getIdentifier() + " accountor being closed");
- removeLimitConsumer(limitConsumer);
- }
- resetFragmentLimits();
-
- if (ENABLE_ACCOUNTING && !buffers.isEmpty()) {
- StringBuffer sb = new StringBuffer();
- sb.append("Attempted to close accountor with ");
- sb.append(buffers.size());
- sb.append(" buffer(s) still allocated for ");
- sb.append(limitConsumer.getIdentifier());
- sb.append(".\n");
-
- Multimap<DebugStackTrace, DebugStackTrace> multi = LinkedListMultimap.create();
- for (DebugStackTrace t : buffers.values()) {
- multi.put(t, t);
- }
-
- for (DebugStackTrace entry : multi.keySet()) {
- Collection<DebugStackTrace> allocs = multi.get(entry);
-
- sb.append("\n\n\tTotal ");
- sb.append(allocs.size());
- sb.append(" allocation(s) of byte size(s): ");
- for (DebugStackTrace alloc : allocs) {
- sb.append(alloc.size);
- sb.append(", ");
- }
-
- sb.append("at stack location:\n");
- entry.addToString(sb);
- }
- if (!buffers.isEmpty()) {
- IllegalStateException e = new IllegalStateException(sb.toString());
- if (errorOnLeak) {
- throw e;
- } else {
- logger.warn("Memory leaked.", e);
- }
- }
- }
-
- remainder.close();
-
- }
-
- public void setFragmentLimit(long add) {
- // We ADD the limit to the current allocation. If none has been allocated, this
- // sets a new limit. If memory has already been allocated, the fragment gets its
- // limit based on the allocation, though this might still result in reducing the
- // limit.
-
- if (parent != null && parent.parent==null) { // This is a fragment level accountor
- this.fragmentLimit=getAllocation()+add;
- this.remainder.setLimit(this.fragmentLimit);
- logger.trace("Fragment " + limitConsumer.getIdentifier() + " memory limit set to " + this.fragmentLimit);
- }
- }
-
- public long getFragmentLimit(){
- return this.fragmentLimit;
- }
-
- public class DebugStackTrace {
-
- private StackTraceElement[] elements;
- private long size;
-
- public DebugStackTrace(long size, StackTraceElement[] elements) {
- super();
- this.elements = elements;
- this.size = size;
- }
-
- public void addToString(StringBuffer sb) {
- for (int i = 3; i < elements.length; i++) {
- sb.append("\t\t");
- sb.append(elements[i]);
- sb.append("\n");
- }
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(elements);
-// result = prime * result + (int) (size ^ (size >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- DebugStackTrace other = (DebugStackTrace) obj;
- if (!Arrays.equals(elements, other.elements)) {
- return false;
- }
- // weird equal where size doesn't matter for multimap purposes.
-// if (size != other.size)
-// return false;
- return true;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
deleted file mode 100644
index 0f6b8b0..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- *
- *
- * TODO: Fix this so that preallocation can never be released back to general pool until allocator is closed.
- */
-public class AtomicRemainder {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AtomicRemainder.class);
-
- private final AtomicRemainder parent;
- private final AtomicLong availableShared;
- private final AtomicLong availablePrivate;
- private final long initTotal;
- private final long initShared;
- private final long initPrivate;
- private long limit; // An Allocator can set a variable limit less than or equal to the initTotal
- private boolean hasLimit; // True for Atomic Remainders associated with a Fragment. May be true for Operator Level allocators some day.
- private boolean closed = false;
- private final boolean errorOnLeak;
- private final boolean applyFragmentLimit;
-
- public AtomicRemainder(boolean errorOnLeak, AtomicRemainder parent, long max, long pre, boolean applyFragLimit) {
- this.errorOnLeak = errorOnLeak;
- this.parent = parent;
- this.availableShared = new AtomicLong(max - pre);
- this.availablePrivate = new AtomicLong(pre);
- this.initTotal = max;
- this.initShared = max - pre;
- this.initPrivate = pre;
- this.limit = max;
- this.hasLimit=false;
- this.applyFragmentLimit=applyFragLimit; // If this is an operator that is exempt from the fragment limit, set this to false.
-// logger.info("new AtomicRemainder. a.s. {} a.p. {} hashcode {}", availableShared, availablePrivate, hashCode(), new Exception());
- }
-
- public long getRemainder() {
- return availableShared.get() + availablePrivate.get();
- }
-
- public long getUsed() {
- return initTotal - getRemainder();
- }
-
- /**
- * Allow an allocator to constrain the remainder to a particular limit that is lower than the initTotal.
- * If limit is larger than initTotal, then the function will do nothing and the hasLimit flag will not be set.
- * @param limit new remainder limit
- */
- public void setLimit(long limit) {
- if(limit<initTotal){
- this.hasLimit=true;
- this.limit=limit;
- }
-
- }
- /**
- * Automatically allocate memory. This is used when an actual allocation happened to be larger than requested, or when
- * a buffer has it's ownership passed to another allocator.<br>
- * This memory has already been used up so it must be accurately accounted for in future allocations.
- *
- * @param size extra allocated memory that needs to be accounted for
- */
- public boolean forceGet(long size) {
- if (get(size, this.applyFragmentLimit)) {
- return true;
- } else {
- availableShared.addAndGet(-size);
- if (parent != null) {
- parent.forceGet(size);
- }
- return false;
- }
- }
-
- public boolean get(long size, boolean applyFragmentLimitForChild) {
- if (availablePrivate.get() < 1) {
- // if there is no preallocated memory, we can operate normally.
-
- // if there is a parent allocator, check it before allocating.
- if (parent != null && !parent.get(size, this.applyFragmentLimit)) {
- return false;
- }
-
- // If we need to allocate memory beyond the allowed Fragment Limit
- if(applyFragmentLimitForChild && this.applyFragmentLimit && this.hasLimit && (getUsed()+size > this.limit)){
- if (parent != null) {
- parent.returnAllocation(size);
- }
- StackTraceElement[] ste = (new Throwable()).getStackTrace();
- StringBuilder sb = new StringBuilder();
- for (StackTraceElement s : ste) {
- sb.append(s.toString());
- sb.append("\n");
- }
- logger.warn("No more memory. Fragment limit ({} bytes) reached. Trying to allocate {} bytes. {} bytes " +
- "already allocated.\n{}", limit, size, getUsed(), sb.toString());
- return false;
- }
-
- // attempt to get shared memory, if fails, return false.
- long outcome = availableShared.addAndGet(-size);
-// assert outcome <= initShared;
- if (outcome < 0) {
- availableShared.addAndGet(size);
- if (parent != null) {
- parent.returnAllocation(size);
- }
- return false;
- } else {
-// if (DEBUG)
-// logger.info("Get {}. a.s. {} a.p. {} hashcode {}", size, availableShared, availablePrivate, hashCode(), new Exception());
- return true;
- }
-
- } else {
- // if there is preallocated memory, use that first.
- long unaccount = availablePrivate.addAndGet(-size);
- if (unaccount >= 0) {
-// if (DEBUG)
-// logger.info("Get {}. a.s. {} a.p. {} hashcode {}", size, availableShared, availablePrivate, hashCode(), new Exception());
- return true;
- } else {
-
- long additionalSpaceNeeded = -unaccount;
- // if there is a parent allocator, check it before allocating.
- if (parent != null && !parent.get(additionalSpaceNeeded, this.applyFragmentLimit)) {
- // parent allocation failed, return space to private pool.
- availablePrivate.getAndAdd(size);
- return false;
- }
-
- // we got space from parent pool. lets make sure we have space locally available.
- long account = availableShared.addAndGet(-additionalSpaceNeeded);
- if (account >= 0) {
- // we were succesful, move private back to zero (since we allocated using shared).
- availablePrivate.addAndGet(additionalSpaceNeeded);
-// if (DEBUG)
-// logger.info("Get {}. a.s. {} a.p. {} hashcode {}", size, availableShared, availablePrivate, hashCode(), new Exception());
- return true;
- } else {
- // we failed to get space from available shared. Return allocations to initial state.
- availablePrivate.addAndGet(size);
- availableShared.addAndGet(additionalSpaceNeeded);
- if (parent != null) {
- parent.returnAllocation(additionalSpaceNeeded);
- }
- return false;
- }
- }
-
- }
-
- }
-
- /**
- * Return the memory accounting to the allocation pool. Make sure to first maintain hold of the preallocated memory.
- *
- * @param size amount of memory returned
- */
- public void returnAllocation(long size) {
- long privateSize = availablePrivate.get();
- long privateChange = Math.min(size, initPrivate - privateSize);
- long sharedChange = size - privateChange;
- availablePrivate.addAndGet(privateChange);
- availableShared.addAndGet(sharedChange);
-// if (DEBUG)
-// logger.info("Return allocation {}, a.s. {} a.p. {} hashcode {}", size, availableShared, availablePrivate, hashCode(), new Exception());
- if (parent != null) {
- parent.returnAllocation(sharedChange);
- }
- }
-
- public void close() {
- if (closed) {
- logger.warn("Tried to close remainder, but it has already been closed", new Exception());
- return;
- }
- if (availablePrivate.get() != initPrivate || availableShared.get() != initShared) {
- IllegalStateException e = new IllegalStateException(
- String
- .format(ERROR, initPrivate, availablePrivate.get(), initPrivate - availablePrivate.get(), initShared, availableShared.get(), initShared - availableShared.get()));
- if (errorOnLeak) {
- throw e;
- } else {
- logger.warn("Memory leaked during query.", e);
- }
- }
- if (parent != null) {
- parent.returnAllocation(initPrivate);
- }
- closed = true;
- }
-
- static final String ERROR = "Failure while closing accountor. Expected private and shared pools to be set to initial values. However, one or more were not. Stats are\n\tzone\tinit\tallocated\tdelta \n\tprivate\t%d\t%d\t%d \n\tshared\t%d\t%d\t%d.";
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/memory/RootAllocatorFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/RootAllocatorFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/RootAllocatorFactory.java
deleted file mode 100644
index a413e4a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/RootAllocatorFactory.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import org.apache.drill.common.config.DrillConfig;
-
-import com.google.common.annotations.VisibleForTesting;
-
-public class RootAllocatorFactory {
- /**
- * Constructor to prevent instantiation of this static utility class.
- */
- private RootAllocatorFactory() {}
-
- /**
- * Factory method.
- *
- * @param drillConfig the DrillConfig
- * @return a new root allocator
- */
- public static BufferAllocator newRoot(final DrillConfig drillConfig) {
-/* TODO(DRILL-1942)
- if (BaseAllocator.DEBUG) {
- return new TopLevelAllocator(drillConfig);
- }
-*/
-
- return new TopLevelAllocator(drillConfig);
- }
-
- /**
- * Factory method for testing, where a DrillConfig may not be available.
- *
- * @param allocationPolicy the allocation policy
- * @param initAllocation initial allocation from parent
- * @param maxReservation maximum amount of memory this can hand out
- * @param flags one of BufferAllocator's F_* flags
- * @return a new root allocator
- */
-/* TODO(DRILL-1942)
- @VisibleForTesting
- public static BufferAllocator newRoot(
- final AllocationPolicy allocationPolicy,
- final long initAllocation, final long maxReservation, final int flags) {
- if (BaseAllocator.DEBUG) {
- return new RootAllocatorDebug(allocationPolicy, initAllocation, maxReservation, flags);
- }
-
- return new RootAllocator(allocationPolicy, initAllocation, maxReservation, flags);
- }
-*/
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
deleted file mode 100644
index ba04fa3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ /dev/null
@@ -1,389 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.memory;
-
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.DrillBuf;
-import io.netty.buffer.PooledByteBufAllocatorL;
-import io.netty.buffer.UnsafeDirectLittleEndian;
-
-import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.exception.OutOfMemoryException;
-import org.apache.drill.exec.metrics.DrillMetrics;
-import org.apache.drill.exec.testing.ControlsInjector;
-import org.apache.drill.exec.testing.ControlsInjectorFactory;
-import org.apache.drill.exec.util.AssertionUtil;
-import org.apache.drill.exec.util.Pointer;
-
-public class TopLevelAllocator implements BufferAllocator {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopLevelAllocator.class);
- private static final ControlsInjector injector = ControlsInjectorFactory.getInjector(TopLevelAllocator.class);
- public static final String CHILD_BUFFER_INJECTION_SITE = "child.buffer";
-
- public static long MAXIMUM_DIRECT_MEMORY;
-
- private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
- private final Map<ChildAllocator, StackTraceElement[]> childrenMap;
- private final PooledByteBufAllocatorL innerAllocator;
- private final AccountorImpl acct;
- private final boolean errorOnLeak;
- private final DrillBuf empty;
- private final DrillConfig config;
-
- private TopLevelAllocator(DrillConfig config, long maximumAllocation, boolean errorOnLeak){
- MAXIMUM_DIRECT_MEMORY = maximumAllocation;
- innerAllocator = new PooledByteBufAllocatorL(DrillMetrics.getInstance());
- this.config=(config!=null) ? config : DrillConfig.create();
- this.errorOnLeak = errorOnLeak;
- this.acct = new AccountorImpl(config, errorOnLeak, null, null, maximumAllocation, 0, true);
- this.empty = DrillBuf.getEmpty(this, acct);
- this.childrenMap = ENABLE_ACCOUNTING ? new IdentityHashMap<ChildAllocator, StackTraceElement[]>() : null;
- }
-
- TopLevelAllocator(DrillConfig config) {
- this(config, Math.min(DrillConfig.getMaxDirectMemory(), config.getLong(ExecConstants.TOP_LEVEL_MAX_ALLOC)),
- config.getBoolean(ExecConstants.ERROR_ON_MEMORY_LEAK)
- );
- }
-
- @Override
- public boolean takeOwnership(DrillBuf buf) {
- return buf.transferAccounting(acct);
- }
-
- @Override
- public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> out) {
- DrillBuf b = new DrillBuf(this, acct, buf);
- out.value = b;
- return acct.transferIn(b, b.capacity());
- }
-
- public DrillBuf buffer(int min, int max) {
- if (min == 0) {
- return empty;
- }
- if(!acct.reserve(min)) {
- throw new OutOfMemoryException(createErrorMsg(this, min));
- }
-
- try {
- UnsafeDirectLittleEndian buffer = innerAllocator.directBuffer(min, max);
- DrillBuf wrapped = new DrillBuf(this, acct, buffer);
- acct.reserved(min, wrapped);
- return wrapped;
- } catch (OutOfMemoryError e) {
- if ("Direct buffer memory".equals(e.getMessage())) {
- acct.release(min);
- throw new OutOfMemoryException(createErrorMsg(this, min), e);
- } else {
- throw e;
- }
- }
- }
-
- @Override
- public DrillBuf buffer(int size) {
- return buffer(size, size);
- }
-
- @Override
- public long getAllocatedMemory() {
- return acct.getAllocation();
- }
-
- @Override
- public long getPeakMemoryAllocation() {
- return acct.getPeakMemoryAllocation();
- }
-
- @Override
- public ByteBufAllocator getUnderlyingAllocator() {
- return innerAllocator;
- }
-
- @Override
- public BufferAllocator getChildAllocator(LimitConsumer limitConsumer, long initialReservation,
- long maximumReservation,
- boolean applyFragmentLimit) {
- if(!acct.reserve(initialReservation)){
- logger.debug(String.format("You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.", initialReservation, acct.getCapacity() - acct.getAllocation()));
- throw new OutOfMemoryException(
- String
- .format(
- "You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.",
- initialReservation, acct.getCapacity() - acct.getAllocation()));
- }
- logger.debug("New child allocator with initial reservation {}", initialReservation);
- ChildAllocator allocator = new ChildAllocator(limitConsumer, acct, maximumReservation, initialReservation,
- childrenMap, applyFragmentLimit);
- if(ENABLE_ACCOUNTING){
- childrenMap.put(allocator, Thread.currentThread().getStackTrace());
- }
-
- return allocator;
- }
-
- @Override
- public void resetLimits() {
- acct.resetFragmentLimits();
- }
-
- @Override
- public void setLimit(long limit){
- acct.setFragmentLimit(limit);
- }
-
- @Override
- public long getLimit(){
- return acct.getFragmentLimit();
- }
-
- @Override
- public void close() {
- if (ENABLE_ACCOUNTING) {
- for (Entry<ChildAllocator, StackTraceElement[]> child : childrenMap.entrySet()) {
- if (!child.getKey().isClosed()) {
- StringBuilder sb = new StringBuilder();
- StackTraceElement[] elements = child.getValue();
- for (int i = 0; i < elements.length; i++) {
- sb.append("\t\t");
- sb.append(elements[i]);
- sb.append("\n");
- }
- throw new IllegalStateException("Failure while trying to close allocator: Child level allocators not closed. Stack trace: \n" + sb);
- }
- }
- }
- acct.close();
- }
-
-
-
- @Override
- public DrillBuf getEmpty() {
- return empty;
- }
-
-
-
- private class ChildAllocator implements BufferAllocator {
- private final DrillBuf empty;
- private AccountorImpl childAcct;
- private Map<ChildAllocator, StackTraceElement[]> children = new HashMap<>();
- private boolean closed = false;
- private LimitConsumer limitConsumer;
- private Map<ChildAllocator, StackTraceElement[]> thisMap;
- private boolean applyFragmentLimit;
-
- public ChildAllocator(LimitConsumer limitConsumer,
- AccountorImpl parentAccountor,
- long max,
- long pre,
- Map<ChildAllocator,
- StackTraceElement[]> map,
- boolean applyFragmentLimit) {
- assert max >= pre;
- this.applyFragmentLimit = applyFragmentLimit;
- this.limitConsumer = limitConsumer;
- childAcct = new AccountorImpl(config, errorOnLeak, limitConsumer, parentAccountor, max, pre, applyFragmentLimit);
- thisMap = map;
- empty = DrillBuf.getEmpty(this, childAcct);
- }
-
- @Override
- public boolean takeOwnership(DrillBuf buf) {
- return buf.transferAccounting(childAcct);
- }
-
- @Override
- public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> out) {
- DrillBuf b = new DrillBuf(this, acct, buf);
- out.value = b;
- return acct.transferIn(b, b.capacity());
- }
-
-
- @Override
- public DrillBuf buffer(int size, int max) {
- if (size == 0) {
- return empty;
- }
- if(!childAcct.reserve(size)) {
- throw new OutOfMemoryException(createErrorMsg(this, size));
- }
-
- try {
- UnsafeDirectLittleEndian buffer = innerAllocator.directBuffer(size, max);
- DrillBuf wrapped = new DrillBuf(this, childAcct, buffer);
- childAcct.reserved(buffer.capacity(), wrapped);
- return wrapped;
- } catch (OutOfMemoryError e) {
- if ("Direct buffer memory".equals(e.getMessage())) {
- childAcct.release(size);
- throw new OutOfMemoryException(createErrorMsg(this, size), e);
- } else {
- throw e;
- }
- }
- }
-
- public DrillBuf buffer(int size) {
- return buffer(size, size);
- }
-
- @Override
- public ByteBufAllocator getUnderlyingAllocator() {
- return innerAllocator;
- }
-
- @Override
- public BufferAllocator getChildAllocator(LimitConsumer limitConsumer, long initialReservation,
- long maximumReservation,
- boolean applyFragmentLimit) {
- if (!childAcct.reserve(initialReservation)) {
- throw new OutOfMemoryException(
- String
- .format(
- "You attempted to create a new child allocator with initial reservation %d but only %d bytes of memory were available.",
- initialReservation, childAcct.getAvailable()));
- }
- logger.debug("New child allocator with initial reservation {}", initialReservation);
- ChildAllocator newChildAllocator = new ChildAllocator(limitConsumer, childAcct, maximumReservation,
- initialReservation, null, applyFragmentLimit);
- this.children.put(newChildAllocator, Thread.currentThread().getStackTrace());
- return newChildAllocator;
- }
-
- public PreAllocator getNewPreAllocator() {
- return new PreAlloc(this, this.childAcct);
- }
-
- @Override
- public void resetLimits(){
- childAcct.resetFragmentLimits();
- }
-
- @Override
- public void setLimit(long limit){
- childAcct.setFragmentLimit(limit);
- }
-
- @Override
- public long getLimit(){
- return childAcct.getFragmentLimit();
- }
-
- @Override
- public void close() {
- if (ENABLE_ACCOUNTING) {
- if (thisMap != null) {
- thisMap.remove(this);
- }
- for (ChildAllocator child : children.keySet()) {
- if (!child.isClosed()) {
- StringBuilder sb = new StringBuilder();
- StackTraceElement[] elements = children.get(child);
- for (int i = 1; i < elements.length; i++) {
- sb.append("\t\t");
- sb.append(elements[i]);
- sb.append("\n");
- }
-
-
- IllegalStateException e = new IllegalStateException(String.format(
- "Failure while trying to close child allocator: Child level allocators not closed. Identifier: %s. Stack trace: \n %s",
- limitConsumer.getIdentifier(), sb.toString()));
- if (errorOnLeak) {
- throw e;
- } else {
- logger.warn("Memory leak.", e);
- }
- }
- }
- }
- childAcct.close();
- closed = true;
- }
-
- public boolean isClosed() {
- return closed;
- }
-
- @Override
- public long getAllocatedMemory() {
- return childAcct.getAllocation();
- }
-
- @Override
- public long getPeakMemoryAllocation() {
- return childAcct.getPeakMemoryAllocation();
- }
-
- @Override
- public DrillBuf getEmpty() {
- return empty;
- }
-
-
- }
-
- public PreAllocator getNewPreAllocator() {
- return new PreAlloc(this, this.acct);
- }
-
- public class PreAlloc implements PreAllocator{
- int bytes = 0;
- final Accountor acct;
- final BufferAllocator allocator;
- private PreAlloc(BufferAllocator allocator, Accountor acct) {
- this.acct = acct;
- this.allocator = allocator;
- }
-
- /**
- *
- */
- public boolean preAllocate(int bytes) {
-
- if (!acct.reserve(bytes)) {
- return false;
- }
- this.bytes += bytes;
- return true;
-
- }
-
-
- public DrillBuf getAllocation() {
- DrillBuf b = new DrillBuf(allocator, acct, innerAllocator.directBuffer(bytes, bytes));
- acct.reserved(bytes, b);
- return b;
- }
- }
-
- private static String createErrorMsg(final BufferAllocator allocator, final int size) {
- return String.format("Unable to allocate buffer of size %d due to memory limit. Current allocation: %d",
- size, allocator.getAllocatedMemory());
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
deleted file mode 100644
index 7ef121e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.metrics;
-
-import java.lang.management.ManagementFactory;
-import java.util.Map.Entry;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
-
-import com.codahale.metrics.JmxReporter;
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.MetricSet;
-import com.codahale.metrics.Slf4jReporter;
-import com.codahale.metrics.jvm.BufferPoolMetricSet;
-import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
-import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
-import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
-
-public class DrillMetrics {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMetrics.class);
- static final DrillConfig config = DrillConfig.create();
-
- private DrillMetrics() {
- }
-
- private static class RegistryHolder {
- public static final MetricRegistry REGISTRY;
- // private static final JmxReporter JMX_REPORTER;
- private static final Slf4jReporter LOG_REPORTER;
-
- static {
- REGISTRY = new MetricRegistry();
- registerSysStats();
- // JMX_REPORTER = getJmxReporter();
- LOG_REPORTER = getLogReporter();
- }
-
- private static void registerSysStats(){
- REGISTRY.registerAll(new GarbageCollectorMetricSet());
- REGISTRY.registerAll(new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()));
- REGISTRY.registerAll(new MemoryUsageGaugeSet());
- REGISTRY.registerAll(new ThreadStatesGaugeSet());
- }
-
- private static JmxReporter getJmxReporter() {
- if (config.getBoolean(ExecConstants.METRICS_JMX_OUTPUT_ENABLED)) {
- JmxReporter reporter = JmxReporter.forRegistry(getInstance()).build();
- reporter.start();
-
- return reporter;
- } else {
- return null;
- }
- }
-
- private static Slf4jReporter getLogReporter() {
- if (config.getBoolean(ExecConstants.METRICS_LOG_OUTPUT_ENABLED)) {
- Slf4jReporter reporter = Slf4jReporter.forRegistry(getInstance()).outputTo(logger)
- .convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build();
- reporter.start(config.getInt(ExecConstants.METRICS_LOG_OUTPUT_INTERVAL), TimeUnit.SECONDS);
-
- return reporter;
- } else {
- return null;
- }
- }
- }
-
- private static void registerAll(String prefix, MetricSet metricSet, MetricRegistry registry) {
- for (Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
- if (entry.getValue() instanceof MetricSet) {
- registerAll(prefix + "." + entry.getKey(), (MetricSet) entry.getValue(), registry);
- } else {
- registry.register(prefix + "." + entry.getKey(), entry.getValue());
- }
- }
- }
-
- public static MetricRegistry getInstance() {
- return RegistryHolder.REGISTRY;
- }
-
- public static void resetMetrics(){
- RegistryHolder.REGISTRY.removeMatching(new MetricFilter(){
- @Override
- public boolean matches(String name, Metric metric) {
- return true;
- }});
- RegistryHolder.registerSysStats();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
deleted file mode 100644
index 18a39ac..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.metrics;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.MetricRegistry;
-
-/**
- * Wraps a parent counter so that local in-thread metrics can be collected while collecting for a global counter. Note
- * that this is one writer, many reader safe.
- */
-public class SingleThreadNestedCounter {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SingleThreadNestedCounter.class);
-
- private volatile long count;
- private final Counter counter;
-
- public SingleThreadNestedCounter(MetricRegistry registry, String name) {
- super();
- this.counter = registry.counter(name);
- }
-
- public long inc(long n) {
- counter.inc(n);
- count += n;
- return count;
- }
-
- public long dec(long n) {
- counter.dec(n);
- count -= n;
- return count;
- }
-
- public long get() {
- return count;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/AbstractHandshakeHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/AbstractHandshakeHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/AbstractHandshakeHandler.java
deleted file mode 100644
index 9f426e4..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/AbstractHandshakeHandler.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.MessageToMessageDecoder;
-
-import java.util.List;
-
-import com.google.protobuf.Internal.EnumLite;
-import com.google.protobuf.MessageLite;
-import com.google.protobuf.Parser;
-
-public abstract class AbstractHandshakeHandler<T extends MessageLite> extends MessageToMessageDecoder<InboundRpcMessage> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractHandshakeHandler.class);
-
- protected final EnumLite handshakeType;
- protected final Parser<T> parser;
- protected volatile int coordinationId;
-
- public AbstractHandshakeHandler(EnumLite handshakeType, Parser<T> parser) {
- super();
- this.handshakeType = handshakeType;
- this.parser = parser;
- }
-
-
- @Override
- protected void decode(ChannelHandlerContext ctx, InboundRpcMessage inbound, List<Object> outputs) throws Exception {
- try {
- if (RpcConstants.EXTRA_DEBUGGING) {
- logger.debug("Received handshake {}", inbound);
- }
- this.coordinationId = inbound.coordinationId;
- ctx.channel().pipeline().remove(this);
- if (inbound.rpcType != handshakeType.getNumber()) {
- throw new RpcException(String.format("Handshake failure. Expected %s[%d] but received number [%d]",
- handshakeType, handshakeType.getNumber(), inbound.rpcType));
- }
-
- T msg = parser.parseFrom(inbound.getProtobufBodyAsIS());
- consumeHandshake(ctx, msg);
- } finally {
- // Consuming a handshake may result in exceptions, so make sure to release the message buffers.
- inbound.pBody.release();
- if (inbound.dBody != null) {
- inbound.dBody.release();
- }
- }
- }
-
- protected abstract void consumeHandshake(ChannelHandlerContext ctx, T msg) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Acks.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Acks.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Acks.java
deleted file mode 100644
index 3656717..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Acks.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
-
-public class Acks {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Acks.class);
-
- public static final Ack OK = Ack.newBuilder().setOk(true).build();
- public static final Ack FAIL = Ack.newBuilder().setOk(false).build();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/1fde9bb1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BaseRpcOutcomeListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BaseRpcOutcomeListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BaseRpcOutcomeListener.java
deleted file mode 100644
index dfc62e0..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BaseRpcOutcomeListener.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.rpc;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
-
-public class BaseRpcOutcomeListener<T> implements RpcOutcomeListener<T> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseRpcOutcomeListener.class);
-
- @Override
- public void failed(RpcException ex) {
- }
-
- @Override
- public void success(T value, ByteBuf buffer) {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void interrupted(final InterruptedException ex) {
- }
-}
[37/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java b/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
deleted file mode 100644
index 611e70b..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import org.apache.drill.common.expression.LogicalExpression;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class JoinCondition {
- private final String relationship;
- private final LogicalExpression left;
- private final LogicalExpression right;
-
- @JsonCreator
- public JoinCondition(@JsonProperty("relationship") String relationship,
- @JsonProperty("left") LogicalExpression left, @JsonProperty("right") LogicalExpression right) {
- super();
- this.relationship = relationship;
- this.left = left;
- this.right = right;
- }
-
- public String getRelationship() {
- return relationship;
- }
-
- public LogicalExpression getLeft() {
- return left;
- }
-
- public LogicalExpression getRight() {
- return right;
- }
-
- public JoinCondition flip(){
- return new JoinCondition(relationship, right, left);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Limit.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Limit.java b/common/src/main/java/org/apache/drill/common/logical/data/Limit.java
deleted file mode 100644
index 56ae8d9..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Limit.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Iterators;
-
-@JsonTypeName("limit")
-public class Limit extends SingleInputOperator {
-
- private final Integer first;
- private final Integer last;
-
- @JsonCreator
- public Limit(@JsonProperty("first") Integer first, @JsonProperty("last") Integer last) {
- super();
- this.first = first;
- this.last = last;
- }
-
- public Integer getFirst() {
- return first;
- }
-
- public Integer getLast() {
- return last;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitLimit(this, value);
- }
-
- @Override
- public NodeBuilder nodeBuilder() {
- return new LimitNodeBuilder(); //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
- public static class LimitNodeBuilder implements NodeBuilder<Limit> {
-
- @Override
- public ObjectNode convert(ObjectMapper mapper, Limit operator, Integer inputId) {
- ObjectNode limitNode = mapper.createObjectNode();
- limitNode.put("op", "limit");
- limitNode.put("input", inputId);
- limitNode.put("first", operator.first);
- limitNode.put("last", operator.last);
- return limitNode;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java b/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
deleted file mode 100644
index 3343d4e..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.drill.common.graph.GraphValue;
-import org.apache.drill.common.logical.ValidationError;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonIdentityInfo;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import com.fasterxml.jackson.annotation.ObjectIdGenerators;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-@JsonPropertyOrder({"@id", "memo", "input"}) // op will always be first since it is wrapped.
-@JsonIdentityInfo(generator = ObjectIdGenerators.IntSequenceGenerator.class, property = "@id")
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "op")
-public interface LogicalOperator extends GraphValue<LogicalOperator> {
-
- public void setupAndValidate(List<LogicalOperator> operators, Collection<ValidationError> errors);
-
- /**
- * Provides capability to build a set of output based on traversing a query graph tree.
- *
- * @param logicalVisitor
- * @return
- */
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E;
-
- public void registerAsSubscriber(LogicalOperator operator);
-
- NodeBuilder nodeBuilder();
-
- public interface NodeBuilder<T extends LogicalOperator> {
- ObjectNode convert(ObjectMapper mapper, T operator, Integer inputId);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java b/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
deleted file mode 100644
index a213c8d..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.drill.common.graph.GraphVisitor;
-import org.apache.drill.common.logical.ValidationError;
-import org.apache.drill.common.scanner.persistence.ScanResult;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-
-public abstract class LogicalOperatorBase implements LogicalOperator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalOperatorBase.class);
-
- private List<LogicalOperator> children = new ArrayList<LogicalOperator>();
-
- private String memo;
-
- @Override
- public final int hashCode() {
- return super.hashCode();
- }
-
- @Override
- public void setupAndValidate(List<LogicalOperator> operators, Collection<ValidationError> errors) {
- // TODO: remove this and implement individually.
- }
-
- @Override
- public NodeBuilder nodeBuilder() {
- // FIXME: Implement this on all logical operators
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public void accept(GraphVisitor<LogicalOperator> visitor) {
- if (visitor.enter(this)) {
- visitor.leave(this);
- }
- }
-
- @Override
- public void registerAsSubscriber(LogicalOperator operator) {
- if (operator == null) {
- throw new IllegalArgumentException("You attempted to register a null operators.");
- }
- children.add(operator);
- }
-
- @Override
- public String toString() {
- return this.getClass().getSimpleName() + " [memo=" + memo + "]";
- }
-
- @JsonInclude(Include.NON_EMPTY)
- @JsonProperty("memo")
- public String getMemo() {
- return memo;
- }
-
- public void setMemo(String memo) {
- this.memo = memo;
- }
-
- public static Set<Class<? extends LogicalOperator>> getSubTypes(final ScanResult classpathScan) {
- final Set<Class<? extends LogicalOperator>> ops = classpathScan.getImplementations(LogicalOperator.class);
- logger.debug("Found {} logical operator classes: {}.", ops.size(), ops);
- return ops;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/NamedExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/NamedExpression.java b/common/src/main/java/org/apache/drill/common/logical/data/NamedExpression.java
deleted file mode 100644
index a166c25..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/NamedExpression.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-
-@JsonPropertyOrder({"ref", "expr"})
-public class NamedExpression {
- private final LogicalExpression expr;
- private final FieldReference ref;
-
- @JsonCreator
- public NamedExpression(@JsonProperty("expr") LogicalExpression expr, @JsonProperty("ref") FieldReference ref) {
- super();
- this.expr = expr;
- this.ref = ref;
- }
-
- public LogicalExpression getExpr() {
- return expr;
- }
-
- public FieldReference getRef() {
- return ref;
- }
-
- @Override
- public String toString() {
- return "NamedExpression [expr=" + expr + ", ref=" + ref + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Order.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Order.java b/common/src/main/java/org/apache/drill/common/logical/data/Order.java
deleted file mode 100644
index fca6010..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Order.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-import org.apache.calcite.rel.RelFieldCollation;
-import org.apache.calcite.rel.RelFieldCollation.Direction;
-import org.apache.calcite.rel.RelFieldCollation.NullDirection;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-
-@JsonTypeName("order")
-public class Order extends SingleInputOperator {
-
- private final Ordering[] orderings;
- private final FieldReference within;
-
- @JsonCreator
- public Order(@JsonProperty("within") FieldReference within, @JsonProperty("orderings") Ordering... orderings) {
- this.orderings = orderings;
- this.within = within;
- }
-
- public Ordering[] getOrderings() {
- return orderings;
- }
-
- public FieldReference getWithin() {
- return within;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitOrder(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
-
- /**
- * Representation of a SQL <sort specification>.
- */
- public static class Ordering {
-
- private final LogicalExpression expr;
- /** Net <ordering specification>. */
- private final Direction direction;
- /** Net <null ordering> */
- private final NullDirection nullOrdering;
-
- /**
- * Constructs a sort specification.
- * @param expr ...
- * @param strOrderingSpec the <ordering specification> as string;
- * allowed values: {@code "ASC"}, {@code "DESC"}, {@code null};
- * null specifies default <ordering specification>
- * ({@code "ASC"} / {@link Direction#ASCENDING})
- * @param strNullOrdering the <null ordering> as string;
- * allowed values: {@code "FIRST"}, {@code "LAST"},
- * {@code "UNSPECIFIED"}, {@code null};
- * null specifies default <null ordering>
- * (omitted / {@link NullDirection#UNSPECIFIED}, interpreted later)
- */
- @JsonCreator
- public Ordering( @JsonProperty("expr") LogicalExpression expr,
- @JsonProperty("order") String strOrderingSpec,
- @JsonProperty("nullDirection") String strNullOrdering ) {
- this.expr = expr;
- this.direction = getOrderingSpecFromString( strOrderingSpec );
- this.nullOrdering = getNullOrderingFromString( strNullOrdering );
- }
-
- public Ordering(Direction direction, LogicalExpression e, NullDirection nullOrdering) {
- this.expr = e;
- this.direction = direction;
- this.nullOrdering = nullOrdering;
- }
-
- public Ordering(Direction direction, LogicalExpression e) {
- this(direction, e, NullDirection.FIRST);
- }
-
- private static Direction getOrderingSpecFromString( String strDirection ) {
- final Direction direction;
- if ( null == strDirection
- || Direction.ASCENDING.shortString.equals( strDirection ) ) {
- direction = Direction.ASCENDING;
- }
- else if ( Direction.DESCENDING.shortString.equals( strDirection ) ) {
- direction = Direction.DESCENDING;
- }
- else {
- throw new DrillRuntimeException(
- "Unknown <ordering specification> string (not \"ASC\", \"DESC\", "
- + "or null): \"" + strDirection + "\"" );
- }
- return direction;
- }
-
- private static NullDirection getNullOrderingFromString( String strNullOrdering ) {
- final RelFieldCollation.NullDirection nullOrdering;
- if ( null == strNullOrdering ) {
- nullOrdering = NullDirection.UNSPECIFIED;
- }
- else {
- try {
- nullOrdering = NullDirection.valueOf( strNullOrdering );
- }
- catch ( IllegalArgumentException e ) {
- throw new DrillRuntimeException(
- "Internal error: Unknown <null ordering> string (not "
- + "\"" + NullDirection.FIRST.name() + "\", "
- + "\"" + NullDirection.LAST.name() + "\", or "
- + "\"" + NullDirection.UNSPECIFIED.name() + "\" or null): "
- + "\"" + strNullOrdering + "\"" );
- }
- }
- return nullOrdering;
- }
-
- @Override
- public String toString() {
- return
- super.toString()
- + "[ "
- + " expr = " + expr
- + ", direction = " + direction
- + ", nullOrdering = " + nullOrdering
- + "] ";
- }
-
- @JsonIgnore
- public Direction getDirection() {
- return direction;
- }
-
- public LogicalExpression getExpr() {
- return expr;
- }
-
- public String getOrder() {
- switch (direction) {
- case ASCENDING:
- return Direction.ASCENDING.shortString;
- case DESCENDING:
- return Direction.DESCENDING.shortString;
- default:
- throw new DrillRuntimeException(
- "Unexpected " + Direction.class.getName() + " value other than "
- + Direction.ASCENDING + " or " + Direction.DESCENDING + ": "
- + direction );
- }
- }
-
- public NullDirection getNullDirection() {
- return nullOrdering;
- }
-
- /**
- * Reports whether NULL sorts high or low in this ordering.
- *
- * @return
- * {@code true} if NULL sorts higher than any other value;
- * {@code false} if NULL sorts lower than any other value
- */
- public boolean nullsSortHigh() {
- final boolean nullsHigh;
-
- switch (nullOrdering) {
-
- case UNSPECIFIED:
- // Default: NULL sorts high: like NULLS LAST if ASC, FIRST if DESC.
- nullsHigh = true;
- break;
-
- case FIRST:
- // FIRST: NULL sorts low with ASC, high with DESC.
- nullsHigh = Direction.DESCENDING == getDirection();
- break;
-
- case LAST:
- // LAST: NULL sorts high with ASC, low with DESC.
- nullsHigh = Direction.ASCENDING == getDirection();
- break;
-
- default:
- throw new DrillRuntimeException(
- "Unexpected " + NullDirection.class.getName() + " value other than "
- + NullDirection.FIRST + ", " + NullDirection.LAST + " or " + NullDirection.UNSPECIFIED + ": "
- + nullOrdering );
- }
-
- return nullsHigh;
- }
-
- }
-
- public static Builder builder(){
- return new Builder();
- }
-
- public static class Builder extends AbstractSingleBuilder<Order, Builder>{
- private List<Ordering> orderings = Lists.newArrayList();
- private FieldReference within;
-
- public Builder setWithin(FieldReference within){
- this.within = within;
- return this;
- }
-
- public Builder addOrdering(Direction direction, LogicalExpression e, NullDirection collation){
- orderings.add(new Ordering(direction, e, collation));
- return this;
- }
-
- @Override
- public Order internalBuild() {
- return new Order(within, orderings.toArray(new Ordering[orderings.size()]));
- }
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Project.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Project.java b/common/src/main/java/org/apache/drill/common/logical/data/Project.java
deleted file mode 100644
index 4345c80..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Project.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.List;
-
-import org.apache.drill.common.exceptions.ExpressionParsingException;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Lists;
-
-@JsonTypeName("project")
-public class Project extends SingleInputOperator {
-
- private final NamedExpression[] selections;
-
- @JsonCreator
- public Project(@JsonProperty("projections") NamedExpression[] selections) {
- this.selections = selections;
- if (selections == null || selections.length == 0) {
- throw new ExpressionParsingException(
- "Project did not provide any projection selections. At least one projection must be provided.");
-// for (int i = 0; i < selections.length; i++) {
-// PathSegment segment = selections[i].getRef().getRootSegment();
-// CharSequence path = segment.getNameSegment().getPath();
-// if (!segment.isNamed() || !path.equals("output"))
-// throw new ExpressionParsingException(
-// String
-// .format(
-// "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]",
-// path, segment.isNamed()));
-//
-// }
- }
- }
-
- @JsonProperty("projections")
- public NamedExpression[] getSelections() {
- return selections;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitProject(this, value);
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static class Builder extends AbstractSingleBuilder<Project, Builder> {
-
- private List<NamedExpression> exprs = Lists.newArrayList();
-
- public Builder addExpr(NamedExpression expr) {
- exprs.add(expr);
- return this;
- }
-
- public Builder addExpr(FieldReference ref, LogicalExpression expr) {
- exprs.add(new NamedExpression(expr, ref));
- return this;
- }
-
- @Override
- public Project internalBuild() {
- return new Project(aN(exprs));
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/RunningAggregate.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/RunningAggregate.java b/common/src/main/java/org/apache/drill/common/logical/data/RunningAggregate.java
deleted file mode 100644
index e280627..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/RunningAggregate.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-
-@JsonTypeName("runningaggregate")
-public class RunningAggregate extends SingleInputOperator{
-
- private final FieldReference within;
- private final NamedExpression[] aggregations;
-
- @JsonCreator
- public RunningAggregate(@JsonProperty("within") FieldReference within, @JsonProperty("aggregations") NamedExpression[] aggregations) {
- super();
- this.within = within;
- this.aggregations = aggregations;
- }
-
- public FieldReference getWithin() {
- return within;
- }
-
- public NamedExpression[] getAggregations() {
- return aggregations;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitRunningAggregate(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Scan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Scan.java b/common/src/main/java/org/apache/drill/common/logical/data/Scan.java
deleted file mode 100644
index 68eabbb..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Scan.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-@JsonTypeName("scan")
-public class Scan extends SourceOperator {
- private final String storageEngine;
- private final JSONOptions selection;
-
- @JsonCreator
- public Scan(@JsonProperty("storageengine") String storageEngine, @JsonProperty("selection") JSONOptions selection) {
- super();
- this.storageEngine = storageEngine;
- this.selection = selection;
- }
-
- @JsonProperty("storageengine")
- public String getStorageEngine() {
- return storageEngine;
- }
-
- public JSONOptions getSelection() {
- return selection;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitScan(this, value);
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static class Builder extends AbstractBuilder<Scan>{
- private String storageEngine;
- private JSONOptions selection;
-
- public Builder storageEngine(String storageEngine) {
- this.storageEngine = storageEngine;
- return this;
- }
-
- public Builder selection(JSONOptions selection) {
- this.selection = selection;
- return this;
- }
-
- @Override
- public Scan build() {
- return new Scan(storageEngine, selection);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Sequence.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Sequence.java b/common/src/main/java/org/apache/drill/common/logical/data/Sequence.java
deleted file mode 100644
index f04b09c..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Sequence.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.drill.common.logical.data.Sequence.De;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.fasterxml.jackson.annotation.ObjectIdGenerator;
-import com.fasterxml.jackson.annotation.ObjectIdGenerators;
-import com.fasterxml.jackson.core.JsonLocation;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.JsonToken;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JavaType;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.deser.impl.ReadableObjectId;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.google.common.collect.Iterators;
-
-/**
- * Describes a list of operators where each operator only has one input and that
- * input is the operator that came before.
- *
- */
-@JsonDeserialize(using = De.class)
-@JsonTypeName("sequence")
-public class Sequence extends LogicalOperatorBase {
- static final Logger logger = LoggerFactory.getLogger(Sequence.class);
-
- private Sequence() {}
-
- public boolean openTop;
- public LogicalOperator input;
- @JsonProperty("do")
- public LogicalOperator[] stream;
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitSequence(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(stream[stream.length - 1]);
- }
-
- public static class De extends StdDeserializer<LogicalOperator> {
-
- protected De() {
- super(Sequence.class);
- }
-
- @Override
- public LogicalOperator deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
- JsonProcessingException {
- ObjectIdGenerator<Integer> idGenerator = new ObjectIdGenerators.IntSequenceGenerator();
- JsonLocation start = jp.getCurrentLocation();
- JsonToken t = jp.getCurrentToken();
- LogicalOperator parent = null;
- LogicalOperator first = null;
- LogicalOperator prev = null;
- Integer id = null;
-
- while (true) {
- String fieldName = jp.getText();
- t = jp.nextToken();
- switch (fieldName) { // switch on field names.
- case "@id":
- id = _parseIntPrimitive(jp, ctxt);
- break;
- case "input":
- JavaType tp = ctxt.constructType(LogicalOperator.class);
- JsonDeserializer<Object> d = ctxt.findRootValueDeserializer(tp);
- parent = (LogicalOperator) d.deserialize(jp, ctxt);
- break;
-
- case "do":
- if (!jp.isExpectedStartArrayToken()) {
- throwE(
- jp,
- "The do parameter of sequence should be an array of SimpleOperators. Expected a JsonToken.START_ARRAY token but received a "
- + t.name() + "token.");
- }
-
- int pos = 0;
- while ((t = jp.nextToken()) != JsonToken.END_ARRAY) {
- // logger.debug("Reading sequence child {}.", pos);
- JsonLocation l = jp.getCurrentLocation(); // get current location
- // first so we can
- // correctly reference the
- // start of the object in
- // the case that the type
- // is wrong.
- LogicalOperator o = jp.readValueAs(LogicalOperator.class);
-
- if (pos == 0) {
- if (!(o instanceof SingleInputOperator) && !(o instanceof SourceOperator)) {
- throwE(
- l,
- "The first operator in a sequence must be either a ZeroInput or SingleInput operator. The provided first operator was not. It was of type "
- + o.getClass().getName());
- }
- first = o;
- } else {
- if (!(o instanceof SingleInputOperator)) {
- throwE(l, "All operators after the first must be single input operators. The operator at position "
- + pos + " was not. It was of type " + o.getClass().getName());
- }
- SingleInputOperator now = (SingleInputOperator) o;
- now.setInput(prev);
- }
- prev = o;
-
- pos++;
- }
- break;
- default:
- throwE(jp, "Unknown field name provided for Sequence: " + jp.getText());
- }
-
- t = jp.nextToken();
- if (t == JsonToken.END_OBJECT) {
- break;
- }
- }
-
- if (first == null) {
- throwE(start, "A sequence must include at least one operator.");
- }
- if ((parent == null && first instanceof SingleInputOperator)
- || (parent != null && first instanceof SourceOperator)) {
- throwE(start,
- "A sequence must either start with a ZeroInputOperator or have a provided input. It cannot have both or neither.");
- }
-
- if (parent != null && first instanceof SingleInputOperator) {
- ((SingleInputOperator) first).setInput(parent);
- }
-
- // set input reference.
- if (id != null) {
-
- ReadableObjectId rid = ctxt.findObjectId(id, idGenerator);
- rid.bindItem(prev);
- // logger.debug("Binding id {} to item {}.", rid.id, rid.item);
-
- }
-
- return first;
- }
-
- }
-
- private static void throwE(JsonLocation l, String e) throws JsonParseException {
- throw new JsonParseException(e, l);
- }
-
- private static void throwE(JsonParser jp, String e) throws JsonParseException {
- throw new JsonParseException(e, jp.getCurrentLocation());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java b/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
deleted file mode 100644
index e46cb6b..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.logical.UnexpectedOperatorType;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.Iterators;
-
-/**
- * SimpleOperator is an operator that has one inputs at most.
- */
-public abstract class SingleInputOperator extends LogicalOperatorBase {
-
- private LogicalOperator input;
-
- @JsonProperty("input")
- public LogicalOperator getInput() {
- return input;
- }
-
- @JsonProperty(value="input", required=true)
- public void setInput(LogicalOperator input) {
- if (input instanceof SinkOperator) {
- throw new UnexpectedOperatorType("You have set the input of a sink node of type ["+input.getClass().getSimpleName()+ "] as the input for another node of type ["+this.getClass().getSimpleName()+ "]. This is invalid.");
- }
- this.input = input;
- input.registerAsSubscriber(this);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(input);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/SinkOperator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/SinkOperator.java b/common/src/main/java/org/apache/drill/common/logical/data/SinkOperator.java
deleted file mode 100644
index 2ca4006..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/SinkOperator.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-
-/**
- * An operator that cannot be subscribed to.
- */
-public abstract class SinkOperator extends SingleInputOperator{
-
- @Override
- public void registerAsSubscriber(LogicalOperator operator) {
- throw new UnsupportedOperationException("You can not subscribe to a sink operator such as " + this.getClass().getSimpleName());
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/SourceOperator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/SourceOperator.java b/common/src/main/java/org/apache/drill/common/logical/data/SourceOperator.java
deleted file mode 100644
index d4f8246..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/SourceOperator.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-
-import com.google.common.collect.Iterators;
-
-
-/**
- * An operator that produces data without any parents. (zero input operator)
- */
-public abstract class SourceOperator extends LogicalOperatorBase{
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.emptyIterator();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Store.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Store.java b/common/src/main/java/org/apache/drill/common/logical/data/Store.java
deleted file mode 100644
index 8f99eed..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Store.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-
-@JsonTypeName("store")
-public class Store extends SinkOperator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Store.class);
-
- private final String storageEngine;
- private final JSONOptions target;
-
- @JsonCreator
- public Store(@JsonProperty("storageengine") String storageEngine, @JsonProperty("target") JSONOptions target) {
- super();
- this.storageEngine = storageEngine;
- this.target = target;
- }
-
- public String getStorageEngine() {
- return storageEngine;
- }
-
- public JSONOptions getTarget() {
- return target;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitStore(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static class Builder extends AbstractSingleBuilder<Store, Builder>{
- private String storageEngine;
- private JSONOptions target;
-
- public Builder storageEngine(String storageEngine) {
- this.storageEngine = storageEngine;
- return this;
- }
-
- public Builder target(JSONOptions target) {
- this.target = target;
- return this;
- }
-
- @Override
- public Store internalBuild() {
- return new Store(storageEngine, target);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Transform.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Transform.java b/common/src/main/java/org/apache/drill/common/logical/data/Transform.java
deleted file mode 100644
index 129ae2c..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Transform.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-
-@JsonTypeName("transform")
-public class Transform extends SingleInputOperator {
-
- private final NamedExpression[] transforms;
-
- @JsonCreator
- public Transform(@JsonProperty("transforms") NamedExpression[] transforms) {
- super();
- this.transforms = transforms;
- }
-
- public NamedExpression[] getTransforms() {
- return transforms;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitTransform(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Union.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Union.java b/common/src/main/java/org/apache/drill/common/logical/data/Union.java
deleted file mode 100644
index 113ffe0..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Union.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-
-@JsonTypeName("union")
-public class Union extends LogicalOperatorBase {
- private final LogicalOperator[] inputs;
- private final boolean distinct;
-
-// @JsonCreator
-// public Union(@JsonProperty("inputs") LogicalOperator[] inputs){
-// this(inputs, false);
-// }
-
- @JsonCreator
- public Union(@JsonProperty("inputs") LogicalOperator[] inputs, @JsonProperty("distinct") Boolean distinct){
- this.inputs = inputs;
- for (LogicalOperator o : inputs) {
- o.registerAsSubscriber(this);
- }
- this.distinct = distinct == null ? false : distinct;
- }
-
- public LogicalOperator[] getInputs() {
- return inputs;
- }
-
- public boolean isDistinct() {
- return distinct;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitUnion(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.forArray(inputs);
- }
-
-
- public static Builder builder(){
- return new Builder();
- }
-
- public static class Builder extends AbstractBuilder<Union>{
- private List<LogicalOperator> inputs = Lists.newArrayList();
- private boolean distinct;
-
- public Builder addInput(LogicalOperator o){
- inputs.add(o);
- return this;
- }
-
- public Builder setDistinct(boolean distinct){
- this.distinct = distinct;
- return this;
- }
-
- @Override
- public Union build() {
- return new Union(inputs.toArray(new LogicalOperator[inputs.size()]), distinct);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Values.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Values.java b/common/src/main/java/org/apache/drill/common/logical/data/Values.java
deleted file mode 100644
index 9276e55..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Values.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.fasterxml.jackson.core.JsonLocation;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
-
-@JsonTypeName("values")
-public class Values extends SourceOperator {
-
- private final JSONOptions content;
-
- @JsonCreator
- public Values(@JsonProperty("content") JSONOptions content){
- super();
- this.content = content;
- Preconditions.checkNotNull(content, "content attribute is required for source operator 'constant'.");
- }
-
- public JSONOptions getContent() {
- return content;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitValues(this, value);
- }
-
- public static Builder builder(){
- return new Builder();
- }
-
- public static class Builder extends AbstractBuilder<Values>{
- private JSONOptions content;
-
- public Builder content(JsonNode n){
- content = new JSONOptions(n, JsonLocation.NA);
- return this;
- }
-
- @Override
- public Values build() {
- return new Values(content);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Window.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Window.java b/common/src/main/java/org/apache/drill/common/logical/data/Window.java
deleted file mode 100644
index 0f550dc..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Window.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import org.apache.drill.common.expression.FieldReference;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
-
-@JsonTypeName("window")
-public class Window extends SingleInputOperator {
- private final NamedExpression[] withins;
- private final NamedExpression[] aggregations;
- private final Order.Ordering[] orderings;
- private final long start;
- private final long end;
-
-
- @JsonCreator
- public Window(@JsonProperty("withins") NamedExpression[] withins,
- @JsonProperty("aggregations") NamedExpression[] aggregations,
- @JsonProperty("orderings") Order.Ordering[] orderings,
- @JsonProperty("start") Long start,
- @JsonProperty("end") Long end) {
- super();
- this.withins = withins;
- this.start = start == null ? Long.MIN_VALUE : start;
- this.end = end == null ? Long.MIN_VALUE : end;
- this.aggregations = aggregations;
- this.orderings = orderings;
- }
-
- public NamedExpression[] getWithins() {
- return withins;
- }
-
- public long getStart() {
- return start;
- }
-
- public long getEnd() {
- return end;
- }
-
- public NamedExpression[] getAggregations() {
- return aggregations;
- }
-
- public Order.Ordering[] getOrderings() {
- return orderings;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitWindow(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
- public static class Builder extends AbstractSingleBuilder<Window, Builder>{
- private List<NamedExpression> aggregations = Lists.newArrayList();
- private List<NamedExpression> withins = Lists.newArrayList();
- private List<Order.Ordering> orderings = Lists.newArrayList();
- private long start = Long.MIN_VALUE;
- private long end = Long.MIN_VALUE;
-
-
- public Builder addAggregation(FieldReference ref, LogicalExpression expr){
- aggregations.add(new NamedExpression(expr, ref));
- return this;
- }
-
- public Builder addWithin(FieldReference within, LogicalExpression expr) {
- withins.add(new NamedExpression(expr, within));
- return this;
- }
-
- public Window internalBuild() {
- //TODO withins can actually be empty: over(), over(order by <expression>), ...
- checkState(!withins.isEmpty(), "Withins in window must not be empty.");
- checkState(!aggregations.isEmpty(), "Aggregations in window must not be empty.");
- return new Window(aN(withins), aN(aggregations), aO(orderings), start, end);
- }
-
- public Builder addOrdering(Order.Ordering ordering) {
- orderings.add(ordering);
- return this;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Writer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Writer.java b/common/src/main/java/org/apache/drill/common/logical/data/Writer.java
deleted file mode 100644
index 6e51df7..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Writer.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-@JsonTypeName("writer")
-public class Writer extends SingleInputOperator {
-
- // TODO: Hack - Make the type generic until "common" and "java-exec" modules are merged (DRILL-507).
- private final Object createTableEntry;
-
- @JsonCreator
- public Writer(@JsonProperty("createTableEntry") Object createTableEntry) {
- this.createTableEntry = createTableEntry;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitWriter(this, value);
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static class Builder extends AbstractSingleBuilder<Writer, Builder> {
-
- // TODO: Hack - Make the type generic until "common" and "java-exec" modules are merged (DRILL-507).
- private Object createTableEntry;
-
- public Builder setCreateTableEntry(Object createTableEntry) {
- this.createTableEntry = createTableEntry;
- return this;
- }
-
- @Override
- public Writer internalBuild() {
- return new Writer(createTableEntry);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/package-info.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/package-info.java b/common/src/main/java/org/apache/drill/common/logical/data/package-info.java
deleted file mode 100644
index fc76c0d..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Logical operators in a query graph.
- */
-package org.apache.drill.common.logical.data;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
deleted file mode 100644
index 165ee66..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data.visitors;
-
-import org.apache.drill.common.logical.data.Values;
-import org.apache.drill.common.logical.data.Filter;
-import org.apache.drill.common.logical.data.Flatten;
-import org.apache.drill.common.logical.data.GroupingAggregate;
-import org.apache.drill.common.logical.data.Join;
-import org.apache.drill.common.logical.data.Limit;
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.common.logical.data.Order;
-import org.apache.drill.common.logical.data.Project;
-import org.apache.drill.common.logical.data.RunningAggregate;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Sequence;
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.common.logical.data.Transform;
-import org.apache.drill.common.logical.data.Union;
-import org.apache.drill.common.logical.data.Window;
-import org.apache.drill.common.logical.data.Writer;
-
-
-public abstract class AbstractLogicalVisitor<T, X, E extends Throwable> implements LogicalVisitor<T, X, E> {
-
- public T visitOp(LogicalOperator op, X value) throws E{
- throw new UnsupportedOperationException(String.format(
- "The LogicalVisitor of type %s does not currently support visiting the PhysicalOperator type %s.", this
- .getClass().getCanonicalName(), op.getClass().getCanonicalName()));
- }
-
- @Override
- public T visitScan(Scan scan, X value) throws E {
- return visitOp(scan, value);
- }
-
- @Override
- public T visitStore(Store store, X value) throws E {
- return visitOp(store, value);
- }
-
- @Override
- public T visitFilter(Filter filter, X value) throws E {
- return visitOp(filter, value);
- }
-
- @Override
- public T visitFlatten(Flatten flatten, X value) throws E {
- return visitOp(flatten, value);
- }
-
- @Override
- public T visitProject(Project project, X value) throws E {
- return visitOp(project, value);
- }
-
- @Override
- public T visitOrder(Order order, X value) throws E {
- return visitOp(order, value);
- }
-
- @Override
- public T visitJoin(Join join, X value) throws E {
- return visitOp(join, value);
- }
-
- @Override
- public T visitLimit(Limit limit, X value) throws E {
- return visitOp(limit, value);
- }
-
- @Override
- public T visitRunningAggregate(RunningAggregate runningAggregate, X value) throws E {
- return visitOp(runningAggregate, value);
- }
-
- @Override
- public T visitGroupingAggregate(GroupingAggregate groupBy, X value) throws E {
- return visitOp(groupBy, value);
- }
-
- @Override
- public T visitSequence(Sequence sequence, X value) throws E {
- return visitOp(sequence, value);
- }
-
- @Override
- public T visitTransform(Transform transform, X value) throws E {
- return visitOp(transform, value);
- }
-
- @Override
- public T visitUnion(Union union, X value) throws E {
- return visitOp(union, value);
- }
-
- @Override
- public T visitWindow(Window window, X value) throws E {
- return visitOp(window, value);
- }
-
- @Override
- public T visitValues(Values constant, X value) throws E {
- return visitOp(constant, value);
- }
-
- @Override
- public T visitWriter(Writer writer, X value) throws E {
- return visitOp(writer, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
deleted file mode 100644
index 1e07dc6..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data.visitors;
-
-
-import org.apache.drill.common.logical.data.Values;
-import org.apache.drill.common.logical.data.Filter;
-import org.apache.drill.common.logical.data.Flatten;
-import org.apache.drill.common.logical.data.GroupingAggregate;
-import org.apache.drill.common.logical.data.Join;
-import org.apache.drill.common.logical.data.Limit;
-import org.apache.drill.common.logical.data.Order;
-import org.apache.drill.common.logical.data.Project;
-import org.apache.drill.common.logical.data.RunningAggregate;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Sequence;
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.common.logical.data.Transform;
-import org.apache.drill.common.logical.data.Union;
-import org.apache.drill.common.logical.data.Window;
-import org.apache.drill.common.logical.data.Window;
-import org.apache.drill.common.logical.data.Writer;
-
-/**
- * Visitor class designed to traversal of a operator tree. Basis for a number of operator manipulations including fragmentation and materialization.
- * @param <RETURN> The class associated with the return of each visit method.
- * @param <EXTRA> The class object associated with additional data required for a particular operator modification.
- * @param <EXCEP> An optional exception class that can be thrown when a portion of a modification or traversal fails. Must extend Throwable. In the case where the visitor does not throw any caught exception, this can be set as RuntimeException.
- */
-public interface LogicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalVisitor.class);
-
-
- public RETURN visitScan(Scan scan, EXTRA value) throws EXCEP;
- public RETURN visitStore(Store store, EXTRA value) throws EXCEP;
- public RETURN visitGroupingAggregate(GroupingAggregate groupBy, EXTRA value) throws EXCEP;
- public RETURN visitFilter(Filter filter, EXTRA value) throws EXCEP;
- public RETURN visitFlatten(Flatten flatten, EXTRA value) throws EXCEP;
-
- public RETURN visitProject(Project project, EXTRA value) throws EXCEP;
- public RETURN visitValues(Values constant, EXTRA value) throws EXCEP;
- public RETURN visitOrder(Order order, EXTRA value) throws EXCEP;
- public RETURN visitJoin(Join join, EXTRA value) throws EXCEP;
- public RETURN visitLimit(Limit limit, EXTRA value) throws EXCEP;
- public RETURN visitRunningAggregate(RunningAggregate runningAggregate, EXTRA value) throws EXCEP;
- public RETURN visitSequence(Sequence sequence, EXTRA value) throws EXCEP;
- public RETURN visitTransform(Transform transform, EXTRA value) throws EXCEP;
- public RETURN visitUnion(Union union, EXTRA value) throws EXCEP;
- public RETURN visitWindow(Window window, EXTRA value) throws EXCEP;
- public RETURN visitWriter(Writer writer, EXTRA value) throws EXCEP;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/visitors/package-info.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/package-info.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/package-info.java
deleted file mode 100644
index 9b82bb3..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Visitors for Drill logical plan graphs.
- */
-package org.apache.drill.common.logical.data.visitors;
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/package-info.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/package-info.java b/common/src/main/java/org/apache/drill/common/logical/package-info.java
deleted file mode 100644
index 1f88f06..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/package-info.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Storage plugin interfaces and logical plan representation.
- *
- * This package includes the interface for storage and format plugins for Drill
- * as well as the top level class for representing a Drill logical plan and a
- * builder for constructing plans.
- */
-package org.apache.drill.common.logical;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/util/AbstractDynamicBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/AbstractDynamicBean.java b/common/src/main/java/org/apache/drill/common/util/AbstractDynamicBean.java
deleted file mode 100644
index 07a3620..0000000
--- a/common/src/main/java/org/apache/drill/common/util/AbstractDynamicBean.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.util;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.drill.common.exceptions.LogicalPlanParsingException;
-
-import com.fasterxml.jackson.annotation.JsonAnyGetter;
-import com.fasterxml.jackson.annotation.JsonAnySetter;
-import com.fasterxml.jackson.core.JsonParser.Feature;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * Captures all properties and turns them into an object node for late bind conversion.
- */
-public abstract class AbstractDynamicBean {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractDynamicBean.class);
-
- private static volatile ObjectMapper MAPPER;
-
- private ObjectNode objectNode = new ObjectNode(null);
-
- @JsonAnySetter
- public void _anySetter(String name, JsonNode value){
- objectNode.put(name, value);
- }
-
- @JsonAnyGetter
- public Map<String,JsonNode> _anyGetter() {
- Map<String, JsonNode> unknowns = new HashMap<String, JsonNode>();
-
- for(Iterator<Entry<String, JsonNode>> i = objectNode.fields(); i.hasNext(); ){
- Entry<String, JsonNode> e = i.next();
- unknowns.put(e.getKey(), e.getValue());
- }
- return unknowns;
- }
-
- public <T> T getWith(Class<T> c){
- try {
- return getMapper().treeToValue(objectNode, c);
- } catch (JsonProcessingException e) {
- throw new LogicalPlanParsingException(String.format("Failure while trying to convert late bound json type to type of %s.", c.getCanonicalName()), e);
- }
- }
-
-
- private static synchronized ObjectMapper getMapper(){
- if(MAPPER == null){
- ObjectMapper mapper = new ObjectMapper();
- mapper.enable(SerializationFeature.INDENT_OUTPUT);
- mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
- mapper.configure(Feature.ALLOW_COMMENTS, true);
- MAPPER = mapper;
- }
- return MAPPER;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/util/Hook.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/Hook.java b/common/src/main/java/org/apache/drill/common/util/Hook.java
deleted file mode 100644
index 36ddb4e..0000000
--- a/common/src/main/java/org/apache/drill/common/util/Hook.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.util;
-
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import com.google.common.base.Function;
-
-/**
- * Collection of hooks that can be set by observers and are executed at various
- * parts of the query preparation process.
- *
- * <p>For testing and debugging rather than for end-users.</p>
- */
-public enum Hook {
- /** Called with the logical plan. */
- LOGICAL_PLAN;
-
- private final List<Function<Object, Object>> handlers =
- new CopyOnWriteArrayList<>();
-
- /** Adds a handler for this Hook.
- *
- * <p>Returns a {@link Hook.Closeable} so that you can use the following
- * try-finally pattern to prevent leaks:</p>
- *
- * <blockquote><pre>
- * final Hook.Closeable closeable = Hook.FOO.add(HANDLER);
- * try {
- * ...
- * } finally {
- * closeable.close();
- * }</pre>
- * </blockquote>
- */
- public Closeable add(final Function handler) {
- handlers.add(handler);
- return new Closeable() {
- public void close() {
- remove(handler);
- }
- };
- }
-
- /** Removes a handler from this Hook. */
- private boolean remove(Function handler) {
- return handlers.remove(handler);
- }
-
- /** Runs all handlers registered for this Hook, with the given argument. */
- public void run(Object arg) {
- for (Function<Object, Object> handler : handlers) {
- handler.apply(arg);
- }
- }
-
- /** Removes a Hook after use. */
- public interface Closeable extends AutoCloseable {
- void close(); // override, removing "throws"
- }
-}
-
-// End Hook.java
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java b/common/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java
deleted file mode 100644
index 97cd578..0000000
--- a/common/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.drill.test.DrillTest;
-import org.junit.Test;
-
-public class PathSegmentTests extends DrillTest {
- protected PathSegment makeArraySegment(final int len, final PathSegment tail) {
- PathSegment node = tail;
- for (int i = 0; i < len; i++) {
- node = new PathSegment.ArraySegment(node);
- }
- return node;
- }
-
- @Test
- public void testIfMultiLevelCloneWorks() {
- final int levels = 10;
- final PathSegment segment = new PathSegment.NameSegment("test", makeArraySegment(levels, null));
- final PathSegment clone = segment.clone();
- assertEquals("result of clone & original segments must be identical", segment, clone);
-
- final PathSegment tail = new PathSegment.NameSegment("tail");
- final PathSegment newSegment = new PathSegment.NameSegment("test", makeArraySegment(levels, tail));
- final PathSegment newClone = segment.cloneWithNewChild(tail);
- assertEquals("result of cloneWithChild & original segment must be identical", newSegment, newClone);
- }
-}
[15/45] drill git commit: DRILL-3987: (REFACTOR) Working TPCH unit
tests
Posted by ja...@apache.org.
DRILL-3987: (REFACTOR) Working TPCH unit tests
- Rename DecimalUtility.integerSize to INTEGER_SIZE as is constant
- Refactor BufferManager from BufferManagerImpl
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a8297dfe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a8297dfe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a8297dfe
Branch: refs/heads/master
Commit: a8297dfe975b3437cf6af2996b6527e1dfe9692e
Parents: 4524fdb
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Nov 7 18:41:03 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 17:00:46 2015 -0800
----------------------------------------------------------------------
exec/java-exec/pom.xml | 32 +++++++++++++++++---
.../Decimal/CastDecimalDenseDecimalSparse.java | 8 ++---
.../templates/Decimal/CastDecimalSimilar.java | 2 +-
.../Decimal/CastDecimalSparseDecimalDense.java | 12 ++++----
.../templates/Decimal/CastDownwardDecimal.java | 2 +-
.../templates/Decimal/CastFloatDecimal.java | 2 +-
.../templates/Decimal/CastIntDecimal.java | 2 +-
.../templates/Decimal/CastSrcDecimalSimple.java | 6 ++--
.../templates/Decimal/CastVarCharDecimal.java | 2 +-
.../templates/Decimal/DecimalFunctions.java | 10 +++---
.../src/main/codegen/templates/TypeHelper.java | 2 +-
.../apache/drill/exec/memory/AccountorImpl.java | 11 +++++--
.../drill/exec/memory/TopLevelAllocator.java | 4 ++-
.../drill/exec/ops/BufferManagerImpl.java | 17 +++++++++++
.../drill/exec/memory/TestAllocators.java | 4 +--
exec/vector/pom.xml | 10 ++++++
.../main/codegen/templates/BasicTypeHelper.java | 4 +--
17 files changed, 95 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index da072d0..a0398b7 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -471,11 +471,12 @@
</profiles>
<build>
+
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
- <execution> <!-- copy all templates in the same location to compile them at once -->
+ <execution> <!-- copy all templates/data in the same location to compile them at once -->
<id>copy-fmpp-resources</id>
<phase>initialize</phase>
<goals><goal>copy-resources</goal></goals>
@@ -492,12 +493,12 @@
</executions>
</plugin>
<plugin>
- <!-- Extract parser grammar template from calcite-core.jar and put
- it under ${project.build.directory}/codegen/templates where all freemarker templates are. -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
+ <!-- Extract parser grammar template from calcite-core.jar and put
+ it under ${project.build.directory}/codegen/templates where all freemarker templates are. -->
<execution>
<id>unpack-parser-template</id>
<phase>initialize</phase>
@@ -517,8 +518,31 @@
</artifactItems>
</configuration>
</execution>
+
+ <!-- Extract ValueVectorTypes.tdd from drill-vector.jar and put
+ it under ${project.build.directory}/codegen/data where all freemarker data files are. -->
+ <execution>
+ <id>unpack-vector-types</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.drill.exec</groupId>
+ <artifactId>vector</artifactId>
+ <type>jar</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>${project.build.directory}/</outputDirectory>
+ <includes>codegen/data/ValueVectorTypes.tdd</includes>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
</executions>
</plugin>
+
<plugin> <!-- generate sources from fmpp -->
<groupId>org.apache.drill.tools</groupId>
<artifactId>drill-fmpp-maven-plugin</artifactId>
@@ -536,7 +560,7 @@
<phase>generate-sources</phase>
<goals><goal>generate</goal></goals>
<configuration>
- <config>src/main/codegen/config.fmpp</config>
+ <config>${project.build.directory}/codegen/config.fmpp</config>
<output>${project.build.directory}/generated-sources</output>
<templates>${project.build.directory}/codegen/templates</templates>
</configuration>
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java
index 4713041..a110ed7 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java
@@ -53,7 +53,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Output ${type.to}Holder out;
public void setup() {
- int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
}
@@ -75,7 +75,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
* representation rearrange the bits so that we use 32 bits represent the digits.
*/
- byte[] intermediateBytes = new byte[(in.nDecimalDigits * org.apache.drill.exec.util.DecimalUtility.integerSize) + 1];
+ byte[] intermediateBytes = new byte[(in.nDecimalDigits * org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE) + 1];
int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
@@ -104,7 +104,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
inputIndex++;
intermediateIndex++;
- if (((intermediateIndex - 1) % org.apache.drill.exec.util.DecimalUtility.integerSize) == 0) {
+ if (((intermediateIndex - 1) % org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE) == 0) {
shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
maskIndex++;
shiftOrder -= 2;
@@ -119,7 +119,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
* Now we transfer the bytes into a integer array and separate out the scale and
* integer part of the decimal. Also pad the scale part with zeroes if needed
*/
- int[] intermediate = new int[(intermediateBytes.length/org.apache.drill.exec.util.DecimalUtility.integerSize) + 1];
+ int[] intermediate = new int[(intermediateBytes.length/org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE) + 1];
java.nio.ByteBuffer wrapper = java.nio.ByteBuffer.wrap(intermediateBytes);
intermediate[0] = wrapper.get(0);
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
index f9b7430..5ab34ad 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java
@@ -55,7 +55,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Output ${type.to}Holder out;
public void setup() {
- int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java
index a94eb12..62b3229 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java
@@ -53,7 +53,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Output ${type.to}Holder out;
public void setup() {
- int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
}
@@ -124,14 +124,14 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
*/
/* Allocate a byte array */
- int size = (((intermediate.length - 1) * org.apache.drill.exec.util.DecimalUtility.integerSize) + 1);
+ int size = (((intermediate.length - 1) * org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE) + 1);
byte[] intermediateBytes = new byte[size];
java.nio.ByteBuffer wrapper = java.nio.ByteBuffer.wrap(intermediateBytes);
wrapper.put((byte) intermediate[0]);
for (int i = 1; i < intermediate.length; i++) {
- wrapper.put(java.nio.ByteBuffer.allocate(org.apache.drill.exec.util.DecimalUtility.integerSize).putInt(intermediate[i]).array());
+ wrapper.put(java.nio.ByteBuffer.allocate(org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE).putInt(intermediate[i]).array());
}
final int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
@@ -139,21 +139,21 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
int shiftOrder = 2;
// Start just after the last integer and shift bits to the right
- index = size - (org.apache.drill.exec.util.DecimalUtility.integerSize+ 1);
+ index = size - (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE+ 1);
while (index >= 0) {
/* get the last bits that need to shifted to the next byte */
byte shiftBits = (byte) ((intermediateBytes[index] & mask[maskIndex]) << (8 - shiftOrder));
- int shiftOrder1 = ((index % org.apache.drill.exec.util.DecimalUtility.integerSize) == 0) ? shiftOrder - 2 : shiftOrder;
+ int shiftOrder1 = ((index % org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE) == 0) ? shiftOrder - 2 : shiftOrder;
/* transfer the bits from the left to the right */
intermediateBytes[index + 1] = (byte) (((intermediateBytes[index + 1] & 0xFF) >>> (shiftOrder1)) | shiftBits);
index--;
- if ((index % org.apache.drill.exec.util.DecimalUtility.integerSize) == 0) {
+ if ((index % org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE) == 0) {
/* We are on a border */
shiftOrder += 2;
maskIndex++;
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java
index 05c9936..d492392 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDownwardDecimal.java
@@ -170,7 +170,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Output ${type.to}Holder out;
public void setup() {
- int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java
index 8a7919c..f0775d8 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java
@@ -60,7 +60,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void setup() {
<#if type.major == "FloatDecimalComplex" || type.major == "DoubleDecimalComplex">
- int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize);
+ int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE);
buffer = buffer.reallocIfNeeded(size);
</#if>
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
index 6d727b0..368eb8b 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java
@@ -56,7 +56,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void setup() {
<#if type.to.startsWith("Decimal28") || type.to.startsWith("Decimal38")>
- int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize);
+ int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE);
buffer = buffer.reallocIfNeeded(size);
</#if>
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
index 40b03a1..fd0aba4 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java
@@ -57,7 +57,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
@Output ${type.to}Holder out;
public void setup() {
- int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
}
@@ -102,7 +102,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
int shiftOrder = 2;
// Start shifting bits just after the first integer
- int byteIndex = in.WIDTH - (org.apache.drill.exec.util.DecimalUtility.integerSize + 1);
+ int byteIndex = in.WIDTH - (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE + 1);
while (byteIndex >= 0) {
@@ -159,7 +159,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
@Output ${type.to}Holder out;
public void setup() {
- int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
index 53c0cf9..2fe55eb 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
@@ -241,7 +241,7 @@ public class CastEmptyString${type.from}ToNullable${type.to} implements DrillSim
</#if>
public void setup() {
- int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.integerSize);
+ int size = ${type.arraySize} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE);
buffer = buffer.reallocIfNeeded(size);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
index 8d5f1a3..efa6354 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java
@@ -141,7 +141,7 @@ public class ${type.name}Functions {
@Output ${type.name}Holder result;
public void setup() {
- int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
outputPrecision = Integer.MIN_VALUE;
}
@@ -179,7 +179,7 @@ public class ${type.name}Functions {
@Output ${type.name}Holder result;
public void setup() {
- int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
outputPrecision = Integer.MIN_VALUE;
}
@@ -218,7 +218,7 @@ public class ${type.name}Functions {
@Output ${type.name}Holder result;
public void setup() {
- int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
tempResult = new int[${type.storage} * ${type.storage}];
outputPrecision = Integer.MIN_VALUE;
@@ -387,7 +387,7 @@ public class ${type.name}Functions {
@Workspace int outputPrecision;
public void setup() {
- int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
outputPrecision = Integer.MIN_VALUE;
}
@@ -424,7 +424,7 @@ public class ${type.name}Functions {
@Workspace int outputPrecision;
public void setup() {
- int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.integerSize));
+ int size = (${type.storage} * (org.apache.drill.exec.util.DecimalUtility.INTEGER_SIZE));
buffer = buffer.reallocIfNeeded(size);
outputPrecision = Integer.MIN_VALUE;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
index 133ed3b..3f7b6e6 100644
--- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -35,7 +35,7 @@ import org.apache.drill.exec.vector.accessor.*;
import org.apache.drill.exec.vector.complex.RepeatedMapVector;
import org.apache.drill.exec.util.CallBack;
-public class TypeHelper extends BasicTypeHelper{
+public class TypeHelper extends BasicTypeHelper {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
public static SqlAccessor getSqlAccessor(ValueVector vector){
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
index 4a72719..fd98f59 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountorImpl.java
@@ -207,6 +207,10 @@ public class AccountorImpl implements Accountor {
}
private void addLimitConsumer(LimitConsumer c) {
+ if (c == null) {
+ return;
+ }
+
if (parent != null){
parent.addLimitConsumer(c);
}else {
@@ -229,6 +233,10 @@ public class AccountorImpl implements Accountor {
}
private void removeLimitConsumer(LimitConsumer c) {
+ if (c == null) {
+ return;
+ }
+
if (parent != null){
if (parent.parent==null){
// only fragment level allocators will have the fragment context saved
@@ -308,8 +316,7 @@ public class AccountorImpl implements Accountor {
public void close() {
// remove the fragment context and reset fragment limits whenever an allocator closes
- if(parent!=null && parent.parent==null) {
-
+ if (parent != null && parent.parent == null && limitConsumer != null) {
logger.debug("Fragment " + limitConsumer.getIdentifier() + " accountor being closed");
removeLimitConsumer(limitConsumer);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index 40f8e2b..ba04fa3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -30,6 +30,7 @@ import java.util.Map.Entry;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.exec.testing.ControlsInjector;
import org.apache.drill.exec.testing.ControlsInjectorFactory;
import org.apache.drill.exec.util.AssertionUtil;
@@ -44,7 +45,7 @@ public class TopLevelAllocator implements BufferAllocator {
private static final boolean ENABLE_ACCOUNTING = AssertionUtil.isAssertionsEnabled();
private final Map<ChildAllocator, StackTraceElement[]> childrenMap;
- private final PooledByteBufAllocatorL innerAllocator = PooledByteBufAllocatorL.DEFAULT;
+ private final PooledByteBufAllocatorL innerAllocator;
private final AccountorImpl acct;
private final boolean errorOnLeak;
private final DrillBuf empty;
@@ -52,6 +53,7 @@ public class TopLevelAllocator implements BufferAllocator {
private TopLevelAllocator(DrillConfig config, long maximumAllocation, boolean errorOnLeak){
MAXIMUM_DIRECT_MEMORY = maximumAllocation;
+ innerAllocator = new PooledByteBufAllocatorL(DrillMetrics.getInstance());
this.config=(config!=null) ? config : DrillConfig.create();
this.errorOnLeak = errorOnLeak;
this.acct = new AccountorImpl(config, errorOnLeak, null, null, maximumAllocation, 0, true);
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
index 38f2736..003fc9b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.ops;
import io.netty.buffer.DrillBuf;
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
index 65adc02..76bc3bf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.memory;
import static org.junit.Assert.fail;
+import io.netty.buffer.DrillBuf;
import java.util.Iterator;
import java.util.List;
@@ -28,6 +29,7 @@ import java.util.Properties;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OpProfileDef;
@@ -48,8 +50,6 @@ import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import io.netty.buffer.DrillBuf;
-
public class TestAllocators {
private static final Properties TEST_CONFIGURATIONS = new Properties() {
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/vector/pom.xml
----------------------------------------------------------------------
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 6001944..5534ad4 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -67,6 +67,16 @@
<build>
+
+ <resources>
+ <resource>
+ <!-- Copy freemarker template and fmpp configuration files of Vector's
+ to allow clients to leverage definitions. -->
+ <directory>${basedir}/src/main/codegen</directory>
+ <targetPath>codegen</targetPath>
+ </resource>
+ </resources>
+
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/a8297dfe/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/BasicTypeHelper.java b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
index 723687a..a81265b 100644
--- a/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
+++ b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
@@ -43,11 +43,11 @@ public class BasicTypeHelper {
// a high value like this will not inflate the size for small values
public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
- private static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
+ protected static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
}
- private static String buildErrorMessage(final String operation, final MajorType type) {
+ protected static String buildErrorMessage(final String operation, final MajorType type) {
return buildErrorMessage(operation, type.getMinorType(), type.getMode());
}
[33/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
new file mode 100644
index 0000000..eb1394c
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+import org.apache.drill.common.expression.BooleanOperator;
+import org.apache.drill.common.expression.CastExpression;
+import org.apache.drill.common.expression.ConvertExpression;
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.IfExpression.IfCondition;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.NullExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedNullConstant;
+import org.apache.drill.common.expression.ValueExpressions;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, RuntimeException> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionValidator.class);
+
+ @Override
+ public Void visitFunctionCall(FunctionCall call, ErrorCollector errors) throws RuntimeException {
+ // we throw an exception here because this is a fundamental operator programming problem as opposed to an expression
+ // problem. At this point in an expression's lifecycle, all function calls should have been converted into
+ // FunctionHolders.
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "
+ + "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public Void visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors)
+ throws RuntimeException {
+ // make sure aggregate functions are not nested inside aggregate functions
+ AggregateChecker.isAggregating(holder, errors);
+
+ // make sure arguments are constant if the function implementation expects constants for any arguments
+ ConstantChecker.checkConstants(holder, errors);
+
+ return null;
+ }
+
+ @Override
+ public Void visitBooleanOperator(BooleanOperator op, ErrorCollector errors) throws RuntimeException {
+ int i = 0;
+ for (LogicalExpression arg : op.args) {
+ if ( arg.getMajorType().getMinorType() != MinorType.BIT) {
+ errors
+ .addGeneralError(
+ arg.getPosition(),
+ String
+ .format(
+ "Failure composing boolean operator %s. All conditions must return a boolean type. Condition %d was of Type %s.",
+ op.getName(), i, arg.getMajorType().getMinorType()));
+ }
+ i++;
+ }
+
+ return null;
+ }
+
+ @Override
+ public Void visitIfExpression(IfExpression ifExpr, ErrorCollector errors) throws RuntimeException {
+ // confirm that all conditions are required boolean values.
+ IfCondition cond = ifExpr.ifCondition;
+ MajorType majorType = cond.condition.getMajorType();
+ if ( majorType
+ .getMinorType() != MinorType.BIT) {
+ errors
+ .addGeneralError(
+ cond.condition.getPosition(),
+ String
+ .format(
+ "Failure composing If Expression. All conditions must return a boolean type. Condition was of Type %s.",
+ majorType.getMinorType()));
+ }
+
+ // confirm that all outcomes are the same type.
+ final MajorType mt = ifExpr.elseExpression.getMajorType();
+ cond = ifExpr.ifCondition;
+ MajorType innerT = cond.expression.getMajorType();
+ if ((innerT.getMode() == DataMode.REPEATED && mt.getMode() != DataMode.REPEATED) || //
+ ((innerT.getMinorType() != mt.getMinorType()) &&
+ (innerT.getMode() != DataMode.OPTIONAL && mt.getMode() != DataMode.OPTIONAL &&
+ (innerT.getMinorType() != MinorType.NULL && mt.getMinorType() != MinorType.NULL)))) {
+ errors
+ .addGeneralError(
+ cond.condition.getPosition(),
+ String
+ .format(
+ "Failure composing If Expression. All expressions must return the same MinorType as the else expression. The if condition returned type type %s but the else expression was of type %s",
+ innerT, mt));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitSchemaPath(SchemaPath path, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitIntConstant(ValueExpressions.IntExpression intExpr, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitFloatConstant(ValueExpressions.FloatExpression fExpr, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitLongConstant(LongExpression intExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitDecimal9Constant(Decimal9Expression decExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitDecimal18Constant(Decimal18Expression decExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitDecimal28Constant(Decimal28Expression decExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitDecimal38Constant(Decimal38Expression decExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitDateConstant(DateExpression intExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitTimeConstant(TimeExpression intExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitIntervalYearConstant(IntervalYearExpression intExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitIntervalDayConstant(IntervalDayExpression intExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitTimeStampConstant(TimeStampExpression intExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitBooleanConstant(BooleanExpression e, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitQuotedStringConstant(QuotedString e, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitUnknown(LogicalExpression e, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitCastExpression(CastExpression e, ErrorCollector value) throws RuntimeException {
+ return e.getInput().accept(this, value);
+ }
+
+ @Override
+ public Void visitNullConstant(TypedNullConstant e, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitNullExpression(NullExpression e, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitConvertExpression(ConvertExpression e, ErrorCollector value)
+ throws RuntimeException {
+ return e.getInput().accept(this, value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/OpVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/OpVisitor.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/OpVisitor.java
new file mode 100644
index 0000000..b3c7879
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/OpVisitor.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+import org.apache.drill.common.graph.GraphVisitor;
+import org.apache.drill.common.logical.data.LogicalOperator;
+
+public interface OpVisitor extends GraphVisitor<LogicalOperator> {
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
new file mode 100644
index 0000000..5f83e93
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
+
+public abstract class SimpleExprVisitor<T> implements ExprVisitor<T, Void, RuntimeException>{
+
+ @Override
+ public T visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
+ return visitFunctionCall(call);
+ }
+
+ @Override
+ public T visitFunctionHolderExpression(FunctionHolderExpression holder, Void value) throws RuntimeException {
+ return visitFunctionHolderExpression(holder);
+ }
+
+ @Override
+ public T visitIfExpression(IfExpression ifExpr, Void value) throws RuntimeException {
+ return visitIfExpression(ifExpr);
+ }
+
+ @Override
+ public T visitSchemaPath(SchemaPath path, Void value) throws RuntimeException {
+ return visitSchemaPath(path);
+ }
+
+ @Override
+ public T visitIntConstant(IntExpression intExpr, Void value) throws RuntimeException {
+ return visitIntConstant(intExpr);
+ }
+
+ @Override
+ public T visitFloatConstant(FloatExpression fExpr, Void value) throws RuntimeException {
+ return visitFloatConstant(fExpr);
+ }
+
+ @Override
+ public T visitLongConstant(LongExpression intExpr, Void value) throws RuntimeException {
+ return visitLongConstant(intExpr);
+ }
+
+ @Override
+ public T visitDateConstant(DateExpression intExpr, Void value) throws RuntimeException {
+ return visitDateConstant(intExpr);
+ }
+
+
+ @Override
+ public T visitTimeConstant(TimeExpression intExpr, Void value) throws RuntimeException {
+ return visitTimeConstant(intExpr);
+ }
+
+ @Override
+ public T visitIntervalYearConstant(IntervalYearExpression intExpr, Void value) throws RuntimeException {
+ return visitIntervalYearConstant(intExpr);
+ }
+
+ @Override
+ public T visitIntervalDayConstant(IntervalDayExpression intExpr, Void value) throws RuntimeException {
+ return visitIntervalDayConstant(intExpr);
+ }
+
+ @Override
+ public T visitTimeStampConstant(TimeStampExpression intExpr, Void value) throws RuntimeException {
+ return visitTimeStampConstant(intExpr);
+ }
+
+ @Override
+ public T visitDecimal9Constant(Decimal9Expression decExpr, Void value) throws RuntimeException {
+ return visitDecimal9Constant(decExpr);
+ }
+
+ @Override
+ public T visitDecimal18Constant(Decimal18Expression decExpr, Void value) throws RuntimeException {
+ return visitDecimal18Constant(decExpr);
+ }
+
+ @Override
+ public T visitDecimal28Constant(Decimal28Expression decExpr, Void value) throws RuntimeException {
+ return visitDecimal28Constant(decExpr);
+ }
+
+ @Override
+ public T visitDecimal38Constant(Decimal38Expression decExpr, Void value) throws RuntimeException {
+ return visitDecimal38Constant(decExpr);
+ }
+
+ @Override
+ public T visitDoubleConstant(DoubleExpression dExpr, Void value) throws RuntimeException {
+ return visitDoubleConstant(dExpr);
+ }
+
+ @Override
+ public T visitBooleanConstant(BooleanExpression e, Void value) throws RuntimeException {
+ return visitBooleanConstant(e);
+ }
+
+ @Override
+ public T visitQuotedStringConstant(QuotedString e, Void value) throws RuntimeException {
+ return visitQuotedStringConstant(e);
+ }
+
+
+ public abstract T visitFunctionCall(FunctionCall call);
+ public abstract T visitFunctionHolderExpression(FunctionHolderExpression call);
+ public abstract T visitIfExpression(IfExpression ifExpr);
+ public abstract T visitSchemaPath(SchemaPath path);
+ public abstract T visitIntConstant(IntExpression intExpr);
+ public abstract T visitFloatConstant(FloatExpression fExpr);
+ public abstract T visitLongConstant(LongExpression intExpr);
+ public abstract T visitDateConstant(DateExpression intExpr);
+ public abstract T visitTimeConstant(TimeExpression intExpr);
+ public abstract T visitIntervalYearConstant(IntervalYearExpression intExpr);
+ public abstract T visitIntervalDayConstant(IntervalDayExpression intExpr);
+ public abstract T visitTimeStampConstant(TimeStampExpression intExpr);
+ public abstract T visitDecimal9Constant(Decimal9Expression intExpr);
+ public abstract T visitDecimal18Constant(Decimal18Expression intExpr);
+ public abstract T visitDecimal28Constant(Decimal28Expression intExpr);
+ public abstract T visitDecimal38Constant(Decimal38Expression intExpr);
+ public abstract T visitDoubleConstant(DoubleExpression dExpr);
+ public abstract T visitBooleanConstant(BooleanExpression e);
+ public abstract T visitQuotedStringConstant(QuotedString e);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/expression/visitors/package-info.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/package-info.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/package-info.java
new file mode 100644
index 0000000..b57362e
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/package-info.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Visitors for validating or optimizing logical expressions.
+ *
+ * These visitors are used to provide feedback to users on invalid expressions
+ * as well as some basic optimizations applied during planning.
+ */
+package org.apache.drill.common.expression.visitors;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/AdjacencyList.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/AdjacencyList.java b/logical/src/main/java/org/apache/drill/common/graph/AdjacencyList.java
new file mode 100644
index 0000000..a4edfc1
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/AdjacencyList.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.graph;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Multimaps;
+
+class AdjacencyList<V extends GraphValue<V>> {
+ private Set<Node> allNodes = new HashSet<Node>();
+ private ListMultimap<Node, Edge<Node>> adjacencies = ArrayListMultimap.create();
+
+ void addEdge(Node source, Node target, int weight) {
+ adjacencies.put(source, new Edge<Node>(source, target, weight));
+ allNodes.add(source);
+ allNodes.add(target);
+ }
+
+ void clearVisited() {
+ for (Edge<Node> e : adjacencies.values()) {
+ e.from.visited = false;
+ e.to.visited = false;
+ }
+ }
+
+ Node getNewNode(V value) {
+ return new Node(value);
+ }
+
+ public List<Edge<Node>> getAdjacent(AdjacencyList<V>.Node source) {
+ return adjacencies.get(source);
+ }
+
+ public void printEdges() {
+ for (Edge<Node> e : adjacencies.values()) {
+ System.out.println(e.from.index + " -> " + e.to.index);
+ }
+ }
+
+ public AdjacencyList<V> getReversedList() {
+ AdjacencyList<V> newlist = new AdjacencyList<V>();
+ for (Edge<Node> e : adjacencies.values()) {
+ newlist.addEdge(e.to, e.from, e.weight);
+ }
+ return newlist;
+ }
+
+ public Set<Node> getNodeSet() {
+ return adjacencies.keySet();
+ }
+
+
+ Collection<Node> getInternalLeafNodes() {
+ // we have to use the allNodes list as otherwise destination only nodes won't be found.
+ List<Node> nodes = new LinkedList<Node>(allNodes);
+
+ for (Iterator<Node> i = nodes.iterator(); i.hasNext();) {
+ final Node n = i.next();
+
+ // remove any nodes that have one or more outbound edges.
+ List<Edge<Node>> adjList = this.getAdjacent(n);
+ if (adjList != null && !adjList.isEmpty()) {
+ i.remove();
+ }
+
+ }
+ return nodes;
+ }
+
+ /**
+ * Get a list of nodes that have no outbound edges.
+ *
+ * @return
+ */
+ public Collection<V> getLeafNodes() {
+ return convert(getInternalLeafNodes());
+ }
+
+
+ Collection<Node> getInternalRootNodes() {
+ Set<Node> nodes = new HashSet<Node>(getNodeSet());
+ for (Edge<Node> e : adjacencies.values()) {
+ nodes.remove(e.to);
+ }
+ return nodes;
+ }
+
+ /**
+ * Get a list of all nodes that have no incoming edges.
+ *
+ * @return
+ */
+ public List<V> getRootNodes() {
+ return convert(getInternalRootNodes());
+ }
+
+ public Collection<Edge<Node>> getAllEdges() {
+ return adjacencies.values();
+ }
+
+ public void fix(boolean requireDirected) {
+ adjacencies = Multimaps.unmodifiableListMultimap(adjacencies);
+ allNodes = Collections.unmodifiableSet(allNodes);
+
+ if (requireDirected) {
+ List<List<Node>> cyclicReferences = GraphAlgos.checkDirected(this);
+ if (cyclicReferences.size() > 0) {
+ throw new IllegalArgumentException(
+ "A logical plan must be a valid DAG. You have cyclic references in your graph. " + cyclicReferences);
+ }
+ }
+ }
+
+ List<V> convert(Collection<Node> nodes) {
+ List<V> out = new ArrayList<V>(nodes.size());
+ for (Node o : nodes) {
+ out.add(o.getNodeValue());
+ }
+ return out;
+ }
+
+ class Node implements Comparable<Node> {
+ final V nodeValue;
+ boolean visited = false; // used for Kosaraju's algorithm and Edmonds's
+ // algorithm
+ int lowlink = -1; // used for Tarjan's algorithm
+ int index = -1; // used for Tarjan's algorithm
+
+ public Node(final V operator) {
+ if (operator == null) {
+ throw new IllegalArgumentException("Operator node was null.");
+ }
+ this.nodeValue = operator;
+ }
+
+ public int compareTo(final Node argNode) {
+ // just do an identity compare since elsewhere you should ensure that only one node exists for each nodeValue.
+ return argNode == this ? 0 : -1;
+ }
+
+ @Override
+ public int hashCode() {
+ return nodeValue.hashCode();
+ }
+
+ public V getNodeValue() {
+ return nodeValue;
+ }
+
+ @Override
+ public String toString() {
+ return "Node [val=" + nodeValue + "]";
+ }
+
+ }
+
+ public static <V extends GraphValue<V>> AdjacencyList<V> newInstance(Collection<V> nodes) {
+ AdjacencyList<V> list = new AdjacencyList<V>();
+ AdjacencyListBuilder<V> builder = new AdjacencyListBuilder<V>(list);
+ for (V v : nodes) {
+ v.accept(builder);
+ }
+ return builder.getAdjacencyList();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/AdjacencyListBuilder.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/AdjacencyListBuilder.java b/logical/src/main/java/org/apache/drill/common/graph/AdjacencyListBuilder.java
new file mode 100644
index 0000000..e90d70a
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/AdjacencyListBuilder.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.graph;
+
+import java.util.HashMap;
+import java.util.Map;
+
+ class AdjacencyListBuilder<V extends GraphValue<V>> implements GraphVisitor<V> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AdjacencyListBuilder.class);
+
+ private Map<V, AdjacencyList<V>.Node> ops = new HashMap<V, AdjacencyList<V>.Node>();
+ private final AdjacencyList<V> parent;
+
+ public AdjacencyListBuilder(AdjacencyList<V> parent) {
+ this.parent = parent;
+ }
+
+ protected boolean requireDirected() {
+ return true;
+ }
+
+ public boolean enter(V o) {
+ visit(o);
+ return true;
+ }
+
+ @Override
+ public void leave(V o) {
+ }
+
+ @Override
+ public boolean visit(V o) {
+ if (o == null) {
+ throw new IllegalArgumentException("Null operator.");
+ }
+
+ if (!ops.containsKey(o)) {
+ ops.put(o, parent.getNewNode(o));
+ return true;
+ }
+
+ return true;
+ }
+
+ public AdjacencyList<V> getAdjacencyList() {
+// logger.debug("Values; {}", ops.values().toArray());
+ AdjacencyList<V> a = new AdjacencyList<V>();
+
+ for (AdjacencyList<V>.Node from : ops.values()) {
+ for (V t : from.getNodeValue()) {
+ AdjacencyList<V>.Node to = ops.get(t);
+ a.addEdge(from, to, 0);
+ }
+
+ }
+ a.fix(true);
+ return a;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/Edge.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/Edge.java b/logical/src/main/java/org/apache/drill/common/graph/Edge.java
new file mode 100644
index 0000000..7b9ca61
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/Edge.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.graph;
+
+
+class Edge<N> implements Comparable<Edge<N>> {
+
+ final N from, to;
+ final int weight;
+
+ public Edge(final N argFrom, final N argTo, final int argWeight) {
+ from = argFrom;
+ to = argTo;
+ weight = argWeight;
+ }
+
+ public int compareTo(final Edge<N> argEdge) {
+ return weight - argEdge.weight;
+ }
+
+ @Override
+ public String toString() {
+ return "Edge [from=" + from + ", to=" + to + "]";
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/Graph.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/Graph.java b/logical/src/main/java/org/apache/drill/common/graph/Graph.java
new file mode 100644
index 0000000..3cf4e7a
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/Graph.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.graph;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.drill.common.logical.UnexpectedOperatorType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class Graph<G extends GraphValue<G>, R extends G, T extends G> {
+
+ static final Logger logger = LoggerFactory.getLogger(Graph.class);
+
+ private AdjacencyList<G> adjList;
+ private final List<R> roots;
+ private final List<T> leaves;
+
+ public Graph(List<G> operators, Class<R> root, Class<T> leaf) {
+ adjList = AdjacencyList.newInstance(operators);
+ roots = checkOperatorType(adjList.getRootNodes(), root, String.format("Root nodes must be a subclass of %s.", root.getSimpleName()));
+ leaves = checkOperatorType(adjList.getLeafNodes(), leaf, String.format("Leaf nodes must be a subclass of %s.", leaf.getSimpleName()));
+ }
+
+ @SuppressWarnings("unchecked")
+ private <O extends G> List<O> checkOperatorType(Collection<G> ops, Class<O> classIdentifier, String error){
+ for(G o : ops){
+ if(!classIdentifier.isAssignableFrom(o.getClass())){
+ throw new UnexpectedOperatorType(o, error);
+ }
+ }
+ return (List<O>) ops;
+ }
+
+ public AdjacencyList<G> getAdjList() {
+ return adjList;
+ }
+
+ public Collection<R> getRoots() {
+ return roots;
+ }
+
+ public Collection<T> getLeaves() {
+ return leaves;
+ }
+
+ public static <G extends GraphValue<G>, R extends G, T extends G> Graph<G, R, T> newGraph(List<G> operators, Class<R> root, Class<T> leaf){
+ return new Graph<G, R, T>(operators, root, leaf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/GraphAlgos.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/GraphAlgos.java b/logical/src/main/java/org/apache/drill/common/graph/GraphAlgos.java
new file mode 100644
index 0000000..0b82c7b
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/GraphAlgos.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.graph;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GraphAlgos {
+ static final Logger logger = LoggerFactory.getLogger(GraphAlgos.class);
+
+ public static class TopoSorter<V extends GraphValue<V>> {
+ final List<AdjacencyList<V>.Node> sorted = new LinkedList<AdjacencyList<V>.Node>();
+ final AdjacencyList<V> rGraph;
+
+ private TopoSorter(AdjacencyList<V> graph, boolean reverse) {
+ graph.clearVisited();
+
+ if (reverse) {
+ this.rGraph = graph.getReversedList();
+ } else {
+ this.rGraph = graph;
+ }
+ Collection<AdjacencyList<V>.Node> sourceNodes = rGraph.getInternalRootNodes();
+
+ for (AdjacencyList<V>.Node n : sourceNodes) {
+ visit(n);
+ }
+ }
+
+ private void visit(AdjacencyList<V>.Node n) {
+ if (n.visited) {
+ return;
+ }
+
+ n.visited = true;
+ List<Edge<AdjacencyList<V>.Node>> edges = rGraph.getAdjacent(n);
+ if (edges != null) {
+ for (Edge<AdjacencyList<V>.Node> e : edges) {
+ visit(e.to);
+ }
+ }
+
+ sorted.add(n);
+
+ }
+
+ /**
+ * Execute a depth-first sort on the reversed DAG.
+ *
+ * @param graph
+ * The adjacency list for the DAG.
+ * @param sourceNodes
+ * List of nodes that
+ * @return
+ */
+ static <V extends GraphValue<V>> List<AdjacencyList<V>.Node> sortInternal(AdjacencyList<V> graph, boolean reverse) {
+ TopoSorter<V> ts = new TopoSorter<V>(graph, reverse);
+ return ts.sorted;
+ }
+
+ public static <V extends GraphValue<V>> List<V> sort(Graph<V, ?, ?> graph) {
+ AdjacencyList<V> l = graph.getAdjList();
+ return l.convert(sortInternal(l, true));
+ }
+
+ public static <V extends GraphValue<V>> List<V> sortLogical(Graph<V, ?, ?> graph) {
+ AdjacencyList<V> l = graph.getAdjList();
+ return l.convert(sortInternal(l, false));
+ }
+ }
+
+ static <V extends GraphValue<V>> List<List<AdjacencyList<V>.Node>> checkDirected(AdjacencyList<V> graph) {
+ Tarjan<V> t = new Tarjan<V>();
+ List<List<AdjacencyList<V>.Node>> subgraphs = t.executeTarjan(graph);
+ for (Iterator<List<AdjacencyList<V>.Node>> i = subgraphs.iterator(); i.hasNext();) {
+ List<AdjacencyList<V>.Node> l = i.next();
+ if (l.size() == 1) {
+ i.remove();
+ }
+ }
+ return subgraphs;
+ }
+
+ public static <V extends GraphValue<V>> List<List<AdjacencyList<V>.Node>> checkDirected(Graph<V, ?, ?> graph) {
+ return checkDirected(graph.getAdjList());
+ }
+
+ public static class Tarjan<V extends GraphValue<V>> {
+
+ private int index = 0;
+ private List<AdjacencyList<V>.Node> stack = new LinkedList<AdjacencyList<V>.Node>();
+ private List<List<AdjacencyList<V>.Node>> SCC = new LinkedList<List<AdjacencyList<V>.Node>>();
+
+ public List<List<AdjacencyList<V>.Node>> executeTarjan(AdjacencyList<V> graph) {
+ SCC.clear();
+ index = 0;
+ stack.clear();
+ if (graph != null) {
+ List<AdjacencyList<V>.Node> nodeList = new LinkedList<AdjacencyList<V>.Node>(graph.getNodeSet());
+ for (AdjacencyList<V>.Node node : nodeList) {
+ if (node.index == -1) {
+ tarjan(node, graph);
+ }
+ }
+ }
+ return SCC;
+ }
+
+ private List<List<AdjacencyList<V>.Node>> tarjan(AdjacencyList<V>.Node v, AdjacencyList<V> list) {
+ v.index = index;
+ v.lowlink = index;
+ index++;
+ stack.add(0, v);
+ List<Edge<AdjacencyList<V>.Node>> l = list.getAdjacent(v);
+ if (l != null) {
+ for (Edge<AdjacencyList<V>.Node> e : l) {
+ AdjacencyList<V>.Node n = e.to;
+ if (n.index == -1) {
+ tarjan(n, list);
+ v.lowlink = Math.min(v.lowlink, n.lowlink);
+ } else if (stack.contains(n)) {
+ v.lowlink = Math.min(v.lowlink, n.index);
+ }
+ }
+ }
+ if (v.lowlink == v.index) {
+ AdjacencyList<V>.Node n;
+ List<AdjacencyList<V>.Node> component = new LinkedList<AdjacencyList<V>.Node>();
+ do {
+ n = stack.remove(0);
+ component.add(n);
+ } while (n != v);
+ SCC.add(component);
+ }
+ return SCC;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/GraphValue.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/GraphValue.java b/logical/src/main/java/org/apache/drill/common/graph/GraphValue.java
new file mode 100644
index 0000000..a919008
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/GraphValue.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.graph;
+
+
+public interface GraphValue<T> extends Iterable<T>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GraphValue.class);
+
+ public void accept(GraphVisitor<T> visitor);
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/GraphVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/GraphVisitor.java b/logical/src/main/java/org/apache/drill/common/graph/GraphVisitor.java
new file mode 100644
index 0000000..810b738
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/GraphVisitor.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.graph;
+
+
+public interface GraphVisitor<T> {
+ public boolean enter(T o);
+ public void leave(T o);
+ public boolean visit(T o);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/Visitable.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/Visitable.java b/logical/src/main/java/org/apache/drill/common/graph/Visitable.java
new file mode 100644
index 0000000..c97062e
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/Visitable.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.graph;
+
+public interface Visitable<T extends Visitable<?>> {
+ public void accept(T node);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/graph/package-info.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/graph/package-info.java b/logical/src/main/java/org/apache/drill/common/graph/package-info.java
new file mode 100644
index 0000000..38138d0
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/graph/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Connected-graph representation shared by logical and physical query graphs.
+ */
+package org.apache.drill.common.graph;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java b/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
new file mode 100644
index 0000000..e7d3d1f
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+/**
+ * Interface for defining a Drill format plugin.
+ *
+ * Format plugins are contained within storage plugins to describe particular
+ * data formats available in a given storage system. These formats are not
+ * necessarily tied to how the data is stored. One current use of this
+ * abstraction is describing different file formats like CSV and JSON that
+ * can be stored in a filesystem. Some storage systems like databases may only
+ * have a single format in which the data is actually stored. This interface
+ * enables flexibility for configuring the different data formats that will
+ * live under one storage system. The storage systems themselves are described
+ * in {@see StoragePluginConfig}s.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
+public interface FormatPluginConfig {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfig.class);
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java b/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
new file mode 100644
index 0000000..50de963
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import java.util.Set;
+
+import org.apache.drill.common.scanner.persistence.ScanResult;
+
+
+public abstract class FormatPluginConfigBase implements FormatPluginConfig{
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfigBase.class);
+
+
+ /**
+ * scan for implementations of {@see FormatPlugin}.
+ *
+ * @param classpathScan - Drill configuration object, used to find the packages to scan
+ * @return - list of classes that implement the interface.
+ */
+ public static Set<Class<? extends FormatPluginConfig>> getSubTypes(final ScanResult classpathScan) {
+ final Set<Class<? extends FormatPluginConfig>> pluginClasses = classpathScan.getImplementations(FormatPluginConfig.class);
+ logger.debug("Found {} format plugin configuration classes: {}.", pluginClasses.size(), pluginClasses);
+ return pluginClasses;
+ }
+
+ @Override
+ public abstract boolean equals(Object o);
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/LogicalPlan.java b/logical/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
new file mode 100644
index 0000000..8a4a5ab
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.config.LogicalPlanPersistence;
+import org.apache.drill.common.graph.Graph;
+import org.apache.drill.common.graph.GraphAlgos;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.SinkOperator;
+import org.apache.drill.common.logical.data.SourceOperator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+@JsonPropertyOrder({ "head", "storage", "query" })
+public class LogicalPlan {
+ static final Logger logger = LoggerFactory.getLogger(LogicalPlan.class);
+
+ private final PlanProperties properties;
+ private final Map<String, StoragePluginConfig> storageEngineMap;
+ private final Graph<LogicalOperator, SinkOperator, SourceOperator> graph;
+
+
+ @JsonCreator
+ public LogicalPlan(@JsonProperty("head") PlanProperties head,
+ @JsonProperty("storage") Map<String, StoragePluginConfig> storageEngineMap,
+ @JsonProperty("query") List<LogicalOperator> operators) {
+ this.storageEngineMap = storageEngineMap != null ? storageEngineMap : new HashMap<String, StoragePluginConfig>();
+ this.properties = head;
+ this.graph = Graph.newGraph(operators, SinkOperator.class, SourceOperator.class);
+ }
+
+ @JsonProperty("query")
+ public List<LogicalOperator> getSortedOperators() {
+ return GraphAlgos.TopoSorter.sortLogical(graph);
+ }
+
+ public StoragePluginConfig getStorageEngineConfig(String name) {
+ return storageEngineMap.get(name);
+ }
+
+ @JsonIgnore
+ public Graph<LogicalOperator, SinkOperator, SourceOperator> getGraph() {
+ return graph;
+ }
+
+ @JsonProperty("head")
+ public PlanProperties getProperties() {
+ return properties;
+ }
+
+ @JsonProperty("storage")
+ public Map<String, StoragePluginConfig> getStorageEngines() {
+ return storageEngineMap;
+ }
+
+ public String toJsonString(LogicalPlanPersistence config) throws JsonProcessingException {
+ return config.getMapper().writeValueAsString(this);
+ }
+
+ public String toJsonStringSafe(LogicalPlanPersistence config){
+ try{
+ return toJsonString(config);
+ }catch(JsonProcessingException e){
+ logger.error("Failure while trying to get JSON representation of plan.", e);
+ return "Unable to generate plan.";
+ }
+ }
+
+ /** Parses a logical plan. */
+ public static LogicalPlan parse(LogicalPlanPersistence config, String planString) {
+ ObjectMapper mapper = config.getMapper();
+ try {
+ LogicalPlan plan = mapper.readValue(planString, LogicalPlan.class);
+ return plan;
+ } catch (IOException e) {
+
+ throw new RuntimeException(String.format("Failure while parsing plan: \n %s}", planString), e);
+ }
+ }
+
+ /** Converts a logical plan to a string. (Opposite of {@link #parse}.) */
+ public String unparse(LogicalPlanPersistence config) {
+ try {
+ return config.getMapper().writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static LogicalPlanBuilder builder() {
+ return new LogicalPlanBuilder();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java b/logical/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
new file mode 100644
index 0000000..28f1402
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
@@ -0,0 +1,56 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.common.logical;
+
+import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
+import org.apache.drill.common.logical.PlanProperties.PlanType;
+import org.apache.drill.common.logical.data.LogicalOperator;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * A programmatic builder for logical plans.
+ */
+public class LogicalPlanBuilder {
+ private PlanProperties planProperties;
+ private ImmutableMap.Builder<String, StoragePluginConfig> storageEngines = ImmutableMap.builder();
+ private ImmutableList.Builder<LogicalOperator> operators = ImmutableList.builder();
+
+ public LogicalPlanBuilder planProperties(PlanProperties planProperties) {
+ this.planProperties = planProperties;
+ return this;
+ }
+
+ public LogicalPlanBuilder planProperties(PlanType type, int version, String generatorType, String generatorInfo, ResultMode mode){
+ this.planProperties = PlanProperties.builder().generator(generatorType, generatorInfo).type(type).version(version).resultMode(mode).build();
+ return this;
+ }
+
+ public LogicalPlanBuilder addStorageEngine(String name, StoragePluginConfig config) {
+ this.storageEngines.put(name, config);
+ return this;
+ }
+ public LogicalPlanBuilder addLogicalOperator(LogicalOperator operator) {
+ this.operators.add(operator);
+ return this;
+ }
+ public LogicalPlan build() {
+ return new LogicalPlan(this.planProperties, this.storageEngines.build(), this.operators.build());
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/PlanProperties.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/PlanProperties.java b/logical/src/main/java/org/apache/drill/common/logical/PlanProperties.java
new file mode 100644
index 0000000..ce9603e
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/PlanProperties.java
@@ -0,0 +1,121 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.common.logical;
+
+import org.apache.drill.common.JSONOptions;
+import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Logical plan meta properties.
+ */
+public class PlanProperties {
+ public static enum PlanType {APACHE_DRILL_LOGICAL, APACHE_DRILL_PHYSICAL}
+
+ public PlanType type;
+ public int version;
+ public Generator generator;
+ public ResultMode resultMode;
+ public JSONOptions options;
+ public int queue;
+
+// @JsonInclude(Include.NON_NULL)
+ public static class Generator {
+ public String type;
+ public String info;
+
+ public static enum ResultMode {
+ EXEC, LOGICAL, PHYSICAL;
+ }
+
+ private Generator(@JsonProperty("type") String type, @JsonProperty("info") String info) {
+ this.type = type;
+ this.info = info;
+ }
+ }
+
+ private PlanProperties(@JsonProperty("version") int version,
+ @JsonProperty("generator") Generator generator,
+ @JsonProperty("type") PlanType type,
+ @JsonProperty("mode") ResultMode resultMode,
+ @JsonProperty("options") JSONOptions options,
+ @JsonProperty("queue") int queue
+ ) {
+ this.version = version;
+ this.queue = queue;
+ this.generator = generator;
+ this.type = type;
+ this.resultMode = resultMode == null ? ResultMode.EXEC : resultMode;
+ this.options = options;
+ }
+
+ public static PlanPropertiesBuilder builder() {
+ return new PlanPropertiesBuilder();
+ }
+
+ public static class PlanPropertiesBuilder {
+ private int version;
+ private Generator generator;
+ private PlanType type;
+ private ResultMode mode = ResultMode.EXEC;
+ private JSONOptions options;
+ private int queueNumber = 0;
+
+ public PlanPropertiesBuilder type(PlanType type) {
+ this.type = type;
+ return this;
+ }
+
+ public PlanPropertiesBuilder version(int version) {
+ this.version = version;
+ return this;
+ }
+
+ public PlanPropertiesBuilder generator(String type, String info) {
+ this.generator = new Generator(type, info);
+ return this;
+ }
+
+ public PlanPropertiesBuilder resultMode(ResultMode mode) {
+ this.mode = mode;
+ return this;
+ }
+
+ public PlanPropertiesBuilder queue(int queueNumber) {
+ this.queueNumber = queueNumber;
+ return this;
+ }
+
+ public PlanPropertiesBuilder options(JSONOptions options) {
+ this.options = options;
+ return this;
+ }
+
+ public PlanPropertiesBuilder generator(Generator generator) {
+ this.generator = generator;
+ return this;
+ }
+
+ public PlanProperties build() {
+ return new PlanProperties(version, generator, type, mode, options, queueNumber);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java b/logical/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
new file mode 100644
index 0000000..a4a0f2b
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
+public abstract class StoragePluginConfig{
+
+ private boolean enabled = true;
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ @Override
+ public abstract boolean equals(Object o);
+
+ @Override
+ public abstract int hashCode();
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java b/logical/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
new file mode 100644
index 0000000..6075483
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import java.util.Set;
+
+import org.apache.drill.common.scanner.persistence.ScanResult;
+
+
+public abstract class StoragePluginConfigBase extends StoragePluginConfig {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginConfigBase.class);
+
+ public static Set<Class<? extends StoragePluginConfig>> getSubTypes(final ScanResult classpathScan) {
+ final Set<Class<? extends StoragePluginConfig>> packages = classpathScan.getImplementations(StoragePluginConfig.class);
+ logger.debug("Found {} logical operator classes: {}.", packages.size(), packages);
+ return packages;
+ }
+
+ @Override
+ public abstract boolean equals(Object o);
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/UnexpectedOperatorType.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/UnexpectedOperatorType.java b/logical/src/main/java/org/apache/drill/common/logical/UnexpectedOperatorType.java
new file mode 100644
index 0000000..ea3f4b2
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/UnexpectedOperatorType.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+
+public class UnexpectedOperatorType extends ValidationError{
+
+ public UnexpectedOperatorType(String message){
+ super(message);
+ }
+
+ public UnexpectedOperatorType(Object operator, String message) {
+ super(message + " Received node of type " + operator.getClass().getCanonicalName());
+ }
+
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/ValidationError.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/ValidationError.java b/logical/src/main/java/org/apache/drill/common/logical/ValidationError.java
new file mode 100644
index 0000000..54c9c82
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/ValidationError.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+public class ValidationError extends RuntimeException{
+
+ public ValidationError() {
+ super();
+ }
+
+
+ public ValidationError(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public ValidationError(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ValidationError(String message) {
+ super(message);
+ }
+
+ public ValidationError(Throwable cause) {
+ super(cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java b/logical/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
new file mode 100644
index 0000000..b178cd9
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+
+public abstract class AbstractBuilder<T extends LogicalOperator> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBuilder.class);
+
+ public abstract T build();
+
+ protected LogicalExpression[] aL(List<LogicalExpression> exprs){
+ return exprs.toArray(new LogicalExpression[exprs.size()]);
+ }
+
+ protected FieldReference[] aF(List<FieldReference> exprs){
+ return exprs.toArray(new FieldReference[exprs.size()]);
+ }
+
+ protected NamedExpression[] aN(List<NamedExpression> exprs){
+ return exprs.toArray(new NamedExpression[exprs.size()]);
+ }
+
+ protected Order.Ordering[] aO(List<Order.Ordering> orderings) {
+ return orderings.toArray(new Order.Ordering[orderings.size()]);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java b/logical/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
new file mode 100644
index 0000000..3ed947e
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import com.google.common.base.Preconditions;
+
+public abstract class AbstractSingleBuilder<T extends SingleInputOperator, X extends AbstractSingleBuilder<T, X>> extends AbstractBuilder<T> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSingleBuilder.class);
+
+ private LogicalOperator input;
+
+ @Override
+ public final T build(){
+ Preconditions.checkNotNull(input);
+ T out = internalBuild();
+ out.setInput(input);
+ return out;
+ }
+
+ public X setInput(LogicalOperator input){
+ this.input = input;
+ return (X) this;
+ }
+
+ public abstract T internalBuild();
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Filter.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Filter.java b/logical/src/main/java/org/apache/drill/common/logical/data/Filter.java
new file mode 100644
index 0000000..02737f8
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Filter.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+
+@JsonTypeName("filter")
+public class Filter extends SingleInputOperator {
+ private final LogicalExpression expr;
+
+ @JsonCreator
+ public Filter(@JsonProperty("expr") LogicalExpression expr) {
+ this.expr = expr;
+ }
+
+ public LogicalExpression getExpr() {
+ return expr;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitFilter(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Flatten.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Flatten.java b/logical/src/main/java/org/apache/drill/common/logical/data/Flatten.java
new file mode 100644
index 0000000..803a4b3
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Flatten.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+
+@JsonTypeName("flatten")
+public class Flatten extends SingleInputOperator {
+ private final FieldReference name;
+ private final LogicalExpression expr;
+ private final boolean drop;
+
+ @JsonCreator
+ public Flatten(@JsonProperty("name") FieldReference name, @JsonProperty("expr") LogicalExpression expr,
+ @JsonProperty("drop") boolean drop) {
+ this.name = name;
+ this.expr = expr;
+ this.drop = drop;
+ }
+
+ public FieldReference getName() {
+ return name;
+ }
+
+ public LogicalExpression getExpr() {
+ return expr;
+ }
+
+ public boolean isDrop() {
+ return drop;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitFlatten(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java b/logical/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
new file mode 100644
index 0000000..2913a7e
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+
+@JsonTypeName("groupingaggregate")
+public class GroupingAggregate extends SingleInputOperator{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GroupingAggregate.class);
+
+ private final NamedExpression[] keys;
+ private final NamedExpression[] exprs;
+
+ public GroupingAggregate(@JsonProperty("keys") NamedExpression[] keys, @JsonProperty("exprs") NamedExpression[] exprs) {
+ super();
+ this.keys = keys;
+ this.exprs = exprs;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitGroupingAggregate(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+ public static Builder builder(){
+ return new Builder();
+ }
+
+ public NamedExpression[] getKeys(){
+ return keys;
+ }
+
+ public NamedExpression[] getExprs(){
+ return exprs;
+ }
+
+ public static class Builder extends AbstractSingleBuilder<GroupingAggregate, Builder>{
+ private List<NamedExpression> keys = Lists.newArrayList();
+ private List<NamedExpression> exprs = Lists.newArrayList();
+
+ public Builder addKey(FieldReference ref, LogicalExpression expr){
+ keys.add(new NamedExpression(expr, ref));
+ return this;
+ }
+
+ public Builder addKey(NamedExpression expr){
+ keys.add(expr);
+ return this;
+ }
+
+ public Builder addExpr(NamedExpression expr){
+ exprs.add(expr);
+ return this;
+ }
+
+ public Builder addExpr(FieldReference ref, LogicalExpression expr){
+ exprs.add(new NamedExpression(expr, ref));
+ return this;
+ }
+
+ @Override
+ public GroupingAggregate internalBuild(){
+ GroupingAggregate ga = new GroupingAggregate(aN(keys), aN(exprs));
+ return ga;
+ }
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Join.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Join.java b/logical/src/main/java/org/apache/drill/common/logical/data/Join.java
new file mode 100644
index 0000000..2d7e6e1
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Join.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExpressionParsingException;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+import org.apache.calcite.rel.core.JoinRelType;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+
+@JsonTypeName("join")
+public class Join extends LogicalOperatorBase {
+ private final LogicalOperator left;
+ private final LogicalOperator right;
+ private final JoinRelType type;
+ private final JoinCondition[] conditions;
+
+ public static JoinRelType resolve(String val) {
+ for (JoinRelType jt : JoinRelType.values()) {
+ if (jt.name().equalsIgnoreCase(val)) {
+ return jt;
+ }
+ }
+ throw new ExpressionParsingException(String.format("Unable to determine join type for value '%s'.", val));
+ }
+
+ @JsonCreator
+ public Join(@JsonProperty("left") LogicalOperator left, @JsonProperty("right") LogicalOperator right,
+ @JsonProperty("conditions") JoinCondition[] conditions, @JsonProperty("type") String type) {
+ this(left, right, conditions, resolve(type));
+ }
+
+ @JsonCreator
+ public Join(@JsonProperty("left") LogicalOperator left, @JsonProperty("right") LogicalOperator right, @JsonProperty("conditions")JoinCondition[] conditions, @JsonProperty("type") JoinRelType type) {
+ super();
+ this.conditions = conditions;
+ this.left = left;
+ this.right = right;
+ left.registerAsSubscriber(this);
+ right.registerAsSubscriber(this);
+ this.type = type;
+
+ }
+
+ public LogicalOperator getLeft() {
+ return left;
+ }
+
+ public LogicalOperator getRight() {
+ return right;
+ }
+
+ public JoinCondition[] getConditions() {
+ return conditions;
+ }
+
+ @JsonIgnore
+ public JoinRelType getJoinType() {
+ return type;
+ }
+
+ public String getType() {
+ return type.name();
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitJoin(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.forArray(getLeft(), getRight());
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractBuilder<Join>{
+ private LogicalOperator left;
+ private LogicalOperator right;
+ private JoinRelType type;
+ private List<JoinCondition> conditions = Lists.newArrayList();
+
+ public Builder type(JoinRelType type) {
+ this.type = type;
+ return this;
+ }
+
+ public Builder left(LogicalOperator left) {
+ this.left = left;
+ return this;
+ }
+ public Builder right(LogicalOperator right) {
+ this.right = right;
+ return this;
+ }
+
+ public Builder addCondition(String relationship, LogicalExpression left, LogicalExpression right) {
+ conditions.add(new JoinCondition(relationship, left, right));
+ return this;
+ }
+
+ @Override
+ public Join build() {
+ Preconditions.checkNotNull(left);
+ Preconditions.checkNotNull(right);
+ Preconditions.checkNotNull(type);
+ return new Join(left, right, conditions.toArray(new JoinCondition[conditions.size()]), type);
+ }
+
+ }
+
+}
[05/45] drill git commit: DRILL-3987: (MOVE) Extract key vector,
field reader, complex/field writer classes.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/UnionWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/UnionWriter.java b/exec/vector/src/main/codegen/templates/UnionWriter.java
new file mode 100644
index 0000000..7a123b4
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/UnionWriter.java
@@ -0,0 +1,228 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/UnionWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
+
+ UnionVector data;
+ private MapWriter mapWriter;
+ private UnionListWriter listWriter;
+ private List<BaseWriter> writers = Lists.newArrayList();
+
+ public UnionWriter(BufferAllocator allocator) {
+ super(null);
+ }
+
+ public UnionWriter(UnionVector vector) {
+ super(null);
+ data = vector;
+ }
+
+ public UnionWriter(UnionVector vector, FieldWriter parent) {
+ super(null);
+ data = vector;
+ }
+
+ @Override
+ public void setPosition(int index) {
+ super.setPosition(index);
+ for (BaseWriter writer : writers) {
+ writer.setPosition(index);
+ }
+ }
+
+
+ @Override
+ public void start() {
+ data.getMutator().setType(idx(), MinorType.MAP);
+ getMapWriter().start();
+ }
+
+ @Override
+ public void end() {
+ getMapWriter().end();
+ }
+
+ @Override
+ public void startList() {
+ getListWriter().startList();
+ data.getMutator().setType(idx(), MinorType.LIST);
+ }
+
+ @Override
+ public void endList() {
+ getListWriter().endList();
+ }
+
+ private MapWriter getMapWriter() {
+ if (mapWriter == null) {
+ mapWriter = new SingleMapWriter(data.getMap(), null, true);
+ mapWriter.setPosition(idx());
+ writers.add(mapWriter);
+ }
+ return mapWriter;
+ }
+
+ public MapWriter asMap() {
+ data.getMutator().setType(idx(), MinorType.MAP);
+ return getMapWriter();
+ }
+
+ private ListWriter getListWriter() {
+ if (listWriter == null) {
+ listWriter = new UnionListWriter(data.getList());
+ listWriter.setPosition(idx());
+ writers.add(listWriter);
+ }
+ return listWriter;
+ }
+
+ public ListWriter asList() {
+ data.getMutator().setType(idx(), MinorType.LIST);
+ return getListWriter();
+ }
+
+ <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+
+ <#if !minor.class?starts_with("Decimal")>
+
+ private ${name}Writer ${name?uncap_first}Writer;
+
+ private ${name}Writer get${name}Writer() {
+ if (${uncappedName}Writer == null) {
+ ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null);
+ ${uncappedName}Writer.setPosition(idx());
+ writers.add(${uncappedName}Writer);
+ }
+ return ${uncappedName}Writer;
+ }
+
+ public ${name}Writer as${name}() {
+ data.getMutator().setType(idx(), MinorType.${name?upper_case});
+ return get${name}Writer();
+ }
+
+ @Override
+ public void write(${name}Holder holder) {
+ data.getMutator().setType(idx(), MinorType.${name?upper_case});
+ get${name}Writer().setPosition(idx());
+ get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
+ }
+
+ public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ data.getMutator().setType(idx(), MinorType.${name?upper_case});
+ get${name}Writer().setPosition(idx());
+ get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ }
+ </#if>
+
+ </#list></#list>
+
+ public void writeNull() {
+ }
+
+ @Override
+ public MapWriter map() {
+ data.getMutator().setType(idx(), MinorType.LIST);
+ getListWriter().setPosition(idx());
+ return getListWriter().map();
+ }
+
+ @Override
+ public ListWriter list() {
+ data.getMutator().setType(idx(), MinorType.LIST);
+ getListWriter().setPosition(idx());
+ return getListWriter().list();
+ }
+
+ @Override
+ public ListWriter list(String name) {
+ data.getMutator().setType(idx(), MinorType.MAP);
+ getMapWriter().setPosition(idx());
+ return getMapWriter().list(name);
+ }
+
+ @Override
+ public MapWriter map(String name) {
+ data.getMutator().setType(idx(), MinorType.MAP);
+ getMapWriter().setPosition(idx());
+ return getMapWriter().map(name);
+ }
+
+ <#list vv.types as type><#list type.minor as minor>
+ <#assign lowerName = minor.class?uncap_first />
+ <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+ <#assign upperName = minor.class?upper_case />
+ <#assign capName = minor.class?cap_first />
+ <#if !minor.class?starts_with("Decimal")>
+ @Override
+ public ${capName}Writer ${lowerName}(String name) {
+ data.getMutator().setType(idx(), MinorType.MAP);
+ getMapWriter().setPosition(idx());
+ return getMapWriter().${lowerName}(name);
+ }
+
+ @Override
+ public ${capName}Writer ${lowerName}() {
+ data.getMutator().setType(idx(), MinorType.LIST);
+ getListWriter().setPosition(idx());
+ return getListWriter().${lowerName}();
+ }
+ </#if>
+ </#list></#list>
+
+ @Override
+ public void allocate() {
+ data.allocateNew();
+ }
+
+ @Override
+ public void clear() {
+ data.clear();
+ }
+
+ @Override
+ public void close() throws Exception {
+ data.close();
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return data.getField();
+ }
+
+ @Override
+ public int getValueCapacity() {
+ return data.getValueCapacity();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/ValueHolders.java b/exec/vector/src/main/codegen/templates/ValueHolders.java
new file mode 100644
index 0000000..4151bbe
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/ValueHolders.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+<#list vv.modes as mode>
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign className="${mode.prefix}${minor.class}Holder" />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/holders/${className}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.holders;
+
+<#include "/@includes/vv_imports.ftl" />
+
+public final class ${className} implements ValueHolder{
+
+ public static final MajorType TYPE = Types.${mode.name?lower_case}(MinorType.${minor.class?upper_case});
+
+ public MajorType getType() {return TYPE;}
+
+ <#if mode.name == "Repeated">
+
+ /** The first index (inclusive) into the Vector. **/
+ public int start;
+
+ /** The last index (exclusive) into the Vector. **/
+ public int end;
+
+ /** The Vector holding the actual values. **/
+ public ${minor.class}Vector vector;
+
+ <#else>
+ public static final int WIDTH = ${type.width};
+
+ <#if mode.name == "Optional">public int isSet;</#if>
+ <#assign fields = minor.fields!type.fields />
+ <#list fields as field>
+ public ${field.type} ${field.name};
+ </#list>
+
+ <#if minor.class.startsWith("Decimal")>
+ public static final int maxPrecision = ${minor.maxPrecisionDigits};
+ <#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")>
+ public static final int nDecimalDigits = ${minor.nDecimalDigits};
+
+ public static int getInteger(int index, int start, DrillBuf buffer) {
+ int value = buffer.getInt(start + (index * 4));
+
+ if (index == 0) {
+ /* the first byte contains sign bit, return value without it */
+ <#if minor.class.endsWith("Sparse")>
+ value = (value & 0x7FFFFFFF);
+ <#elseif minor.class.endsWith("Dense")>
+ value = (value & 0x0000007F);
+ </#if>
+ }
+ return value;
+ }
+
+ public static void setInteger(int index, int value, int start, DrillBuf buffer) {
+ buffer.setInt(start + (index * 4), value);
+ }
+
+ public static void setSign(boolean sign, int start, DrillBuf buffer) {
+ // Set MSB to 1 if sign is negative
+ if (sign == true) {
+ int value = getInteger(0, start, buffer);
+ setInteger(0, (value | 0x80000000), start, buffer);
+ }
+ }
+
+ public static boolean getSign(int start, DrillBuf buffer) {
+ return ((buffer.getInt(start) & 0x80000000) != 0);
+ }
+ </#if></#if>
+
+ @Deprecated
+ public int hashCode(){
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * Reason for deprecation is that ValueHolders are potential scalar replacements
+ * and hence we don't want any methods to be invoked on them.
+ */
+ @Deprecated
+ public String toString(){
+ throw new UnsupportedOperationException();
+ }
+ </#if>
+
+
+
+
+}
+
+</#list>
+</#list>
+</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
new file mode 100644
index 0000000..bba39fe
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -0,0 +1,645 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+
+<#if type.major == "VarLen">
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * ${minor.class}Vector implements a vector of variable width values. Elements in the vector
+ * are accessed by position from the logical start of the vector. A fixed width offsetVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * DrillBuf. Size is inferred by adjacent elements.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+ private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
+ private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
+ private static final int MIN_BYTE_COUNT = 4096;
+
+ public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+ private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4));
+ private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
+ private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
+
+ private final Accessor accessor;
+ private final Mutator mutator;
+
+ private final UInt${type.width}Vector.Accessor oAccessor;
+
+ private int allocationSizeInBytes = INITIAL_BYTE_COUNT;
+ private int allocationMonitor = 0;
+
+ public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ this.oAccessor = offsetVector.getAccessor();
+ this.accessor = new Accessor();
+ this.mutator = new Mutator();
+ }
+
+ @Override
+ public FieldReader getReader(){
+ return reader;
+ }
+
+ @Override
+ public int getBufferSize(){
+ if (getAccessor().getValueCount() == 0) {
+ return 0;
+ }
+ return offsetVector.getBufferSize() + data.writerIndex();
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ final int idx = offsetVector.getAccessor().get(valueCount);
+ return offsetVector.getBufferSizeFor(valueCount + 1) + idx;
+ }
+
+ @Override
+ public int getValueCapacity(){
+ return Math.max(offsetVector.getValueCapacity() - 1, 0);
+ }
+
+ @Override
+ public int getByteCapacity(){
+ return data.capacity();
+ }
+
+ @Override
+ public int getCurrentSizeInBytes() {
+ return offsetVector.getAccessor().get(getAccessor().getValueCount());
+ }
+
+ /**
+ * Return the number of bytes contained in the current var len byte vector.
+ * @return
+ */
+ public int getVarByteLength(){
+ final int valueCount = getAccessor().getValueCount();
+ if(valueCount == 0) {
+ return 0;
+ }
+ return offsetVector.getAccessor().get(valueCount);
+ }
+
+ @Override
+ public SerializedField getMetadata() {
+ return getMetadataBuilder() //
+ .addChild(offsetVector.getMetadata())
+ .setValueCount(getAccessor().getValueCount()) //
+ .setBufferLength(getBufferSize()) //
+ .build();
+ }
+
+ @Override
+ public void load(SerializedField metadata, DrillBuf buffer) {
+ // the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
+ final SerializedField offsetField = metadata.getChild(0);
+ offsetVector.load(offsetField, buffer);
+
+ final int capacity = buffer.capacity();
+ final int offsetsLength = offsetField.getBufferLength();
+ data = buffer.slice(offsetsLength, capacity - offsetsLength);
+ data.retain();
+ }
+
+ @Override
+ public void clear() {
+ super.clear();
+ offsetVector.clear();
+ }
+
+ @Override
+ public DrillBuf[] getBuffers(boolean clear) {
+ final DrillBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(false), super.getBuffers(false), DrillBuf.class);
+ if (clear) {
+ // does not make much sense but we have to retain buffers even when clear is set. refactor this interface.
+ for (final DrillBuf buffer:buffers) {
+ buffer.retain(1);
+ }
+ clear();
+ }
+ return buffers;
+ }
+
+ public long getOffsetAddr(){
+ return offsetVector.getBuffer().memoryAddress();
+ }
+
+ public UInt${type.width}Vector getOffsetVector(){
+ return offsetVector;
+ }
+
+ @Override
+ public TransferPair getTransferPair(){
+ return new TransferImpl(getField());
+ }
+
+ @Override
+ public TransferPair getTransferPair(FieldReference ref){
+ return new TransferImpl(getField().withPath(ref));
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new TransferImpl((${minor.class}Vector) to);
+ }
+
+ public void transferTo(${minor.class}Vector target){
+ target.clear();
+ this.offsetVector.transferTo(target.offsetVector);
+ target.data = data;
+ target.data.retain(1);
+ clear();
+ }
+
+ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
+ UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor();
+ final int startPoint = offsetVectorAccessor.get(startIndex);
+ final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint;
+ target.clear();
+ target.offsetVector.allocateNew(length + 1);
+ offsetVectorAccessor = this.offsetVector.getAccessor();
+ final UInt4Vector.Mutator targetOffsetVectorMutator = target.offsetVector.getMutator();
+ for (int i = 0; i < length + 1; i++) {
+ targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
+ }
+ target.data = data.slice(startPoint, sliceLength);
+ target.data.retain(1);
+ target.getMutator().setValueCount(length);
+}
+
+ protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+ final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
+ final int start = fromOffsetVectorAccessor.get(fromIndex);
+ final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
+ final int len = end - start;
+
+ final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
+ from.data.getBytes(start, data, outputStart, len);
+ offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
+ }
+
+ public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+ final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
+ final int start = fromOffsetVectorAccessor.get(fromIndex);
+ final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
+ final int len = end - start;
+ final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
+
+ while(data.capacity() < outputStart + len) {
+ reAlloc();
+ }
+
+ offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len);
+ from.data.getBytes(start, data, outputStart, len);
+ return true;
+ }
+
+ private class TransferImpl implements TransferPair{
+ ${minor.class}Vector to;
+
+ public TransferImpl(MaterializedField field){
+ to = new ${minor.class}Vector(field, allocator);
+ }
+
+ public TransferImpl(${minor.class}Vector to){
+ this.to = to;
+ }
+
+ @Override
+ public ${minor.class}Vector getTo(){
+ return to;
+ }
+
+ @Override
+ public void transfer(){
+ transferTo(to);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ splitAndTransferTo(startIndex, length, to);
+ }
+
+ @Override
+ public void copyValueSafe(int fromIndex, int toIndex) {
+ to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
+ }
+ }
+
+ @Override
+ public void setInitialCapacity(final int valueCount) {
+ final long size = 1L * valueCount * ${type.width};
+ if (size > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+ }
+ allocationSizeInBytes = (int)size;
+ offsetVector.setInitialCapacity(valueCount + 1);
+ }
+
+ @Override
+ public void allocateNew() {
+ if(!allocateNewSafe()){
+ throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
+ }
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ long curAllocationSize = allocationSizeInBytes;
+ if (allocationMonitor > 10) {
+ curAllocationSize = Math.max(MIN_BYTE_COUNT, curAllocationSize / 2);
+ allocationMonitor = 0;
+ } else if (allocationMonitor < -2) {
+ curAllocationSize = curAllocationSize * 2L;
+ allocationMonitor = 0;
+ }
+
+ if (curAllocationSize > MAX_ALLOCATION_SIZE) {
+ return false;
+ }
+
+ clear();
+ /* Boolean to keep track if all the memory allocations were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ try {
+ final int requestedSize = (int)curAllocationSize;
+ data = allocator.buffer(requestedSize);
+ allocationSizeInBytes = requestedSize;
+ offsetVector.allocateNew();
+ } catch (OutOfMemoryRuntimeException e) {
+ clear();
+ return false;
+ }
+ data.readerIndex(0);
+ offsetVector.zeroVector();
+ return true;
+ }
+
+ @Override
+ public void allocateNew(int totalBytes, int valueCount) {
+ clear();
+ assert totalBytes >= 0;
+ try {
+ data = allocator.buffer(totalBytes);
+ offsetVector.allocateNew(valueCount + 1);
+ } catch (DrillRuntimeException e) {
+ clear();
+ throw e;
+ }
+ data.readerIndex(0);
+ allocationSizeInBytes = totalBytes;
+ offsetVector.zeroVector();
+ }
+
+ @Override
+ public void reset() {
+ allocationSizeInBytes = INITIAL_BYTE_COUNT;
+ allocationMonitor = 0;
+ data.readerIndex(0);
+ offsetVector.zeroVector();
+ super.reset();
+ }
+
+ public void reAlloc() {
+ final long newAllocationSize = allocationSizeInBytes*2L;
+ if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
+ }
+
+ final DrillBuf newBuf = allocator.buffer((int)newAllocationSize);
+ newBuf.setBytes(0, data, 0, data.capacity());
+ data.release();
+ data = newBuf;
+ allocationSizeInBytes = (int)newAllocationSize;
+ }
+
+ public void decrementAllocationMonitor() {
+ if (allocationMonitor > 0) {
+ allocationMonitor = 0;
+ }
+ --allocationMonitor;
+ }
+
+ private void incrementAllocationMonitor() {
+ ++allocationMonitor;
+ }
+
+ @Override
+ public Accessor getAccessor(){
+ return accessor;
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor {
+ final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
+ public long getStartEnd(int index){
+ return oAccessor.getTwoAsLong(index);
+ }
+
+ public byte[] get(int index) {
+ assert index >= 0;
+ final int startIdx = oAccessor.get(index);
+ final int length = oAccessor.get(index + 1) - startIdx;
+ assert length >= 0;
+ final byte[] dst = new byte[length];
+ data.getBytes(startIdx, dst, 0, length);
+ return dst;
+ }
+
+ @Override
+ public int getValueLength(int index) {
+ final UInt${type.width}Vector.Accessor offsetVectorAccessor = offsetVector.getAccessor();
+ return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index);
+ }
+
+ public void get(int index, ${minor.class}Holder holder){
+ holder.start = oAccessor.get(index);
+ holder.end = oAccessor.get(index + 1);
+ holder.buffer = data;
+ }
+
+ public void get(int index, Nullable${minor.class}Holder holder){
+ holder.isSet = 1;
+ holder.start = oAccessor.get(index);
+ holder.end = oAccessor.get(index + 1);
+ holder.buffer = data;
+ }
+
+
+ <#switch minor.class>
+ <#case "VarChar">
+ @Override
+ public ${friendlyType} getObject(int index) {
+ Text text = new Text();
+ text.set(get(index));
+ return text;
+ }
+ <#break>
+ <#case "Var16Char">
+ @Override
+ public ${friendlyType} getObject(int index) {
+ return new String(get(index), Charsets.UTF_16);
+ }
+ <#break>
+ <#default>
+ @Override
+ public ${friendlyType} getObject(int index) {
+ return get(index);
+ }
+ </#switch>
+
+ @Override
+ public int getValueCount() {
+ return Math.max(offsetVector.getAccessor().getValueCount()-1, 0);
+ }
+
+ @Override
+ public boolean isNull(int index){
+ return false;
+ }
+
+ public UInt${type.width}Vector getOffsetVector(){
+ return offsetVector;
+ }
+ }
+
+ /**
+ * Mutable${minor.class} implements a vector of variable width values. Elements in the vector
+ * are accessed by position from the logical start of the vector. A fixed width offsetVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * DrillBuf. Size is inferred by adjacent elements.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public final class Mutator extends BaseValueVector.BaseMutator implements VariableWidthVector.VariableWidthMutator {
+
+ /**
+ * Set the variable length element at the specified index to the supplied byte array.
+ *
+ * @param index position of the bit to set
+ * @param bytes array of bytes to write
+ */
+ protected void set(int index, byte[] bytes) {
+ assert index >= 0;
+ final int currentOffset = offsetVector.getAccessor().get(index);
+ offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
+ data.setBytes(currentOffset, bytes, 0, bytes.length);
+ }
+
+ public void setSafe(int index, byte[] bytes) {
+ assert index >= 0;
+
+ final int currentOffset = offsetVector.getAccessor().get(index);
+ while (data.capacity() < currentOffset + bytes.length) {
+ reAlloc();
+ }
+ offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length);
+ data.setBytes(currentOffset, bytes, 0, bytes.length);
+ }
+
+ /**
+ * Set the variable length element at the specified index to the supplied byte array.
+ *
+ * @param index position of the bit to set
+ * @param bytes array of bytes to write
+ * @param start start index of bytes to write
+ * @param length length of bytes to write
+ */
+ protected void set(int index, byte[] bytes, int start, int length) {
+ assert index >= 0;
+ final int currentOffset = offsetVector.getAccessor().get(index);
+ offsetVector.getMutator().set(index + 1, currentOffset + length);
+ data.setBytes(currentOffset, bytes, start, length);
+ }
+
+ public void setSafe(int index, ByteBuffer bytes, int start, int length) {
+ assert index >= 0;
+
+ int currentOffset = offsetVector.getAccessor().get(index);
+
+ while (data.capacity() < currentOffset + length) {
+ reAlloc();
+ }
+ offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
+ data.setBytes(currentOffset, bytes, start, length);
+ }
+
+ public void setSafe(int index, byte[] bytes, int start, int length) {
+ assert index >= 0;
+
+ final int currentOffset = offsetVector.getAccessor().get(index);
+
+ while (data.capacity() < currentOffset + length) {
+ reAlloc();
+ }
+ offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
+ data.setBytes(currentOffset, bytes, start, length);
+ }
+
+ @Override
+ public void setValueLengthSafe(int index, int length) {
+ final int offset = offsetVector.getAccessor().get(index);
+ while(data.capacity() < offset + length ) {
+ reAlloc();
+ }
+ offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length);
+ }
+
+
+ public void setSafe(int index, int start, int end, DrillBuf buffer){
+ final int len = end - start;
+ final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+ while(data.capacity() < outputStart + len) {
+ reAlloc();
+ }
+
+ offsetVector.getMutator().setSafe( index+1, outputStart + len);
+ buffer.getBytes(start, data, outputStart, len);
+ }
+
+ public void setSafe(int index, Nullable${minor.class}Holder holder){
+ assert holder.isSet == 1;
+
+ final int start = holder.start;
+ final int end = holder.end;
+ final int len = end - start;
+
+ int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+ while(data.capacity() < outputStart + len) {
+ reAlloc();
+ }
+
+ holder.buffer.getBytes(start, data, outputStart, len);
+ offsetVector.getMutator().setSafe( index+1, outputStart + len);
+ }
+
+ public void setSafe(int index, ${minor.class}Holder holder){
+ final int start = holder.start;
+ final int end = holder.end;
+ final int len = end - start;
+ final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+ while(data.capacity() < outputStart + len) {
+ reAlloc();
+ }
+
+ holder.buffer.getBytes(start, data, outputStart, len);
+ offsetVector.getMutator().setSafe( index+1, outputStart + len);
+ }
+
+ protected void set(int index, int start, int length, DrillBuf buffer){
+ assert index >= 0;
+ final int currentOffset = offsetVector.getAccessor().get(index);
+ offsetVector.getMutator().set(index + 1, currentOffset + length);
+ final DrillBuf bb = buffer.slice(start, length);
+ data.setBytes(currentOffset, bb);
+ }
+
+ protected void set(int index, Nullable${minor.class}Holder holder){
+ final int length = holder.end - holder.start;
+ final int currentOffset = offsetVector.getAccessor().get(index);
+ offsetVector.getMutator().set(index + 1, currentOffset + length);
+ data.setBytes(currentOffset, holder.buffer, holder.start, length);
+ }
+
+ protected void set(int index, ${minor.class}Holder holder){
+ final int length = holder.end - holder.start;
+ final int currentOffset = offsetVector.getAccessor().get(index);
+ offsetVector.getMutator().set(index + 1, currentOffset + length);
+ data.setBytes(currentOffset, holder.buffer, holder.start, length);
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ final int currentByteCapacity = getByteCapacity();
+ final int idx = offsetVector.getAccessor().get(valueCount);
+ data.writerIndex(idx);
+ if (valueCount > 0 && currentByteCapacity > idx * 2) {
+ incrementAllocationMonitor();
+ } else if (allocationMonitor > 0) {
+ allocationMonitor = 0;
+ }
+ VectorTrimmer.trim(data, idx);
+ offsetVector.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+ }
+
+ @Override
+ public void generateTestData(int size){
+ boolean even = true;
+ <#switch minor.class>
+ <#case "Var16Char">
+ final java.nio.charset.Charset charset = Charsets.UTF_16;
+ <#break>
+ <#case "VarChar">
+ <#default>
+ final java.nio.charset.Charset charset = Charsets.UTF_8;
+ </#switch>
+ final byte[] evenValue = new String("aaaaa").getBytes(charset);
+ final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset);
+ for(int i =0; i < size; i++, even = !even){
+ set(i, even ? evenValue : oddValue);
+ }
+ setValueCount(size);
+ }
+ }
+}
+
+</#if> <#-- type.major -->
+</#list>
+</#list>
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java b/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
new file mode 100644
index 0000000..5e437b2
--- /dev/null
+++ b/exec/vector/src/main/java/io/netty/buffer/DrillBuf.java
@@ -0,0 +1,763 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.nio.charset.Charset;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.drill.exec.memory.Accountor;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.BufferManager;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
+import org.apache.drill.exec.util.AssertionUtil;
+
+import com.google.common.base.Preconditions;
+
+public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillBuf.class);
+
+ private static final boolean BOUNDS_CHECKING_ENABLED = AssertionUtil.BOUNDS_CHECKING_ENABLED;
+
+ private final ByteBuf b;
+ private final long addr;
+ private final int offset;
+ private final boolean rootBuffer;
+ private final AtomicLong rootRefCnt = new AtomicLong(1);
+ private volatile BufferAllocator allocator;
+ private volatile Accountor acct;
+ private volatile int length;
+
+ // TODO - cleanup
+ // The code is partly shared and partly copy-pasted between
+ // these three types. They should be unified under one interface
+ // to share code and to remove the hacky code here to use only
+ // one of these types at a time and use null checks to find out
+ // which.
+ private OperatorContext context;
+ private FragmentContext fContext;
+ private BufferManager bufManager;
+
+ public DrillBuf(BufferAllocator allocator, Accountor a, UnsafeDirectLittleEndian b) {
+ super(b.maxCapacity());
+ this.b = b;
+ this.addr = b.memoryAddress();
+ this.acct = a;
+ this.length = b.capacity();
+ this.offset = 0;
+ this.rootBuffer = true;
+ this.allocator = allocator;
+ }
+
+ private DrillBuf(BufferAllocator allocator, Accountor a) {
+ super(0);
+ this.b = new EmptyByteBuf(allocator.getUnderlyingAllocator()).order(ByteOrder.LITTLE_ENDIAN);
+ this.allocator = allocator;
+ this.acct = a;
+ this.length = 0;
+ this.addr = 0;
+ this.rootBuffer = false;
+ this.offset = 0;
+ }
+
+ /**
+ * Special constructor used for RPC ownership transfer. Takes a snapshot slice of the current buf
+ * but points directly to the underlying UnsafeLittleEndian buffer. Does this by calling unwrap()
+ * twice on the provided DrillBuf and expecting an UnsafeDirectLittleEndian buffer. This operation
+ * includes taking a new reference count on the underlying buffer and maintaining returning with a
+ * current reference count for itself (masking the underlying reference count).
+ * @param allocator
+ * @param a Allocator used when users try to receive allocator from buffer.
+ * @param b Accountor used for accounting purposes.
+ */
+ public DrillBuf(BufferAllocator allocator, Accountor a, DrillBuf b) {
+ this(allocator, a, getUnderlying(b), b, 0, b.length, true);
+ assert b.unwrap().unwrap() instanceof UnsafeDirectLittleEndian;
+ b.unwrap().unwrap().retain();
+ }
+
+
+ private DrillBuf(DrillBuf buffer, int index, int length) {
+ this(buffer.allocator, null, buffer, buffer, index, length, false);
+ }
+
+ private static ByteBuf getUnderlying(DrillBuf b){
+ ByteBuf underlying = b.unwrap().unwrap();
+ return underlying.slice((int) (b.memoryAddress() - underlying.memoryAddress()), b.length);
+ }
+ private DrillBuf(BufferAllocator allocator, Accountor a, ByteBuf replacement, DrillBuf buffer, int index, int length, boolean root) {
+ super(length);
+ if (index < 0 || index > buffer.capacity() - length) {
+ throw new IndexOutOfBoundsException(buffer.toString() + ".slice(" + index + ", " + length + ')');
+ }
+
+ this.length = length;
+ writerIndex(length);
+
+ this.b = replacement;
+ this.addr = buffer.memoryAddress() + index;
+ this.offset = index;
+ this.acct = a;
+ this.length = length;
+ this.rootBuffer = root;
+ this.allocator = allocator;
+ }
+
+ public void setOperatorContext(OperatorContext c) {
+ this.context = c;
+ }
+ public void setFragmentContext(FragmentContext c) {
+ this.fContext = c;
+ }
+
+ public void setBufferManager(BufferManager bufManager) {
+ this.bufManager = bufManager;
+ }
+
+ public BufferAllocator getAllocator() {
+ return allocator;
+ }
+
+ public DrillBuf reallocIfNeeded(int size) {
+ if (this.capacity() >= size) {
+ return this;
+ }
+ if (context != null) {
+ return context.replace(this, size);
+ } else if(fContext != null) {
+ return fContext.replace(this, size);
+ } else if (bufManager != null) {
+ return bufManager.replace(this, size);
+ } else {
+ throw new UnsupportedOperationException("Realloc is only available in the context of an operator's UDFs");
+ }
+
+ }
+
+ @Override
+ public int refCnt() {
+ if(rootBuffer){
+ return (int) this.rootRefCnt.get();
+ }else{
+ return b.refCnt();
+ }
+
+ }
+
+ private long addr(int index) {
+ return addr + index;
+ }
+
+ private final void checkIndexD(int index) {
+ ensureAccessible();
+ if (index < 0 || index >= capacity()) {
+ throw new IndexOutOfBoundsException(String.format(
+ "index: %d (expected: range(0, %d))", index, capacity()));
+ }
+ }
+
+ private final void checkIndexD(int index, int fieldLength) {
+ ensureAccessible();
+ if (fieldLength < 0) {
+ throw new IllegalArgumentException("length: " + fieldLength + " (expected: >= 0)");
+ }
+ if (index < 0 || index > capacity() - fieldLength) {
+ throw new IndexOutOfBoundsException(String.format(
+ "index: %d, length: %d (expected: range(0, %d))", index, fieldLength, capacity()));
+ }
+ }
+
+ /**
+ * Allows a function to determine whether not reading a particular string of bytes is valid.
+ *
+ * Will throw an exception if the memory is not readable for some reason. Only doesn't something in the
+ * case that AssertionUtil.BOUNDS_CHECKING_ENABLED is true.
+ *
+ * @param start The starting position of the bytes to be read.
+ * @param end The exclusive endpoint of the bytes to be read.
+ */
+ public void checkBytes(int start, int end){
+ if (BOUNDS_CHECKING_ENABLED) {
+ checkIndexD(start, end - start);
+ }
+ }
+
+ private void chk(int index, int width) {
+ if (BOUNDS_CHECKING_ENABLED) {
+ checkIndexD(index, width);
+ }
+ }
+
+ private void chk(int index) {
+ if (BOUNDS_CHECKING_ENABLED) {
+ checkIndexD(index);
+ }
+ }
+
+ private void ensure(int width) {
+ if (BOUNDS_CHECKING_ENABLED) {
+ ensureWritable(width);
+ }
+ }
+
+ public boolean transferAccounting(Accountor target) {
+ if (rootBuffer) {
+ boolean outcome = acct.transferTo(target, this, length);
+ acct = target;
+ return outcome;
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public synchronized boolean release() {
+ return release(1);
+ }
+
+ /**
+ * Release the provided number of reference counts. If this is a root buffer, will decrease accounting if the local reference count returns to zero.
+ */
+ @Override
+ public synchronized boolean release(int decrement) {
+
+ if(rootBuffer){
+ final long newRefCnt = this.rootRefCnt.addAndGet(-decrement);
+ Preconditions.checkArgument(newRefCnt > -1, "Buffer has negative reference count.");
+ if (newRefCnt == 0) {
+ b.release(decrement);
+ acct.release(this, length);
+ return true;
+ }else{
+ return false;
+ }
+ }else{
+ return b.release(decrement);
+ }
+ }
+
+ @Override
+ public int capacity() {
+ return length;
+ }
+
+ @Override
+ public synchronized ByteBuf capacity(int newCapacity) {
+ if (rootBuffer) {
+ if (newCapacity == length) {
+ return this;
+ } else if (newCapacity < length) {
+ b.capacity(newCapacity);
+ int diff = length - b.capacity();
+ acct.releasePartial(this, diff);
+ this.length = length - diff;
+ return this;
+ } else {
+ throw new UnsupportedOperationException("Accounting byte buf doesn't support increasing allocations.");
+ }
+ } else {
+ throw new UnsupportedOperationException("Non root bufs doen't support changing allocations.");
+ }
+ }
+
+ @Override
+ public int maxCapacity() {
+ return length;
+ }
+
+ @Override
+ public ByteBufAllocator alloc() {
+ return b.alloc();
+ }
+
+ @Override
+ public ByteOrder order() {
+ return ByteOrder.LITTLE_ENDIAN;
+ }
+
+ @Override
+ public ByteBuf order(ByteOrder endianness) {
+ // if(endianness != ByteOrder.LITTLE_ENDIAN) throw new
+ // UnsupportedOperationException("Drill buffers only support little endian.");
+ return this;
+ }
+
+ @Override
+ public ByteBuf unwrap() {
+ return b;
+ }
+
+ @Override
+ public boolean isDirect() {
+ return true;
+ }
+
+ @Override
+ public ByteBuf readBytes(int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf readSlice(int length) {
+ ByteBuf slice = slice(readerIndex(), length);
+ readerIndex(readerIndex() + length);
+ return slice;
+ }
+
+ @Override
+ public ByteBuf copy() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf copy(int index, int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBuf slice() {
+ return slice(readerIndex(), readableBytes());
+ }
+
+ @Override
+ public DrillBuf slice(int index, int length) {
+ DrillBuf buf = new DrillBuf(this, index, length);
+ buf.writerIndex = length;
+ return buf;
+ }
+
+ @Override
+ public DrillBuf duplicate() {
+ return new DrillBuf(this, 0, length);
+ }
+
+ @Override
+ public int nioBufferCount() {
+ return 1;
+ }
+
+ @Override
+ public ByteBuffer nioBuffer() {
+ return nioBuffer(readerIndex(), readableBytes());
+ }
+
+ @Override
+ public ByteBuffer nioBuffer(int index, int length) {
+ return b.nioBuffer(offset + index, length);
+ }
+
+ @Override
+ public ByteBuffer internalNioBuffer(int index, int length) {
+ return b.internalNioBuffer(offset + index, length);
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers() {
+ return new ByteBuffer[]{nioBuffer()};
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers(int index, int length) {
+ return new ByteBuffer[]{nioBuffer(index, length)};
+ }
+
+ @Override
+ public boolean hasArray() {
+ return b.hasArray();
+ }
+
+ @Override
+ public byte[] array() {
+ return b.array();
+ }
+
+ @Override
+ public int arrayOffset() {
+ return b.arrayOffset();
+ }
+
+ @Override
+ public boolean hasMemoryAddress() {
+ return true;
+ }
+
+ @Override
+ public long memoryAddress() {
+ return this.addr;
+ }
+
+ @Override
+ public String toString(Charset charset) {
+ return toString(readerIndex, readableBytes(), charset);
+ }
+
+ @Override
+ public String toString(int index, int length, Charset charset) {
+ if (length == 0) {
+ return "";
+ }
+
+ ByteBuffer nioBuffer;
+ if (nioBufferCount() == 1) {
+ nioBuffer = nioBuffer(index, length);
+ } else {
+ nioBuffer = ByteBuffer.allocate(length);
+ getBytes(index, nioBuffer);
+ nioBuffer.flip();
+ }
+
+ return ByteBufUtil.decodeString(nioBuffer, charset);
+ }
+
+ @Override
+ public int hashCode() {
+ return System.identityHashCode(this);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // identity equals only.
+ return this == obj;
+ }
+
+ @Override
+ public synchronized ByteBuf retain(int increment) {
+ if(rootBuffer){
+ this.rootRefCnt.addAndGet(increment);
+ }else{
+ b.retain(increment);
+ }
+ return this;
+ }
+
+ @Override
+ public ByteBuf retain() {
+ return retain(1);
+ }
+
+ @Override
+ public long getLong(int index) {
+ chk(index, 8);
+ long v = PlatformDependent.getLong(addr(index));
+ return v;
+ }
+
+ @Override
+ public float getFloat(int index) {
+ return Float.intBitsToFloat(getInt(index));
+ }
+
+ @Override
+ public double getDouble(int index) {
+ return Double.longBitsToDouble(getLong(index));
+ }
+
+ @Override
+ public char getChar(int index) {
+ return (char) getShort(index);
+ }
+
+ @Override
+ public long getUnsignedInt(int index) {
+ return getInt(index) & 0xFFFFFFFFL;
+ }
+
+ @Override
+ public int getInt(int index) {
+ chk(index, 4);
+ int v = PlatformDependent.getInt(addr(index));
+ return v;
+ }
+
+ @Override
+ public int getUnsignedShort(int index) {
+ return getShort(index) & 0xFFFF;
+ }
+
+ @Override
+ public short getShort(int index) {
+ chk(index, 2);
+ short v = PlatformDependent.getShort(addr(index));
+ return v;
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ chk(index, 2);
+ PlatformDependent.putShort(addr(index), (short) value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ chk(index, 4);
+ PlatformDependent.putInt(addr(index), value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ chk(index, 8);
+ PlatformDependent.putLong(addr(index), value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setChar(int index, int value) {
+ chk(index, 2);
+ PlatformDependent.putShort(addr(index), (short) value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setFloat(int index, float value) {
+ chk(index, 4);
+ PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf setDouble(int index, double value) {
+ chk(index, 8);
+ PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeShort(int value) {
+ ensure(2);
+ PlatformDependent.putShort(addr(writerIndex), (short) value);
+ writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeInt(int value) {
+ ensure(4);
+ PlatformDependent.putInt(addr(writerIndex), value);
+ writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeLong(long value) {
+ ensure(8);
+ PlatformDependent.putLong(addr(writerIndex), value);
+ writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeChar(int value) {
+ ensure(2);
+ PlatformDependent.putShort(addr(writerIndex), (short) value);
+ writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeFloat(float value) {
+ ensure(4);
+ PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value));
+ writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeDouble(double value) {
+ ensure(8);
+ PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value));
+ writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+ b.getBytes(index + offset, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuffer dst) {
+ b.getBytes(index + offset, dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ chk(index, 1);
+ PlatformDependent.putByte(addr(index), (byte) value);
+ return this;
+ }
+
+ public void setByte(int index, byte b){
+ chk(index, 1);
+ PlatformDependent.putByte(addr(index), b);
+ }
+
+ public void writeByteUnsafe(byte b){
+ PlatformDependent.putByte(addr(readerIndex), b);
+ readerIndex++;
+ }
+
+ @Override
+ protected byte _getByte(int index) {
+ return getByte(index);
+ }
+
+ @Override
+ protected short _getShort(int index) {
+ return getShort(index);
+ }
+
+ @Override
+ protected int _getInt(int index) {
+ return getInt(index);
+ }
+
+ @Override
+ protected long _getLong(int index) {
+ return getLong(index);
+ }
+
+ @Override
+ protected void _setByte(int index, int value) {
+ setByte(index, value);
+ }
+
+ @Override
+ protected void _setShort(int index, int value) {
+ setShort(index, value);
+ }
+
+ @Override
+ protected void _setMedium(int index, int value) {
+ setMedium(index, value);
+ }
+
+ @Override
+ protected void _setInt(int index, int value) {
+ setInt(index, value);
+ }
+
+ @Override
+ protected void _setLong(int index, long value) {
+ setLong(index, value);
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+ b.getBytes(index + offset, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException {
+ b.getBytes(index + offset, out, length);
+ return this;
+ }
+
+ @Override
+ protected int _getUnsignedMedium(int index) {
+ long addr = addr(index);
+ return (PlatformDependent.getByte(addr) & 0xff) << 16 |
+ (PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
+ PlatformDependent.getByte(addr + 2) & 0xff;
+ }
+
+ @Override
+ public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
+ return b.getBytes(index + offset, out, length);
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+ b.setBytes(index + offset, src, srcIndex, length);
+ return this;
+ }
+
+ public ByteBuf setBytes(int index, ByteBuffer src, int srcIndex, int length) {
+ if (src.isDirect()) {
+ checkIndex(index, length);
+ PlatformDependent.copyMemory(PlatformDependent.directBufferAddress(src) + srcIndex, this.memoryAddress() + index,
+ length);
+ } else {
+ if (srcIndex == 0 && src.capacity() == length) {
+ b.setBytes(index + offset, src);
+ } else {
+ ByteBuffer newBuf = src.duplicate();
+ newBuf.position(srcIndex);
+ newBuf.limit(srcIndex + length);
+ b.setBytes(index + offset, src);
+ }
+ }
+
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+ b.setBytes(index + offset, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuffer src) {
+ b.setBytes(index + offset, src);
+ return this;
+ }
+
+ @Override
+ public int setBytes(int index, InputStream in, int length) throws IOException {
+ return b.setBytes(index + offset, in, length);
+ }
+
+ @Override
+ public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException {
+ return b.setBytes(index + offset, in, length);
+ }
+
+ @Override
+ public byte getByte(int index) {
+ chk(index, 1);
+ return PlatformDependent.getByte(addr(index));
+ }
+
+ public static DrillBuf getEmpty(BufferAllocator allocator, Accountor a) {
+ return new DrillBuf(allocator, a);
+ }
+
+ public boolean isRootBuffer() {
+ return rootBuffer;
+ }
+
+ @Override
+ public void close() {
+ release();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java b/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
new file mode 100644
index 0000000..721aff9
--- /dev/null
+++ b/exec/vector/src/main/java/io/netty/buffer/FakeAllocator.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import org.apache.drill.exec.memory.Accountor;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.util.Pointer;
+
+class FakeAllocator implements BufferAllocator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FakeAllocator.class);
+
+
+ public static final Accountor FAKE_ACCOUNTOR = new FakeAccountor();
+ public static final BufferAllocator FAKE_ALLOCATOR = new FakeAllocator();
+
+ @Override
+ public DrillBuf buffer(int size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DrillBuf buffer(int minSize, int maxSize) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ByteBufAllocator getUnderlyingAllocator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BufferAllocator getChildAllocator(FragmentContext context, long initialReservation, long maximumReservation,
+ boolean applyFragmentLimit)
+ throws OutOfMemoryException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DrillBuf getEmpty() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public PreAllocator getNewPreAllocator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void resetFragmentLimits() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setFragmentLimit(long l) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getFragmentLimit(){
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public long getAllocatedMemory() {
+ return 0;
+ }
+
+ @Override
+ public long getPeakMemoryAllocation() {
+ return 0;
+ }
+
+ static class FakeAccountor extends Accountor {
+
+ public FakeAccountor() {
+ super(null, false, null, null, 0, 0, true);
+ }
+
+ @Override
+ public boolean transferTo(Accountor target, DrillBuf buf, long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getAvailable() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getCapacity() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getAllocation() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean reserve(long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean forceAdditionalReservation(long size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void reserved(long expected, DrillBuf buf) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void releasePartial(DrillBuf buf, long size) {
+
+ }
+
+ @Override
+ public void release(DrillBuf buf, long size) {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+
+ }
+
+ @Override
+ public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut) {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/io/netty/buffer/LargeBuffer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/LargeBuffer.java b/exec/vector/src/main/java/io/netty/buffer/LargeBuffer.java
new file mode 100644
index 0000000..f1d4842
--- /dev/null
+++ b/exec/vector/src/main/java/io/netty/buffer/LargeBuffer.java
@@ -0,0 +1,350 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This is basically a complete copy of DuplicatedByteBuf. We copy because we can't override the release methods to keep
+ * global track of created Large Buffers.
+ */
+public class LargeBuffer extends AbstractByteBuf {
+
+ private final AtomicLong hugeBufferSize;
+ private final AtomicLong hugeBufferCount;
+
+ @Override
+ public ByteBuffer nioBuffer(int index, int length) {
+ return unwrap().nioBuffer(index, length);
+ }
+
+ private final ByteBuf buffer;
+ private final int initCap;
+
+ public LargeBuffer(ByteBuf buffer, AtomicLong hugeBufferSize, AtomicLong hugeBufferCount) {
+ super(buffer.maxCapacity());
+ initCap = buffer.capacity();
+ this.hugeBufferCount = hugeBufferCount;
+ this.hugeBufferSize = hugeBufferSize;
+
+ if (buffer instanceof LargeBuffer) {
+ this.buffer = ((LargeBuffer) buffer).buffer;
+ } else {
+ this.buffer = buffer;
+ }
+
+ setIndex(buffer.readerIndex(), buffer.writerIndex());
+ }
+
+ @Override
+ public ByteBuf unwrap() {
+ return buffer;
+ }
+
+ @Override
+ public ByteBufAllocator alloc() {
+ return buffer.alloc();
+ }
+
+ @Override
+ public ByteOrder order() {
+ return buffer.order();
+ }
+
+ @Override
+ public boolean isDirect() {
+ return buffer.isDirect();
+ }
+
+ @Override
+ public int capacity() {
+ return buffer.capacity();
+ }
+
+ @Override
+ public ByteBuf capacity(int newCapacity) {
+ buffer.capacity(newCapacity);
+ return this;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return buffer.hasArray();
+ }
+
+ @Override
+ public byte[] array() {
+ return buffer.array();
+ }
+
+ @Override
+ public int arrayOffset() {
+ return buffer.arrayOffset();
+ }
+
+ @Override
+ public boolean hasMemoryAddress() {
+ return buffer.hasMemoryAddress();
+ }
+
+ @Override
+ public long memoryAddress() {
+ return buffer.memoryAddress();
+ }
+
+ @Override
+ public byte getByte(int index) {
+ return _getByte(index);
+ }
+
+ @Override
+ protected byte _getByte(int index) {
+ return buffer.getByte(index);
+ }
+
+ @Override
+ public short getShort(int index) {
+ return _getShort(index);
+ }
+
+ @Override
+ protected short _getShort(int index) {
+ return buffer.getShort(index);
+ }
+
+ @Override
+ public int getUnsignedMedium(int index) {
+ return _getUnsignedMedium(index);
+ }
+
+ @Override
+ protected int _getUnsignedMedium(int index) {
+ return buffer.getUnsignedMedium(index);
+ }
+
+ @Override
+ public int getInt(int index) {
+ return _getInt(index);
+ }
+
+ @Override
+ protected int _getInt(int index) {
+ return buffer.getInt(index);
+ }
+
+ @Override
+ public long getLong(int index) {
+ return _getLong(index);
+ }
+
+ @Override
+ protected long _getLong(int index) {
+ return buffer.getLong(index);
+ }
+
+ @Override
+ public ByteBuf copy(int index, int length) {
+ return new LargeBuffer(buffer.copy(index, length), hugeBufferSize, hugeBufferCount);
+ }
+
+ @Override
+ public ByteBuf slice(int index, int length) {
+ return new SlicedByteBuf(this, index, length);
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
+ buffer.getBytes(index, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) {
+ buffer.getBytes(index, dst, dstIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, ByteBuffer dst) {
+ buffer.getBytes(index, dst);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ _setByte(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setByte(int index, int value) {
+ buffer.setByte(index, value);
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ _setShort(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setShort(int index, int value) {
+ buffer.setShort(index, value);
+ }
+
+ @Override
+ public ByteBuf setMedium(int index, int value) {
+ _setMedium(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setMedium(int index, int value) {
+ buffer.setMedium(index, value);
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ _setInt(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setInt(int index, int value) {
+ buffer.setInt(index, value);
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ _setLong(index, value);
+ return this;
+ }
+
+ @Override
+ protected void _setLong(int index, long value) {
+ buffer.setLong(index, value);
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
+ buffer.setBytes(index, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) {
+ buffer.setBytes(index, src, srcIndex, length);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setBytes(int index, ByteBuffer src) {
+ buffer.setBytes(index, src);
+ return this;
+ }
+
+ @Override
+ public ByteBuf getBytes(int index, OutputStream out, int length)
+ throws IOException {
+ buffer.getBytes(index, out, length);
+ return this;
+ }
+
+ @Override
+ public int getBytes(int index, GatheringByteChannel out, int length)
+ throws IOException {
+ return buffer.getBytes(index, out, length);
+ }
+
+ @Override
+ public int setBytes(int index, InputStream in, int length)
+ throws IOException {
+ return buffer.setBytes(index, in, length);
+ }
+
+ @Override
+ public int setBytes(int index, ScatteringByteChannel in, int length)
+ throws IOException {
+ return buffer.setBytes(index, in, length);
+ }
+
+ @Override
+ public int nioBufferCount() {
+ return buffer.nioBufferCount();
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers(int index, int length) {
+ return buffer.nioBuffers(index, length);
+ }
+
+ @Override
+ public ByteBuffer internalNioBuffer(int index, int length) {
+ return nioBuffer(index, length);
+ }
+
+ @Override
+ public int forEachByte(int index, int length, ByteBufProcessor processor) {
+ return buffer.forEachByte(index, length, processor);
+ }
+
+ @Override
+ public int forEachByteDesc(int index, int length, ByteBufProcessor processor) {
+ return buffer.forEachByteDesc(index, length, processor);
+ }
+
+ @Override
+ public final int refCnt() {
+ return unwrap().refCnt();
+ }
+
+ @Override
+ public final ByteBuf retain() {
+ unwrap().retain();
+ return this;
+ }
+
+ @Override
+ public final ByteBuf retain(int increment) {
+ unwrap().retain(increment);
+ return this;
+ }
+
+ @Override
+ public boolean release() {
+ return release(1);
+ }
+
+ @Override
+ public boolean release(int decrement) {
+ boolean released = unwrap().release(decrement);
+ if (released) {
+ hugeBufferSize.addAndGet(-initCap);
+ hugeBufferCount.decrementAndGet();
+ }
+ return released;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
new file mode 100644
index 0000000..2ca79f0
--- /dev/null
+++ b/exec/vector/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.netty.buffer;
+
+import io.netty.util.internal.StringUtil;
+
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.metrics.DrillMetrics;
+import org.apache.drill.exec.util.AssertionUtil;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+
+public class PooledByteBufAllocatorL extends PooledByteBufAllocator{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PooledByteBufAllocatorL.class);
+
+ private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
+ private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
+
+ private static final String METRIC_PREFIX = "drill.allocator.";
+ public static final PooledByteBufAllocatorL DEFAULT = new PooledByteBufAllocatorL();
+
+ private final MetricRegistry registry = DrillMetrics.getInstance();
+ private final AtomicLong hugeBufferSize = new AtomicLong(0);
+ private final AtomicLong hugeBufferCount = new AtomicLong(0);
+ private final AtomicLong normalBufferSize = new AtomicLong(0);
+ private final AtomicLong normalBufferCount = new AtomicLong(0);
+
+ private final PoolArena<ByteBuffer>[] directArenas;
+ private final MemoryStatusThread statusThread;
+ private final Histogram largeBuffersHist;
+ private final Histogram normalBuffersHist;
+
+ private PooledByteBufAllocatorL() {
+ super(true);
+ try {
+ Field f = PooledByteBufAllocator.class.getDeclaredField("directArenas");
+ f.setAccessible(true);
+ this.directArenas = (PoolArena<ByteBuffer>[]) f.get(this);
+ } catch (Exception e) {
+ throw new RuntimeException("Failure while initializing allocator. Unable to retrieve direct arenas field.", e);
+ }
+
+ if (memoryLogger.isTraceEnabled()) {
+ statusThread = new MemoryStatusThread();
+ statusThread.start();
+ } else {
+ statusThread = null;
+ }
+ removeOldMetrics();
+
+ registry.register(METRIC_PREFIX + "normal.size", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return normalBufferSize.get();
+ }
+ });
+
+ registry.register(METRIC_PREFIX + "normal.count", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return normalBufferCount.get();
+ }
+ });
+
+ registry.register(METRIC_PREFIX + "huge.size", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return hugeBufferSize.get();
+ }
+ });
+
+ registry.register(METRIC_PREFIX + "huge.count", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return hugeBufferCount.get();
+ }
+ });
+
+ largeBuffersHist = registry.histogram(METRIC_PREFIX + "huge.hist");
+ normalBuffersHist = registry.histogram(METRIC_PREFIX + "normal.hist");
+
+ }
+
+ private synchronized void removeOldMetrics() {
+ registry.removeMatching(new MetricFilter() {
+ @Override
+ public boolean matches(String name, Metric metric) {
+ return name.startsWith("drill.allocator.");
+ }
+ });
+ }
+
+ @Override
+ protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
+ throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+ }
+
+ @Override
+ protected UnsafeDirectLittleEndian newDirectBuffer(int initialCapacity, int maxCapacity) {
+ PoolThreadCache cache = threadCache.get();
+ PoolArena<ByteBuffer> directArena = cache.directArena;
+
+ if (directArena != null) {
+
+ if (initialCapacity > directArena.chunkSize) {
+ // This is beyond chunk size so we'll allocate separately.
+ ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity);
+
+ hugeBufferCount.incrementAndGet();
+ hugeBufferSize.addAndGet(buf.capacity());
+ largeBuffersHist.update(buf.capacity());
+ // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
+ return new UnsafeDirectLittleEndian(new LargeBuffer(buf, hugeBufferSize, hugeBufferCount));
+
+ } else {
+ // within chunk, use arena.
+ ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity);
+ if (!(buf instanceof PooledUnsafeDirectByteBuf)) {
+ fail();
+ }
+
+ normalBuffersHist.update(buf.capacity());
+ if (AssertionUtil.ASSERT_ENABLED) {
+ normalBufferSize.addAndGet(buf.capacity());
+ normalBufferCount.incrementAndGet();
+ }
+
+ return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount, normalBufferSize);
+ }
+
+ } else {
+ throw fail();
+ }
+ }
+
+ private UnsupportedOperationException fail() {
+ return new UnsupportedOperationException(
+ "Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
+ }
+
+
+ @Override
+ public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
+ if (initialCapacity == 0 && maxCapacity == 0) {
+ newDirectBuffer(initialCapacity, maxCapacity);
+ }
+ validate(initialCapacity, maxCapacity);
+ return newDirectBuffer(initialCapacity, maxCapacity);
+ }
+
+ @Override
+ public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
+ throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+ }
+
+
+ private static void validate(int initialCapacity, int maxCapacity) {
+ if (initialCapacity < 0) {
+ throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)");
+ }
+ if (initialCapacity > maxCapacity) {
+ throw new IllegalArgumentException(String.format(
+ "initialCapacity: %d (expected: not greater than maxCapacity(%d)",
+ initialCapacity, maxCapacity));
+ }
+ }
+
+ private class MemoryStatusThread extends Thread {
+
+ public MemoryStatusThread() {
+ super("memory-status-logger");
+ this.setDaemon(true);
+ this.setName("allocation.logger");
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString());
+ try {
+ Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000);
+ } catch (InterruptedException e) {
+ return;
+ }
+
+ }
+ }
+
+ }
+
+ public void checkAndReset() {
+ if (hugeBufferCount.get() != 0 || normalBufferCount.get() != 0) {
+ StringBuilder buf = new StringBuilder();
+ buf.append("Large buffers outstanding: ");
+ buf.append(hugeBufferCount.get());
+ buf.append(" totaling ");
+ buf.append(hugeBufferSize.get());
+ buf.append(" bytes.");
+ buf.append('\n');
+ buf.append("Normal buffers outstanding: ");
+ buf.append(normalBufferCount.get());
+ buf.append(" totaling ");
+ buf.append(normalBufferSize.get());
+ buf.append(" bytes.");
+ hugeBufferCount.set(0);
+ normalBufferCount.set(0);
+ hugeBufferSize.set(0);
+ normalBufferSize.set(0);
+ throw new DrillRuntimeException(buf.toString());
+ }
+ }
+
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append(directArenas.length);
+ buf.append(" direct arena(s):");
+ buf.append(StringUtil.NEWLINE);
+ for (PoolArena<ByteBuffer> a : directArenas) {
+ buf.append(a);
+ }
+
+ buf.append("Large buffers outstanding: ");
+ buf.append(this.hugeBufferCount.get());
+ buf.append(" totaling ");
+ buf.append(this.hugeBufferSize.get());
+ buf.append(" bytes.");
+ buf.append('\n');
+ buf.append("Normal buffers outstanding: ");
+ buf.append(this.normalBufferCount.get());
+ buf.append(" totaling ");
+ buf.append(this.normalBufferSize.get());
+ buf.append(" bytes.");
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/vector/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
new file mode 100644
index 0000000..e332b13
--- /dev/null
+++ b/exec/vector/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.netty.buffer;
+
+import io.netty.util.internal.PlatformDependent;
+
+import java.nio.ByteOrder;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.drill.exec.util.AssertionUtil;
+
+public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
+ private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
+ private final AbstractByteBuf wrapped;
+ private final long memoryAddress;
+ private AtomicLong bufferCount;
+ private AtomicLong bufferSize;
+ private long initCap = -1;
+
+ UnsafeDirectLittleEndian(LargeBuffer buf) {
+ this(buf, true);
+ }
+
+ UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong bufferCount, AtomicLong bufferSize) {
+ this(buf, true);
+ this.bufferCount = bufferCount;
+ this.bufferSize = bufferSize;
+
+ // initCap is used if we're tracking memory release. If we're in non-debug mode, we'll skip this.
+ this.initCap = AssertionUtil.ASSERT_ENABLED ? capacity() : -1;
+ }
+
+ private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake) {
+ super(buf);
+ if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
+ throw new IllegalStateException("Drill only runs on LittleEndian systems.");
+ }
+ wrapped = buf;
+ this.memoryAddress = buf.memoryAddress();
+ }
+ private long addr(int index) {
+ return memoryAddress + index;
+ }
+
+ @Override
+ public long getLong(int index) {
+// wrapped.checkIndex(index, 8);
+ long v = PlatformDependent.getLong(addr(index));
+ return v;
+ }
+
+ @Override
+ public float getFloat(int index) {
+ return Float.intBitsToFloat(getInt(index));
+ }
+
+ @Override
+ public ByteBuf slice() {
+ return slice(this.readerIndex(), readableBytes());
+ }
+
+ @Override
+ public ByteBuf slice(int index, int length) {
+ return new SlicedByteBuf(this, index, length);
+ }
+
+ @Override
+ public ByteOrder order() {
+ return ByteOrder.LITTLE_ENDIAN;
+ }
+
+ @Override
+ public ByteBuf order(ByteOrder endianness) {
+ return this;
+ }
+
+ @Override
+ public double getDouble(int index) {
+ return Double.longBitsToDouble(getLong(index));
+ }
+
+ @Override
+ public char getChar(int index) {
+ return (char) getShort(index);
+ }
+
+ @Override
+ public long getUnsignedInt(int index) {
+ return getInt(index) & 0xFFFFFFFFL;
+ }
+
+ @Override
+ public int getInt(int index) {
+// wrapped.checkIndex(index, 4);
+ int v = PlatformDependent.getInt(addr(index));
+ return v;
+ }
+
+ @Override
+ public int getUnsignedShort(int index) {
+ return getShort(index) & 0xFFFF;
+ }
+
+ @Override
+ public short getShort(int index) {
+// wrapped.checkIndex(index, 2);
+ short v = PlatformDependent.getShort(addr(index));
+ return v;
+ }
+
+ @Override
+ public ByteBuf setShort(int index, int value) {
+ wrapped.checkIndex(index, 2);
+ _setShort(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setInt(int index, int value) {
+ wrapped.checkIndex(index, 4);
+ _setInt(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setLong(int index, long value) {
+ wrapped.checkIndex(index, 8);
+ _setLong(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setChar(int index, int value) {
+ setShort(index, value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf setFloat(int index, float value) {
+ setInt(index, Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf setDouble(int index, double value) {
+ setLong(index, Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeShort(int value) {
+ wrapped.ensureWritable(2);
+ _setShort(wrapped.writerIndex, value);
+ wrapped.writerIndex += 2;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeInt(int value) {
+ wrapped.ensureWritable(4);
+ _setInt(wrapped.writerIndex, value);
+ wrapped.writerIndex += 4;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeLong(long value) {
+ wrapped.ensureWritable(8);
+ _setLong(wrapped.writerIndex, value);
+ wrapped.writerIndex += 8;
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeChar(int value) {
+ writeShort(value);
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeFloat(float value) {
+ writeInt(Float.floatToRawIntBits(value));
+ return this;
+ }
+
+ @Override
+ public ByteBuf writeDouble(double value) {
+ writeLong(Double.doubleToRawLongBits(value));
+ return this;
+ }
+
+ private void _setShort(int index, int value) {
+ PlatformDependent.putShort(addr(index), (short) value);
+ }
+
+ private void _setInt(int index, int value) {
+ PlatformDependent.putInt(addr(index), value);
+ }
+
+ private void _setLong(int index, long value) {
+ PlatformDependent.putLong(addr(index), value);
+ }
+
+ @Override
+ public byte getByte(int index) {
+ return PlatformDependent.getByte(addr(index));
+ }
+
+ @Override
+ public ByteBuf setByte(int index, int value) {
+ PlatformDependent.putByte(addr(index), (byte) value);
+ return this;
+ }
+
+ @Override
+ public boolean release() {
+ return release(1);
+ }
+
+ @Override
+ public boolean release(int decrement) {
+ boolean released = super.release(decrement);
+ if (released && initCap != -1) {
+ bufferCount.decrementAndGet();
+ bufferSize.addAndGet(-initCap);
+ }
+ return released;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java b/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
new file mode 100644
index 0000000..f5ae70c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/exception/OversizedAllocationException.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.exception;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.record.RecordBatch;
+
+/**
+ * An exception that is used to signal that allocation request in bytes is greater than the maximum allowed by
+ * {@link org.apache.drill.exec.memory.BufferAllocator#buffer(int) allocator}.
+ *
+ * <p>Operators should handle this exception to split the batch and later resume the execution on the next
+ * {@link RecordBatch#next() iteration}.</p>
+ *
+ */
+public class OversizedAllocationException extends DrillRuntimeException {
+ public OversizedAllocationException() {
+ super();
+ }
+
+ public OversizedAllocationException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public OversizedAllocationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public OversizedAllocationException(String message) {
+ super(message);
+ }
+
+ public OversizedAllocationException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/exception/SchemaChangeRuntimeException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/exception/SchemaChangeRuntimeException.java b/exec/vector/src/main/java/org/apache/drill/exec/exception/SchemaChangeRuntimeException.java
new file mode 100644
index 0000000..f2a7e63
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/exception/SchemaChangeRuntimeException.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.exception;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+public class SchemaChangeRuntimeException extends DrillRuntimeException {
+ public SchemaChangeRuntimeException() {
+ super();
+ }
+
+ public SchemaChangeRuntimeException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ public SchemaChangeRuntimeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public SchemaChangeRuntimeException(String message) {
+ super(message);
+ }
+
+ public SchemaChangeRuntimeException(Throwable cause) {
+ super(cause);
+ }
+}
[39/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
deleted file mode 100644
index 2fd8e67..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ /dev/null
@@ -1,664 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.math.BigDecimal;
-import java.util.GregorianCalendar;
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.common.util.CoreDecimalUtility;
-
-import com.google.common.collect.Iterators;
-
-public class ValueExpressions {
-
- public static LogicalExpression getBigInt(long l){
- return new LongExpression(l);
- }
-
- public static LogicalExpression getInt(int i){
- return new IntExpression(i, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getFloat8(double d){
- return new DoubleExpression(d, ExpressionPosition.UNKNOWN);
- }
- public static LogicalExpression getFloat4(float f){
- return new FloatExpression(f, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getBit(boolean b){
- return new BooleanExpression(Boolean.toString(b), ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getChar(String s){
- return new QuotedString(s, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getDate(GregorianCalendar date) {
- return new org.apache.drill.common.expression.ValueExpressions.DateExpression(date.getTimeInMillis());
- }
-
- public static LogicalExpression getTime(GregorianCalendar time) {
- int millis = time.get(GregorianCalendar.HOUR_OF_DAY) * 60 * 60 * 1000 +
- time.get(GregorianCalendar.MINUTE) * 60 * 1000 +
- time.get(GregorianCalendar.SECOND) * 1000 +
- time.get(GregorianCalendar.MILLISECOND);
-
- return new TimeExpression(millis);
- }
-
- public static LogicalExpression getTimeStamp(GregorianCalendar date) {
- return new org.apache.drill.common.expression.ValueExpressions.TimeStampExpression(date.getTimeInMillis());
- }
- public static LogicalExpression getIntervalYear(int months) {
- return new IntervalYearExpression(months);
- }
-
- public static LogicalExpression getIntervalDay(long intervalInMillis) {
- return new IntervalDayExpression(intervalInMillis);
- }
-
- public static LogicalExpression getDecimal9(BigDecimal i) {
- return new Decimal9Expression(i, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getDecimal18(BigDecimal i) {
- return new Decimal18Expression(i, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getDecimal28(BigDecimal i) {
- return new Decimal28Expression(i, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getDecimal38(BigDecimal i) {
- return new Decimal38Expression(i, ExpressionPosition.UNKNOWN);
- }
-
- public static LogicalExpression getNumericExpression(String sign, String s, ExpressionPosition ep) {
- String numStr = (sign == null) ? s : sign+s;
- try {
- int a = Integer.parseInt(numStr);
- return new IntExpression(a, ep);
- } catch (Exception e) {
-
- }
- try {
- long l = Long.parseLong(numStr);
- return new LongExpression(l, ep);
- } catch (Exception e) {
-
- }
-
- try {
- double d = Double.parseDouble(numStr);
- return new DoubleExpression(d, ep);
- } catch (Exception e) {
-
- }
-
- throw new IllegalArgumentException(String.format("Unable to parse string %s as integer or floating point number.",
- numStr));
-
- }
-
- protected static abstract class ValueExpression<V> extends LogicalExpressionBase {
- public final V value;
-
- protected ValueExpression(String value, ExpressionPosition pos) {
- super(pos);
- this.value = parseValue(value);
- }
-
- protected abstract V parseValue(String s);
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
- }
-
- public static class BooleanExpression extends ValueExpression<Boolean> {
-
-
- public BooleanExpression(String value, ExpressionPosition pos) {
- super(value, pos);
- }
-
-
- @Override
- protected Boolean parseValue(String s) {
- return Boolean.parseBoolean(s);
- }
-
- @Override
- public MajorType getMajorType() {
- return Types.REQUIRED_BIT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitBooleanConstant(this, value);
- }
-
- public boolean getBoolean() {
- return value;
- }
-
- }
-
- public static class FloatExpression extends LogicalExpressionBase {
- private float f;
-
- private static final MajorType FLOAT_CONSTANT = Types.required(MinorType.FLOAT4);
-
- public FloatExpression(float f, ExpressionPosition pos) {
- super(pos);
- this.f = f;
- }
-
- public float getFloat() {
- return f;
- }
-
- @Override
- public MajorType getMajorType() {
- return FLOAT_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitFloatConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class IntExpression extends LogicalExpressionBase {
-
- private static final MajorType INT_CONSTANT = Types.required(MinorType.INT);
-
- private int i;
-
- public IntExpression(int i, ExpressionPosition pos) {
- super(pos);
- this.i = i;
- }
-
- public int getInt() {
- return i;
- }
-
- @Override
- public MajorType getMajorType() {
- return INT_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitIntConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class Decimal9Expression extends LogicalExpressionBase {
-
- private int decimal;
- private int scale;
- private int precision;
-
- public Decimal9Expression(BigDecimal input, ExpressionPosition pos) {
- super(pos);
- this.scale = input.scale();
- this.precision = input.precision();
- this.decimal = CoreDecimalUtility.getDecimal9FromBigDecimal(input, scale, precision);
- }
-
-
- public int getIntFromDecimal() {
- return decimal;
- }
-
- public int getScale() {
- return scale;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- @Override
- public MajorType getMajorType() {
- return MajorType.newBuilder().setMinorType(MinorType.DECIMAL9).setScale(scale).setPrecision(precision).setMode(DataMode.REQUIRED).build();
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitDecimal9Constant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
- }
-
- public static class Decimal18Expression extends LogicalExpressionBase {
-
- private long decimal;
- private int scale;
- private int precision;
-
- public Decimal18Expression(BigDecimal input, ExpressionPosition pos) {
- super(pos);
- this.scale = input.scale();
- this.precision = input.precision();
- this.decimal = CoreDecimalUtility.getDecimal18FromBigDecimal(input, scale, precision);
- }
-
-
- public long getLongFromDecimal() {
- return decimal;
- }
-
- public int getScale() {
- return scale;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- @Override
- public MajorType getMajorType() {
- return MajorType.newBuilder().setMinorType(MinorType.DECIMAL18).setScale(scale).setPrecision(precision).setMode(DataMode.REQUIRED).build();
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitDecimal18Constant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class Decimal28Expression extends LogicalExpressionBase {
-
- private BigDecimal bigDecimal;
-
- public Decimal28Expression(BigDecimal input, ExpressionPosition pos) {
- super(pos);
- this.bigDecimal = input;
- }
-
-
- public BigDecimal getBigDecimal() {
- return bigDecimal;
- }
-
- @Override
- public MajorType getMajorType() {
- return MajorType.newBuilder().setMinorType(MinorType.DECIMAL28SPARSE).setScale(bigDecimal.scale()).setPrecision(bigDecimal.precision()).setMode(DataMode.REQUIRED).build();
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitDecimal28Constant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class Decimal38Expression extends LogicalExpressionBase {
-
- private BigDecimal bigDecimal;
-
- public Decimal38Expression(BigDecimal input, ExpressionPosition pos) {
- super(pos);
- this.bigDecimal = input;
- }
-
- public BigDecimal getBigDecimal() {
- return bigDecimal;
- }
-
- @Override
- public MajorType getMajorType() {
- return MajorType.newBuilder().setMinorType(MinorType.DECIMAL38SPARSE).setScale(bigDecimal.scale()).setPrecision(bigDecimal.precision()).setMode(DataMode.REQUIRED).build();
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitDecimal38Constant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
-
- public static class DoubleExpression extends LogicalExpressionBase {
- private double d;
-
- private static final MajorType DOUBLE_CONSTANT = Types.required(MinorType.FLOAT8);
-
- public DoubleExpression(double d, ExpressionPosition pos) {
- super(pos);
- this.d = d;
- }
-
- public double getDouble() {
- return d;
- }
-
- @Override
- public MajorType getMajorType() {
- return DOUBLE_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitDoubleConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class LongExpression extends LogicalExpressionBase {
-
- private static final MajorType LONG_CONSTANT = Types.required(MinorType.BIGINT);
-
- private long l;
-
- public LongExpression(long l) {
- this(l, ExpressionPosition.UNKNOWN);
- }
-
- public LongExpression(long l, ExpressionPosition pos) {
- super(pos);
- this.l = l;
- }
-
- public long getLong() {
- return l;
- }
-
- @Override
- public MajorType getMajorType() {
- return LONG_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitLongConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
-
- public static class DateExpression extends LogicalExpressionBase {
-
- private static final MajorType DATE_CONSTANT = Types.required(MinorType.DATE);
-
- private long dateInMillis;
-
- public DateExpression(long l) {
- this(l, ExpressionPosition.UNKNOWN);
- }
-
- public DateExpression(long dateInMillis, ExpressionPosition pos) {
- super(pos);
- this.dateInMillis = dateInMillis;
- }
-
- public long getDate() {
- return dateInMillis;
- }
-
- @Override
- public MajorType getMajorType() {
- return DATE_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitDateConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
-
- public static class TimeExpression extends LogicalExpressionBase {
-
- private static final MajorType TIME_CONSTANT = Types.required(MinorType.TIME);
-
- private int timeInMillis;
-
- public TimeExpression(int timeInMillis) {
- this(timeInMillis, ExpressionPosition.UNKNOWN);
- }
-
- public TimeExpression(int timeInMillis, ExpressionPosition pos) {
- super(pos);
- this.timeInMillis = timeInMillis;
- }
-
- public int getTime() {
- return timeInMillis;
- }
-
- @Override
- public MajorType getMajorType() {
- return TIME_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitTimeConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class TimeStampExpression extends LogicalExpressionBase {
-
- private static final MajorType TIMESTAMP_CONSTANT = Types.required(MinorType.TIMESTAMP);
-
- private long timeInMillis;
-
- public TimeStampExpression(long timeInMillis) {
- this(timeInMillis, ExpressionPosition.UNKNOWN);
- }
-
- public TimeStampExpression(long timeInMillis, ExpressionPosition pos) {
- super(pos);
- this.timeInMillis = timeInMillis;
- }
-
- public long getTimeStamp() {
- return timeInMillis;
- }
-
- @Override
- public MajorType getMajorType() {
- return TIMESTAMP_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitTimeStampConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class IntervalYearExpression extends LogicalExpressionBase {
-
- private static final MajorType INTERVALYEAR_CONSTANT = Types.required(MinorType.INTERVALYEAR);
-
- private int months;
-
- public IntervalYearExpression(int months) {
- this(months, ExpressionPosition.UNKNOWN);
- }
-
- public IntervalYearExpression(int months, ExpressionPosition pos) {
- super(pos);
- this.months = months;
- }
-
- public int getIntervalYear() {
- return months;
- }
-
- @Override
- public MajorType getMajorType() {
- return INTERVALYEAR_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitIntervalYearConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class IntervalDayExpression extends LogicalExpressionBase {
-
- private static final MajorType INTERVALDAY_CONSTANT = Types.required(MinorType.INTERVALDAY);
- private static final long MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
-
- private int days;
- private int millis;
-
- public IntervalDayExpression(long intervalInMillis) {
- this((int) (intervalInMillis / MILLIS_IN_DAY), (int) (intervalInMillis % MILLIS_IN_DAY), ExpressionPosition.UNKNOWN);
- }
-
- public IntervalDayExpression(int days, int millis, ExpressionPosition pos) {
- super(pos);
- this.days = days;
- this.millis = millis;
- }
-
- public int getIntervalDay() {
- return days;
- }
-
- public int getIntervalMillis() {
- return millis;
- }
-
- @Override
- public MajorType getMajorType() {
- return INTERVALDAY_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitIntervalDayConstant(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- }
-
- public static class QuotedString extends ValueExpression<String> {
-
- private static final MajorType QUOTED_STRING_CONSTANT = Types.required(MinorType.VARCHAR);
-
- public QuotedString(String value, ExpressionPosition pos) {
- super(value, pos);
- }
-
- public String getString() {
- return value;
- }
-
- @Override
- protected String parseValue(String s) {
- return s;
- }
-
- @Override
- public MajorType getMajorType() {
- return QUOTED_STRING_CONSTANT;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitQuotedStringConstant(this, value);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
deleted file mode 100644
index 34997ab..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-public class CastFunctions {
-
- private static Map<MinorType, String> TYPE2FUNC = new HashMap<>();
- /** The cast functions that need to be replaced (if
- * "drill.exec.functions.cast_empty_string_to_null" is set to true). */
- private static Set<String> CAST_FUNC_REPLACEMENT_NEEDED = new HashSet<>();
- /** Map from the replaced functions to the new ones (for non-nullable VARCHAR). */
- private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE = new HashMap<>();
- /** Map from the replaced functions to the new ones (for nullable VARCHAR). */
- private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NULLABLE = new HashMap<>();
-
- static {
- TYPE2FUNC.put(MinorType.UNION, "castUNION");
- TYPE2FUNC.put(MinorType.BIGINT, "castBIGINT");
- TYPE2FUNC.put(MinorType.INT, "castINT");
- TYPE2FUNC.put(MinorType.BIT, "castBIT");
- TYPE2FUNC.put(MinorType.TINYINT, "castTINYINT");
- TYPE2FUNC.put(MinorType.FLOAT4, "castFLOAT4");
- TYPE2FUNC.put(MinorType.FLOAT8, "castFLOAT8");
- TYPE2FUNC.put(MinorType.VARCHAR, "castVARCHAR");
- TYPE2FUNC.put(MinorType.VAR16CHAR, "castVAR16CHAR");
- TYPE2FUNC.put(MinorType.VARBINARY, "castVARBINARY");
- TYPE2FUNC.put(MinorType.DATE, "castDATE");
- TYPE2FUNC.put(MinorType.TIME, "castTIME");
- TYPE2FUNC.put(MinorType.TIMESTAMP, "castTIMESTAMP");
- TYPE2FUNC.put(MinorType.TIMESTAMPTZ, "castTIMESTAMPTZ");
- TYPE2FUNC.put(MinorType.INTERVALDAY, "castINTERVALDAY");
- TYPE2FUNC.put(MinorType.INTERVALYEAR, "castINTERVALYEAR");
- TYPE2FUNC.put(MinorType.INTERVAL, "castINTERVAL");
- TYPE2FUNC.put(MinorType.DECIMAL9, "castDECIMAL9");
- TYPE2FUNC.put(MinorType.DECIMAL18, "castDECIMAL18");
- TYPE2FUNC.put(MinorType.DECIMAL28SPARSE, "castDECIMAL28SPARSE");
- TYPE2FUNC.put(MinorType.DECIMAL28DENSE, "castDECIMAL28DENSE");
- TYPE2FUNC.put(MinorType.DECIMAL38SPARSE, "castDECIMAL38SPARSE");
- TYPE2FUNC.put(MinorType.DECIMAL38DENSE, "castDECIMAL38DENSE");
-
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.INT));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.BIGINT));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.FLOAT4));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.FLOAT8));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL9));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL18));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE));
-
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringVarCharToNullableINT");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringVarCharToNullableBIGINT");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringVarCharToNullableFLOAT4");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringVarCharToNullableFLOAT8");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringVarCharToNullableDECIMAL9");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringVarCharToNullableDECIMAL18");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringVarCharToNullableDECIMAL28SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringVarCharToNullableDECIMAL38SPARSE");
-
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringNullableVarCharToNullableINT");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringNullableVarCharToNullableBIGINT");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringNullableVarCharToNullableFLOAT4");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringNullableVarCharToNullableFLOAT8");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringNullableVarCharToNullableDECIMAL9");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringNullableVarCharToNullableDECIMAL18");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringNullableVarCharToNullableDECIMAL28SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringNullableVarCharToNullableDECIMAL38SPARSE");
- }
-
- /**
- * Given the target type, get the appropriate cast function
- * @param targetMinorType the target data type
- * @return
- */
- public static String getCastFunc(MinorType targetMinorType) {
- String func = TYPE2FUNC.get(targetMinorType);
- if (func != null) {
- return func;
- }
-
- throw new RuntimeException(
- String.format("cast function for type %s is not defined", targetMinorType.name()));
- }
-
- /**
- * Get a replacing cast function for the original function, based on the specified data mode
- * @param originalCastFunction original cast function
- * @param dataMode data mode of the input data
- * @return
- */
- public static String getReplacingCastFunction(String originalCastFunction, org.apache.drill.common.types.TypeProtos.DataMode dataMode) {
- if(dataMode == TypeProtos.DataMode.OPTIONAL) {
- return getReplacingCastFunctionFromNullable(originalCastFunction);
- }
-
- if(dataMode == TypeProtos.DataMode.REQUIRED) {
- return getReplacingCastFunctionFromNonNullable(originalCastFunction);
- }
-
- throw new RuntimeException(
- String.format("replacing cast function for datatype %s is not defined", dataMode));
- }
-
- /**
- * Check if a replacing cast function is available for the the original function
- * @param originalfunction original cast function
- * @return
- */
- public static boolean isReplacementNeeded(MinorType inputType, String originalfunction) {
- return inputType == MinorType.VARCHAR && CAST_FUNC_REPLACEMENT_NEEDED.contains(originalfunction);
- }
-
- private static String getReplacingCastFunctionFromNonNullable(String originalCastFunction) {
- if(CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.containsKey(originalCastFunction)) {
- return CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE.get(originalCastFunction);
- }
-
- throw new RuntimeException(
- String.format("replacing cast function for %s is not defined", originalCastFunction));
- }
-
- private static String getReplacingCastFunctionFromNullable(String originalCastFunction) {
- if(CAST_FUNC_REPLACEMENT_FROM_NULLABLE.containsKey(originalCastFunction)) {
- return CAST_FUNC_REPLACEMENT_FROM_NULLABLE.get(originalCastFunction);
- }
-
- throw new RuntimeException(
- String.format("replacing cast function for %s is not defined", originalCastFunction));
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/fn/FuncHolder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/FuncHolder.java b/common/src/main/java/org/apache/drill/common/expression/fn/FuncHolder.java
deleted file mode 100644
index a8824e0..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/FuncHolder.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-/** This should be removed once common and exec/java-exec modules are merged (DRILL-507). */
-public interface FuncHolder {
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/fn/package-info.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/package-info.java b/common/src/main/java/org/apache/drill/common/expression/fn/package-info.java
deleted file mode 100644
index 6af3f5f..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Logical function definitions.
- */
-package org.apache.drill.common.expression.fn;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/package-info.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/package-info.java b/common/src/main/java/org/apache/drill/common/expression/package-info.java
deleted file mode 100644
index 0c32bf6..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/package-info.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Logical expression tree representation.
- *
- * Drill manages expressions provided in many different parts of SQL queries.
- * This includes scalar expressions in select, filter and join conditions, as
- * well as aggregate and window functions. These expressions are represented
- * logically as ASTs during planning. The classes defined here provide the
- * different nodes in the expression tree, as well as utilities for building
- * and manipulating expressions during parsing and planning.
- *
- */
-package org.apache.drill.common.expression;
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/types/AtomType.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/types/AtomType.java b/common/src/main/java/org/apache/drill/common/expression/types/AtomType.java
deleted file mode 100644
index 2918216..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/types/AtomType.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.types;
-
-
-public class AtomType extends DataType {
- private String name;
- private Comparability comparability;
- private boolean isNumericType;
-
- public AtomType(String name, Comparability comparability, boolean isNumericType) {
- super();
- this.name = name;
- this.comparability = comparability;
- this.isNumericType = isNumericType;
- }
-
-
- @Override
- public boolean isNumericType() {
- return isNumericType;
- }
-
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public boolean isLateBind() {
- return false;
- }
-
- @Override
- public boolean hasChildType() {
- return false;
- }
-
- @Override
- public DataType getChildType() {
- return null;
- }
-
- @Override
- public Comparability getComparability() {
- return comparability;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/types/DataType.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/types/DataType.java b/common/src/main/java/org/apache/drill/common/expression/types/DataType.java
deleted file mode 100644
index 56102ee..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/types/DataType.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.types;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-
-@JsonSerialize(using = DataType.Se.class)
-@JsonDeserialize(using = DataType.De.class)
-abstract class DataType {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataType.class);
-
- public static enum Comparability{
- UNKNOWN, NONE, EQUAL, ORDERED;
- }
-
- public abstract String getName();
- public abstract boolean isLateBind();
- public abstract boolean hasChildType();
- public abstract DataType getChildType();
- public abstract Comparability getComparability();
- public abstract boolean isNumericType();
-
-
-
- public static final DataType LATEBIND = new LateBindType();
- public static final DataType BOOLEAN = new AtomType("BOOLEAN", Comparability.EQUAL, false);
- public static final DataType BYTES = new AtomType("BYTES", Comparability.ORDERED, false);
- public static final DataType SIGNED_BYTE = new AtomType("SIGNED_BYTE", Comparability.ORDERED, true);
- public static final DataType SIGNED_INT16 = new AtomType("SIGNED_INT16", Comparability.ORDERED, true);
- public static final DataType NVARCHAR = new AtomType("VARCHAR", Comparability.ORDERED, false);
- public static final DataType FLOAT32 = new AtomType("FLOAT32", Comparability.ORDERED, true);
- public static final DataType FLOAT64 = new AtomType("FLOAT64", Comparability.ORDERED, true);
- public static final DataType INT64 = new AtomType("INT64", Comparability.ORDERED, true);
- public static final DataType INT32 = new AtomType("INT32", Comparability.ORDERED, true);
- public static final DataType INT16 = new AtomType("INT16", Comparability.ORDERED, true);
- public static final DataType UINT16 = new AtomType("UINT16", Comparability.ORDERED, true);
-// public static final DataType INT16 = new AtomType("int16", Comparability.ORDERED, true);
-// public static final DataType BIG_INTEGER = new AtomType("bigint", Comparability.ORDERED, true);
-// public static final DataType BIG_DECIMAL = new AtomType("bigdecimal", Comparability.ORDERED, true);
- public static final DataType DATE = new AtomType("DATE", Comparability.ORDERED, false);
- public static final DataType DATETIME = new AtomType("DATETIME", Comparability.ORDERED, false);
- public static final DataType MAP = new AtomType("MAP", Comparability.NONE, false);
- public static final DataType ARRAY = new AtomType("ARRAY", Comparability.NONE, false);
- public static final DataType NULL = new AtomType("NULL", Comparability.NONE, false);
-
- //TODO: Hack to get some size data, needs to be fixed so that each type reveals it's size.
- public int size() {
- if(this == BOOLEAN) {
- return 1;
- }else if(this == INT32) {
- return 4;
- }else if(this == INT16) {
- return 4;
- }
- return 2;
- }
-
- static final Map<String, DataType> TYPES;
- static {
- Field[] fields = DataType.class.getFields();
- Map<String, DataType> types = new HashMap<String, DataType>();
- for (Field f : fields) {
- //logger.debug("Reviewing {}, Field: {}", f.getClass(), f);
- if (Modifier.isStatic(f.getModifiers())) {
- try {
- Object o = f.get(null);
- //logger.debug("Object {}", o);
-
- if (o instanceof DataType) {
- types.put(((DataType) o).getName(), (DataType) o);
- }
- } catch (IllegalArgumentException | IllegalAccessException e) {
- logger.warn("Failure while reading DataType.", e);
- }
- }
- }
- TYPES = Collections.unmodifiableMap(types);
- }
-
- public static DataType getDataType(String name) {
- if (TYPES.containsKey(name)) {
- return TYPES.get(name);
- } else {
- throw new IllegalArgumentException(String.format("Unknown type requested of [%s].", name));
- }
- }
-
- public static class De extends StdDeserializer<DataType> {
-
- public De() {
- super(DataType.class);
- }
-
- @Override
- public DataType deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
- JsonProcessingException {
- return getDataType(this._parseString(jp, ctxt));
- }
-
- }
-
- public static class Se extends StdSerializer<DataType> {
-
- public Se() {
- super(DataType.class);
- }
-
- @Override
- public void serialize(DataType value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
- JsonGenerationException {
- jgen.writeString(value.getName());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/types/DataTypeFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/types/DataTypeFactory.java b/common/src/main/java/org/apache/drill/common/expression/types/DataTypeFactory.java
deleted file mode 100644
index c95bf1f..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/types/DataTypeFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.types;
-
-public abstract class DataTypeFactory {
-
- public abstract DataType getArrayType(DataType containedType);
- public abstract DataType getMapType(DataType keyType, DataType valueType);
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/types/LateBindType.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/types/LateBindType.java b/common/src/main/java/org/apache/drill/common/expression/types/LateBindType.java
deleted file mode 100644
index db0b09c..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/types/LateBindType.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.types;
-
-class LateBindType extends DataType {
-
- @Override
- public String getName() {
- return "LATE";
- }
-
- @Override
- public boolean isLateBind() {
- return true;
- }
-
- @Override
- public boolean hasChildType() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DataType getChildType() {
- throw new UnsupportedOperationException();
- }
-
-
- @Override
- public boolean isNumericType() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Comparability getComparability() {
- return Comparability.UNKNOWN;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/types/package-info.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/types/package-info.java b/common/src/main/java/org/apache/drill/common/expression/types/package-info.java
deleted file mode 100644
index 744b174..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/types/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Logical type definitions.
- */
-package org.apache.drill.common.expression.types;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
deleted file mode 100644
index a31e6e8..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-import org.apache.drill.common.expression.BooleanOperator;
-import org.apache.drill.common.expression.CastExpression;
-import org.apache.drill.common.expression.ConvertExpression;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionHolderExpression;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.NullExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.TypedNullConstant;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DateExpression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
-import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
-
-public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> implements ExprVisitor<T, VAL, EXCEP> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractExprVisitor.class);
-
- @Override
- public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP {
- return visitUnknown(call, value);
- }
-
- @Override
- public T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP {
- return visitUnknown(holder, value);
- }
-
- @Override
- public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP {
- return visitUnknown(ifExpr, value);
- }
-
- @Override
- public T visitBooleanOperator(BooleanOperator op, VAL value) throws EXCEP {
- return visitUnknown(op, value);
- }
-
- @Override
- public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP {
- return visitUnknown(path, value);
- }
-
- @Override
- public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP {
- return visitUnknown(fExpr, value);
- }
-
- @Override
- public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP {
- return visitUnknown(intExpr, value);
- }
-
- @Override
- public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP {
- return visitUnknown(intExpr, value);
- }
-
-
- @Override
- public T visitDecimal9Constant(Decimal9Expression decExpr, VAL value) throws EXCEP {
- return visitUnknown(decExpr, value);
- }
-
- @Override
- public T visitDecimal18Constant(Decimal18Expression decExpr, VAL value) throws EXCEP {
- return visitUnknown(decExpr, value);
- }
-
- @Override
- public T visitDecimal28Constant(Decimal28Expression decExpr, VAL value) throws EXCEP {
- return visitUnknown(decExpr, value);
- }
-
- @Override
- public T visitDecimal38Constant(Decimal38Expression decExpr, VAL value) throws EXCEP {
- return visitUnknown(decExpr, value);
- }
-
- @Override
- public T visitDateConstant(DateExpression intExpr, VAL value) throws EXCEP {
- return visitUnknown(intExpr, value);
- }
-
- @Override
- public T visitTimeConstant(TimeExpression intExpr, VAL value) throws EXCEP {
- return visitUnknown(intExpr, value);
- }
-
- @Override
- public T visitTimeStampConstant(TimeStampExpression intExpr, VAL value) throws EXCEP {
- return visitUnknown(intExpr, value);
- }
-
- @Override
- public T visitIntervalYearConstant(IntervalYearExpression intExpr, VAL value) throws EXCEP {
- return visitUnknown(intExpr, value);
- }
-
- @Override
- public T visitIntervalDayConstant(IntervalDayExpression intExpr, VAL value) throws EXCEP {
- return visitUnknown(intExpr, value);
- }
-
- @Override
- public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP {
- return visitUnknown(dExpr, value);
- }
-
- @Override
- public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP {
- return visitUnknown(e, value);
- }
-
- @Override
- public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP {
- return visitUnknown(e, value);
- }
-
- @Override
- public T visitCastExpression(CastExpression e, VAL value) throws EXCEP {
- return visitUnknown(e, value);
- }
-
- @Override
- public T visitConvertExpression(ConvertExpression e, VAL value) throws EXCEP {
- return visitUnknown(e, value);
- }
-
- @Override
- public T visitNullConstant(TypedNullConstant e, VAL value) throws EXCEP {
- return visitUnknown(e, value);
- }
-
- @Override
- public T visitNullExpression(NullExpression e, VAL value) throws EXCEP {
- return visitUnknown(e, value);
- }
-
- @Override
- public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP {
- throw new UnsupportedOperationException(String.format("Expression of type %s not handled by visitor type %s.", e.getClass().getCanonicalName(), this.getClass().getCanonicalName()));
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
deleted file mode 100644
index 46789e3..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-import org.apache.drill.common.expression.BooleanOperator;
-import org.apache.drill.common.expression.CastExpression;
-import org.apache.drill.common.expression.ConvertExpression;
-import org.apache.drill.common.expression.ErrorCollector;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionHolderExpression;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.IfExpression.IfCondition;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.NullExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.TypedNullConstant;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DateExpression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
-import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
-
-public final class AggregateChecker implements ExprVisitor<Boolean, ErrorCollector, RuntimeException>{
-
- public static final AggregateChecker INSTANCE = new AggregateChecker();
-
- public static boolean isAggregating(LogicalExpression e, ErrorCollector errors) {
- return e.accept(INSTANCE, errors);
- }
-
- @Override
- public Boolean visitFunctionCall(FunctionCall call, ErrorCollector errors) {
- throw new UnsupportedOperationException("FunctionCall is not expected here. "+
- "It should have been converted to FunctionHolderExpression in materialization");
- }
-
- @Override
- public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors) {
- if (holder.isAggregating()) {
- for (int i = 0; i < holder.args.size(); i++) {
- LogicalExpression e = holder.args.get(i);
- if(e.accept(this, errors)) {
- errors.addGeneralError(e.getPosition(),
- String.format("Aggregating function call %s includes nested aggregations at arguments number %d. " +
- "This isn't allowed.", holder.getName(), i));
- }
- }
- return true;
- } else {
- for (LogicalExpression e : holder.args) {
- if (e.accept(this, errors)) {
- return true;
- }
- }
- return false;
- }
- }
-
- @Override
- public Boolean visitBooleanOperator(BooleanOperator op, ErrorCollector errors) {
- for (LogicalExpression arg : op.args) {
- if (arg.accept(this, errors)) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public Boolean visitIfExpression(IfExpression ifExpr, ErrorCollector errors) {
- IfCondition c = ifExpr.ifCondition;
- if (c.condition.accept(this, errors) || c.expression.accept(this, errors)) {
- return true;
- }
- return ifExpr.elseExpression.accept(this, errors);
- }
-
- @Override
- public Boolean visitSchemaPath(SchemaPath path, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitIntConstant(IntExpression intExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitFloatConstant(FloatExpression fExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitLongConstant(LongExpression intExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitBooleanConstant(BooleanExpression e, ErrorCollector errors) {
- return false;
- }
- public Boolean visitDecimal9Constant(Decimal9Expression decExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitDecimal18Constant(Decimal18Expression decExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitDecimal28Constant(Decimal28Expression decExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitDecimal38Constant(Decimal38Expression decExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitQuotedStringConstant(QuotedString e, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitUnknown(LogicalExpression e, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitCastExpression(CastExpression e, ErrorCollector errors) {
- return e.getInput().accept(this, errors);
- }
-
- @Override
- public Boolean visitConvertExpression(ConvertExpression e, ErrorCollector errors) throws RuntimeException {
- return e.getInput().accept(this, errors);
- }
-
- @Override
- public Boolean visitDateConstant(DateExpression intExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitTimeConstant(TimeExpression intExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitTimeStampConstant(TimeStampExpression intExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitIntervalYearConstant(IntervalYearExpression intExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitIntervalDayConstant(IntervalDayExpression intExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitNullConstant(TypedNullConstant e, ErrorCollector value) throws RuntimeException {
- return false;
- }
-
- @Override
- public Boolean visitNullExpression(NullExpression e, ErrorCollector value) throws RuntimeException {
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
deleted file mode 100644
index 254926a..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.common.expression.visitors;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.drill.common.expression.BooleanOperator;
-import org.apache.drill.common.expression.CastExpression;
-import org.apache.drill.common.expression.ConvertExpression;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionHolderExpression;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.IfExpression.IfCondition;
-import org.apache.drill.common.expression.LogicalExpression;
-
-import com.google.common.collect.Lists;
-
-public class ConditionalExprOptimizer extends AbstractExprVisitor<LogicalExpression, Void, RuntimeException> {
-
- public static ConditionalExprOptimizer INSTANCE = new ConditionalExprOptimizer();
-
- @Override
- public LogicalExpression visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException {
-
- List<LogicalExpression> newArgs = Lists.newArrayList();
-
- newArgs.addAll(op.args);
-
- Collections.sort(newArgs, costComparator);
-
- return new BooleanOperator(op.getName(), newArgs, op.getPosition());
- }
-
-
- @Override
- public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holder, Void value) throws RuntimeException {
- List<LogicalExpression> args = Lists.newArrayList();
- for (int i = 0; i < holder.args.size(); ++i) {
- LogicalExpression newExpr = holder.args.get(i).accept(this, value);
- assert newExpr != null;
- args.add(newExpr);
- }
-
- //replace with a new function call, since its argument could be changed.
-
- return holder.copy(args);
- }
-
- @Override
- public LogicalExpression visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
- return e;
- }
-
-
- @Override
- public LogicalExpression visitIfExpression(IfExpression ifExpr, Void value) throws RuntimeException{
- LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, value);
- IfCondition conditions = ifExpr.ifCondition;
-
- LogicalExpression newCondition = conditions.condition.accept(this, value);
- LogicalExpression newExpr = conditions.expression.accept(this, value);
- conditions = new IfExpression.IfCondition(newCondition, newExpr);
-
- return IfExpression.newBuilder().setElse(newElseExpr).setIfCondition(conditions).build();
- }
-
- @Override
- public LogicalExpression visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
- throw new UnsupportedOperationException("FunctionCall is not expected here. "
- + "It should have been converted to FunctionHolderExpression in materialization");
- }
-
- @Override
- public LogicalExpression visitCastExpression(CastExpression cast, Void value) throws RuntimeException {
- throw new UnsupportedOperationException("CastExpression is not expected here. "
- + "It should have been converted to FunctionHolderExpression in materialization");
- }
-
-
- @Override
- public LogicalExpression visitConvertExpression(ConvertExpression cast, Void value) throws RuntimeException {
- throw new UnsupportedOperationException("ConvertExpression is not expected here. "
- + "It should have been converted to FunctionHolderExpression in materialization");
- }
-
- private static Comparator<LogicalExpression> costComparator = new Comparator<LogicalExpression> () {
- public int compare(LogicalExpression e1, LogicalExpression e2) {
- return e1.getCumulativeCost() <= e2.getCumulativeCost() ? -1 : 1;
- }
- };
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
deleted file mode 100644
index ccb11a6..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-import org.apache.drill.common.expression.BooleanOperator;
-import org.apache.drill.common.expression.CastExpression;
-import org.apache.drill.common.expression.ConvertExpression;
-import org.apache.drill.common.expression.ErrorCollector;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionHolderExpression;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.IfExpression.IfCondition;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.NullExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.TypedNullConstant;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DateExpression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
-import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
-
-final class ConstantChecker implements ExprVisitor<Boolean, ErrorCollector, RuntimeException> {
-
-
- private final static ConstantChecker INSTANCE = new ConstantChecker();
-
- private ConstantChecker() {}
-
- public static void checkConstants(LogicalExpression e, ErrorCollector errors) {
- e.accept(INSTANCE, errors);
- }
-
- @Override
- public Boolean visitFunctionCall(FunctionCall call, ErrorCollector errors) {
- throw new UnsupportedOperationException("FunctionCall is not expected here. "
- + "It should have been converted to FunctionHolderExpression in materialization");
- }
-
- @Override
- public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors) {
- boolean allArgsAreConstant = true;
- for (int i = 0; i < holder.args.size(); i++) {
- boolean thisArgIsConstant = holder.args.get(i).accept(this, errors);
- if (!thisArgIsConstant) {
- allArgsAreConstant = false;
- if (holder.argConstantOnly(i)) {
- errors.addGeneralError( //
- holder.args.get(i).getPosition(), //
- String.format("Function %s expects constant input for argument number %d", holder.getName(), i));
- }
- }
- }
- return allArgsAreConstant;
- }
-
- @Override
- public Boolean visitBooleanOperator(BooleanOperator op, ErrorCollector errors) {
- for (LogicalExpression e : op.args) {
- if (!e.accept(this, errors)) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public Boolean visitIfExpression(IfExpression ifExpr, ErrorCollector errors) {
- IfCondition c = ifExpr.ifCondition;
- if (!c.condition.accept(this, errors) || !c.expression.accept(this, errors)) {
- return false;
- }
-
- if (!ifExpr.elseExpression.accept(this, errors)) {
- return false;
- }
- return true;
- }
-
- @Override
- public Boolean visitSchemaPath(SchemaPath path, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitIntConstant(IntExpression intExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitFloatConstant(FloatExpression fExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitLongConstant(LongExpression intExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitDateConstant(DateExpression intExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitTimeConstant(TimeExpression intExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitTimeStampConstant(TimeStampExpression intExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitIntervalYearConstant(IntervalYearExpression intExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitIntervalDayConstant(IntervalDayExpression intExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitDecimal9Constant(Decimal9Expression decExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitDecimal18Constant(Decimal18Expression decExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitDecimal28Constant(Decimal28Expression decExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitDecimal38Constant(Decimal38Expression decExpr, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitBooleanConstant(BooleanExpression e, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitQuotedStringConstant(QuotedString e, ErrorCollector errors) {
- return true;
- }
-
- @Override
- public Boolean visitUnknown(LogicalExpression e, ErrorCollector errors) {
- return false;
- }
-
- @Override
- public Boolean visitCastExpression(CastExpression e, ErrorCollector value) throws RuntimeException {
- return e.getInput().accept(this, value);
- }
-
- @Override
- public Boolean visitConvertExpression(ConvertExpression e, ErrorCollector value) throws RuntimeException {
- return e.getInput().accept(this, value);
- }
-
- @Override
- public Boolean visitNullConstant(TypedNullConstant e, ErrorCollector value) throws RuntimeException {
- return true;
- }
-
- @Override
- public Boolean visitNullExpression(NullExpression e, ErrorCollector value) throws RuntimeException {
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
deleted file mode 100644
index 2d0cd8c..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-import org.apache.drill.common.expression.BooleanOperator;
-import org.apache.drill.common.expression.CastExpression;
-import org.apache.drill.common.expression.ConvertExpression;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionHolderExpression;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.NullExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.TypedNullConstant;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DateExpression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
-import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
-
-public interface ExprVisitor<T, VAL, EXCEP extends Exception> {
- public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP;
- public T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP;
- public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP;
- public T visitBooleanOperator(BooleanOperator call, VAL value) throws EXCEP;
- public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
- public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP;
- public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP;
- public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP;
- public T visitDateConstant(DateExpression intExpr, VAL value) throws EXCEP;
- public T visitTimeConstant(TimeExpression intExpr, VAL value) throws EXCEP;
- public T visitTimeStampConstant(TimeStampExpression intExpr, VAL value) throws EXCEP;
- public T visitIntervalYearConstant(IntervalYearExpression intExpr, VAL value) throws EXCEP;
- public T visitIntervalDayConstant(IntervalDayExpression intExpr, VAL value) throws EXCEP;
- public T visitDecimal9Constant(Decimal9Expression decExpr, VAL value) throws EXCEP;
- public T visitDecimal18Constant(Decimal18Expression decExpr, VAL value) throws EXCEP;
- public T visitDecimal28Constant(Decimal28Expression decExpr, VAL value) throws EXCEP;
- public T visitDecimal38Constant(Decimal38Expression decExpr, VAL value) throws EXCEP;
- public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP;
- public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;
- public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP;
- public T visitNullConstant(TypedNullConstant e, VAL value) throws EXCEP;
- public T visitNullExpression(NullExpression e, VAL value) throws EXCEP;
- public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP;
- public T visitCastExpression(CastExpression e, VAL value) throws EXCEP;
- public T visitConvertExpression(ConvertExpression e, VAL value) throws EXCEP;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
deleted file mode 100644
index e905638..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-public class ExpressionValidationError {
- String message;
-
- public ExpressionValidationError(String message) {
- this.message = message;
- }
-
- @Override
- public String toString() {
- return message;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
deleted file mode 100644
index 3d944da..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-public class ExpressionValidationException extends RuntimeException {
-
- public ExpressionValidationException() {
- super();
- }
-
- public ExpressionValidationException(String arg0, Throwable arg1) {
- super(arg0, arg1);
- }
-
- public ExpressionValidationException(String arg0) {
- super(arg0);
- }
-
- public ExpressionValidationException(Throwable arg0) {
- super(arg0);
- }
-
-}
[45/45] drill git commit: Roll forward to 1.4.0-SNAPSHOT
Posted by ja...@apache.org.
Roll forward to 1.4.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7a151d69
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7a151d69
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7a151d69
Branch: refs/heads/master
Commit: 7a151d691979acb12f1d9b5fac54f1179a4b095b
Parents: 2359294
Author: Jacques Nadeau <ja...@apache.org>
Authored: Thu Nov 12 17:32:06 2015 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Nov 12 18:36:35 2015 -0800
----------------------------------------------------------------------
common/pom.xml | 2 +-
contrib/data/pom.xml | 2 +-
contrib/data/tpch-sample-data/pom.xml | 2 +-
contrib/gis/pom.xml | 2 +-
contrib/pom.xml | 2 +-
contrib/sqlline/pom.xml | 2 +-
contrib/storage-hbase/pom.xml | 2 +-
contrib/storage-hive/core/pom.xml | 2 +-
contrib/storage-hive/hive-exec-shade/pom.xml | 2 +-
contrib/storage-hive/pom.xml | 2 +-
contrib/storage-jdbc/pom.xml | 2 +-
contrib/storage-mongo/pom.xml | 2 +-
distribution/pom.xml | 2 +-
exec/java-exec/pom.xml | 2 +-
exec/jdbc-all/pom.xml | 2 +-
exec/jdbc/pom.xml | 2 +-
exec/memory/base/pom.xml | 2 +-
exec/memory/impl/pom.xml | 2 +-
exec/memory/pom.xml | 2 +-
exec/pom.xml | 2 +-
exec/rpc/pom.xml | 2 +-
exec/vector/pom.xml | 2 +-
logical/pom.xml | 2 +-
pom.xml | 2 +-
protocol/pom.xml | 2 +-
tools/fmpp/pom.xml | 2 +-
tools/pom.xml | 2 +-
27 files changed, 27 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 337d848..79968c1 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-common</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/data/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/data/pom.xml b/contrib/data/pom.xml
index 2edaf48..e1bde90 100644
--- a/contrib/data/pom.xml
+++ b/contrib/data/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>drill-contrib-parent</artifactId>
<groupId>org.apache.drill.contrib</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill.contrib.data</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/data/tpch-sample-data/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/data/tpch-sample-data/pom.xml b/contrib/data/tpch-sample-data/pom.xml
index 9ed2fc1..1345466 100644
--- a/contrib/data/tpch-sample-data/pom.xml
+++ b/contrib/data/tpch-sample-data/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>drill-contrib-data-parent</artifactId>
<groupId>org.apache.drill.contrib.data</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>tpch-sample-data</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/gis/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/gis/pom.xml b/contrib/gis/pom.xml
index 133dc2e..7264af3 100644
--- a/contrib/gis/pom.xml
+++ b/contrib/gis/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>drill-contrib-parent</artifactId>
<groupId>org.apache.drill.contrib</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-gis</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/pom.xml b/contrib/pom.xml
index 2b08094..bdd47ce 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill.contrib</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/sqlline/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/sqlline/pom.xml b/contrib/sqlline/pom.xml
index 95fb021..dbc913f 100644
--- a/contrib/sqlline/pom.xml
+++ b/contrib/sqlline/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>drill-contrib-parent</artifactId>
<groupId>org.apache.drill.contrib</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-sqlline</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/pom.xml b/contrib/storage-hbase/pom.xml
index b9c7e08..06478b4 100644
--- a/contrib/storage-hbase/pom.xml
+++ b/contrib/storage-hbase/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>drill-contrib-parent</artifactId>
<groupId>org.apache.drill.contrib</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-storage-hbase</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/storage-hive/core/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index f083507..b54ed92 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.drill.contrib.storage-hive</groupId>
<artifactId>drill-contrib-storage-hive-parent</artifactId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-storage-hive-core</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/storage-hive/hive-exec-shade/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml b/contrib/storage-hive/hive-exec-shade/pom.xml
index 32f1a63..5bb02ad 100644
--- a/contrib/storage-hive/hive-exec-shade/pom.xml
+++ b/contrib/storage-hive/hive-exec-shade/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.drill.contrib.storage-hive</groupId>
<artifactId>drill-contrib-storage-hive-parent</artifactId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-hive-exec-shaded</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/storage-hive/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/pom.xml b/contrib/storage-hive/pom.xml
index 6a7e971..0d04e04 100644
--- a/contrib/storage-hive/pom.xml
+++ b/contrib/storage-hive/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-contrib-parent</artifactId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill.contrib.storage-hive</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/storage-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml
index b930951..69492d3 100755
--- a/contrib/storage-jdbc/pom.xml
+++ b/contrib/storage-jdbc/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>drill-contrib-parent</artifactId>
<groupId>org.apache.drill.contrib</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-jdbc-storage</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/contrib/storage-mongo/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/pom.xml b/contrib/storage-mongo/pom.xml
index 6997625..8f8c6bf 100644
--- a/contrib/storage-mongo/pom.xml
+++ b/contrib/storage-mongo/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>drill-contrib-parent</artifactId>
<groupId>org.apache.drill.contrib</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-mongo-storage</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 3a4db67..3a95a06 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>distribution</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index c5eb2b0..a5d2e11 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>exec-parent</artifactId>
<groupId>org.apache.drill.exec</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-java-exec</artifactId>
<name>exec/Java Execution Engine</name>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/jdbc-all/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 8bacbf5..78fa165 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.drill.exec</groupId>
<artifactId>exec-parent</artifactId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-jdbc-all</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml
index cae5538..96eb9b3 100644
--- a/exec/jdbc/pom.xml
+++ b/exec/jdbc/pom.xml
@@ -15,7 +15,7 @@
<parent>
<groupId>org.apache.drill.exec</groupId>
<artifactId>exec-parent</artifactId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-jdbc</artifactId>
<name>exec/JDBC Driver using dependencies</name>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/memory/base/pom.xml
----------------------------------------------------------------------
diff --git a/exec/memory/base/pom.xml b/exec/memory/base/pom.xml
index 59f6a29..c841e72 100644
--- a/exec/memory/base/pom.xml
+++ b/exec/memory/base/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>memory-parent</artifactId>
<groupId>org.apache.drill.memory</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-memory-base</artifactId>
<name>exec/memory/base</name>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/memory/impl/pom.xml
----------------------------------------------------------------------
diff --git a/exec/memory/impl/pom.xml b/exec/memory/impl/pom.xml
index aa5f0e7..b7c00ac 100644
--- a/exec/memory/impl/pom.xml
+++ b/exec/memory/impl/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>memory-parent</artifactId>
<groupId>org.apache.drill.memory</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-memory-impl</artifactId>
<name>exec/memory/impl</name>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/memory/pom.xml
----------------------------------------------------------------------
diff --git a/exec/memory/pom.xml b/exec/memory/pom.xml
index 8da11aa..dac840c 100644
--- a/exec/memory/pom.xml
+++ b/exec/memory/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>exec-parent</artifactId>
<groupId>org.apache.drill.exec</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill.memory</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/pom.xml b/exec/pom.xml
index 0ad4d9c..e5b9f21 100644
--- a/exec/pom.xml
+++ b/exec/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill.exec</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/rpc/pom.xml
----------------------------------------------------------------------
diff --git a/exec/rpc/pom.xml b/exec/rpc/pom.xml
index 751a18a..ed8c6ef 100644
--- a/exec/rpc/pom.xml
+++ b/exec/rpc/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>exec-parent</artifactId>
<groupId>org.apache.drill.exec</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-rpc</artifactId>
<name>exec/rpc</name>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/exec/vector/pom.xml
----------------------------------------------------------------------
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 0109371..b7273b6 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>exec-parent</artifactId>
<groupId>org.apache.drill.exec</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>vector</artifactId>
<name>exec/Vectors</name>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/logical/pom.xml
----------------------------------------------------------------------
diff --git a/logical/pom.xml b/logical/pom.xml
index fbae5bc..eeee224 100644
--- a/logical/pom.xml
+++ b/logical/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-logical</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ae6f402..e76b547 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.drill</groupId>
<artifactId>drill-root</artifactId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Drill Root POM</name>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/protocol/pom.xml
----------------------------------------------------------------------
diff --git a/protocol/pom.xml b/protocol/pom.xml
index 94a2139..864696b 100644
--- a/protocol/pom.xml
+++ b/protocol/pom.xml
@@ -17,7 +17,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-protocol</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/tools/fmpp/pom.xml
----------------------------------------------------------------------
diff --git a/tools/fmpp/pom.xml b/tools/fmpp/pom.xml
index 95b13ce..15d707f 100644
--- a/tools/fmpp/pom.xml
+++ b/tools/fmpp/pom.xml
@@ -16,7 +16,7 @@
<parent>
<artifactId>tools-parent</artifactId>
<groupId>org.apache.drill.tools</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<artifactId>drill-fmpp-maven-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/drill/blob/7a151d69/tools/pom.xml
----------------------------------------------------------------------
diff --git a/tools/pom.xml b/tools/pom.xml
index a45674c..8007880 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill.tools</groupId>
[32/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java b/logical/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
new file mode 100644
index 0000000..611e70b
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import org.apache.drill.common.expression.LogicalExpression;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class JoinCondition {
+ private final String relationship;
+ private final LogicalExpression left;
+ private final LogicalExpression right;
+
+ @JsonCreator
+ public JoinCondition(@JsonProperty("relationship") String relationship,
+ @JsonProperty("left") LogicalExpression left, @JsonProperty("right") LogicalExpression right) {
+ super();
+ this.relationship = relationship;
+ this.left = left;
+ this.right = right;
+ }
+
+ public String getRelationship() {
+ return relationship;
+ }
+
+ public LogicalExpression getLeft() {
+ return left;
+ }
+
+ public LogicalExpression getRight() {
+ return right;
+ }
+
+ public JoinCondition flip(){
+ return new JoinCondition(relationship, right, left);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Limit.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Limit.java b/logical/src/main/java/org/apache/drill/common/logical/data/Limit.java
new file mode 100644
index 0000000..56ae8d9
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Limit.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Iterators;
+
+@JsonTypeName("limit")
+public class Limit extends SingleInputOperator {
+
+ private final Integer first;
+ private final Integer last;
+
+ @JsonCreator
+ public Limit(@JsonProperty("first") Integer first, @JsonProperty("last") Integer last) {
+ super();
+ this.first = first;
+ this.last = last;
+ }
+
+ public Integer getFirst() {
+ return first;
+ }
+
+ public Integer getLast() {
+ return last;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitLimit(this, value);
+ }
+
+ @Override
+ public NodeBuilder nodeBuilder() {
+ return new LimitNodeBuilder(); //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+ public static class LimitNodeBuilder implements NodeBuilder<Limit> {
+
+ @Override
+ public ObjectNode convert(ObjectMapper mapper, Limit operator, Integer inputId) {
+ ObjectNode limitNode = mapper.createObjectNode();
+ limitNode.put("op", "limit");
+ limitNode.put("input", inputId);
+ limitNode.put("first", operator.first);
+ limitNode.put("last", operator.last);
+ return limitNode;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java b/logical/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
new file mode 100644
index 0000000..3343d4e
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/LogicalOperator.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.drill.common.graph.GraphValue;
+import org.apache.drill.common.logical.ValidationError;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonIdentityInfo;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.ObjectIdGenerators;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+@JsonPropertyOrder({"@id", "memo", "input"}) // op will always be first since it is wrapped.
+@JsonIdentityInfo(generator = ObjectIdGenerators.IntSequenceGenerator.class, property = "@id")
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "op")
+public interface LogicalOperator extends GraphValue<LogicalOperator> {
+
+ public void setupAndValidate(List<LogicalOperator> operators, Collection<ValidationError> errors);
+
+ /**
+ * Provides capability to build a set of output based on traversing a query graph tree.
+ *
+ * @param logicalVisitor
+ * @return
+ */
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E;
+
+ public void registerAsSubscriber(LogicalOperator operator);
+
+ NodeBuilder nodeBuilder();
+
+ public interface NodeBuilder<T extends LogicalOperator> {
+ ObjectNode convert(ObjectMapper mapper, T operator, Integer inputId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java b/logical/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
new file mode 100644
index 0000000..a213c8d
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.drill.common.graph.GraphVisitor;
+import org.apache.drill.common.logical.ValidationError;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+
+public abstract class LogicalOperatorBase implements LogicalOperator{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalOperatorBase.class);
+
+ private List<LogicalOperator> children = new ArrayList<LogicalOperator>();
+
+ private String memo;
+
+ @Override
+ public final int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public void setupAndValidate(List<LogicalOperator> operators, Collection<ValidationError> errors) {
+ // TODO: remove this and implement individually.
+ }
+
+ @Override
+ public NodeBuilder nodeBuilder() {
+ // FIXME: Implement this on all logical operators
+ throw new UnsupportedOperationException("Not yet implemented.");
+ }
+
+ @Override
+ public void accept(GraphVisitor<LogicalOperator> visitor) {
+ if (visitor.enter(this)) {
+ visitor.leave(this);
+ }
+ }
+
+ @Override
+ public void registerAsSubscriber(LogicalOperator operator) {
+ if (operator == null) {
+ throw new IllegalArgumentException("You attempted to register a null operators.");
+ }
+ children.add(operator);
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getSimpleName() + " [memo=" + memo + "]";
+ }
+
+ @JsonInclude(Include.NON_EMPTY)
+ @JsonProperty("memo")
+ public String getMemo() {
+ return memo;
+ }
+
+ public void setMemo(String memo) {
+ this.memo = memo;
+ }
+
+ public static Set<Class<? extends LogicalOperator>> getSubTypes(final ScanResult classpathScan) {
+ final Set<Class<? extends LogicalOperator>> ops = classpathScan.getImplementations(LogicalOperator.class);
+ logger.debug("Found {} logical operator classes: {}.", ops.size(), ops);
+ return ops;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/NamedExpression.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/NamedExpression.java b/logical/src/main/java/org/apache/drill/common/logical/data/NamedExpression.java
new file mode 100644
index 0000000..a166c25
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/NamedExpression.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+@JsonPropertyOrder({"ref", "expr"})
+public class NamedExpression {
+ private final LogicalExpression expr;
+ private final FieldReference ref;
+
+ @JsonCreator
+ public NamedExpression(@JsonProperty("expr") LogicalExpression expr, @JsonProperty("ref") FieldReference ref) {
+ super();
+ this.expr = expr;
+ this.ref = ref;
+ }
+
+ public LogicalExpression getExpr() {
+ return expr;
+ }
+
+ public FieldReference getRef() {
+ return ref;
+ }
+
+ @Override
+ public String toString() {
+ return "NamedExpression [expr=" + expr + ", ref=" + ref + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Order.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Order.java b/logical/src/main/java/org/apache/drill/common/logical/data/Order.java
new file mode 100644
index 0000000..fca6010
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Order.java
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelFieldCollation.Direction;
+import org.apache.calcite.rel.RelFieldCollation.NullDirection;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+
+@JsonTypeName("order")
+public class Order extends SingleInputOperator {
+
+ private final Ordering[] orderings;
+ private final FieldReference within;
+
+ @JsonCreator
+ public Order(@JsonProperty("within") FieldReference within, @JsonProperty("orderings") Ordering... orderings) {
+ this.orderings = orderings;
+ this.within = within;
+ }
+
+ public Ordering[] getOrderings() {
+ return orderings;
+ }
+
+ public FieldReference getWithin() {
+ return within;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitOrder(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+
+ /**
+ * Representation of a SQL <sort specification>.
+ */
+ public static class Ordering {
+
+ private final LogicalExpression expr;
+ /** Net <ordering specification>. */
+ private final Direction direction;
+ /** Net <null ordering> */
+ private final NullDirection nullOrdering;
+
+ /**
+ * Constructs a sort specification.
+ * @param expr ...
+ * @param strOrderingSpec the <ordering specification> as string;
+ * allowed values: {@code "ASC"}, {@code "DESC"}, {@code null};
+ * null specifies default <ordering specification>
+ * ({@code "ASC"} / {@link Direction#ASCENDING})
+ * @param strNullOrdering the <null ordering> as string;
+ * allowed values: {@code "FIRST"}, {@code "LAST"},
+ * {@code "UNSPECIFIED"}, {@code null};
+ * null specifies default <null ordering>
+ * (omitted / {@link NullDirection#UNSPECIFIED}, interpreted later)
+ */
+ @JsonCreator
+ public Ordering( @JsonProperty("expr") LogicalExpression expr,
+ @JsonProperty("order") String strOrderingSpec,
+ @JsonProperty("nullDirection") String strNullOrdering ) {
+ this.expr = expr;
+ this.direction = getOrderingSpecFromString( strOrderingSpec );
+ this.nullOrdering = getNullOrderingFromString( strNullOrdering );
+ }
+
+ public Ordering(Direction direction, LogicalExpression e, NullDirection nullOrdering) {
+ this.expr = e;
+ this.direction = direction;
+ this.nullOrdering = nullOrdering;
+ }
+
+ public Ordering(Direction direction, LogicalExpression e) {
+ this(direction, e, NullDirection.FIRST);
+ }
+
+ private static Direction getOrderingSpecFromString( String strDirection ) {
+ final Direction direction;
+ if ( null == strDirection
+ || Direction.ASCENDING.shortString.equals( strDirection ) ) {
+ direction = Direction.ASCENDING;
+ }
+ else if ( Direction.DESCENDING.shortString.equals( strDirection ) ) {
+ direction = Direction.DESCENDING;
+ }
+ else {
+ throw new DrillRuntimeException(
+ "Unknown <ordering specification> string (not \"ASC\", \"DESC\", "
+ + "or null): \"" + strDirection + "\"" );
+ }
+ return direction;
+ }
+
+ private static NullDirection getNullOrderingFromString( String strNullOrdering ) {
+ final RelFieldCollation.NullDirection nullOrdering;
+ if ( null == strNullOrdering ) {
+ nullOrdering = NullDirection.UNSPECIFIED;
+ }
+ else {
+ try {
+ nullOrdering = NullDirection.valueOf( strNullOrdering );
+ }
+ catch ( IllegalArgumentException e ) {
+ throw new DrillRuntimeException(
+ "Internal error: Unknown <null ordering> string (not "
+ + "\"" + NullDirection.FIRST.name() + "\", "
+ + "\"" + NullDirection.LAST.name() + "\", or "
+ + "\"" + NullDirection.UNSPECIFIED.name() + "\" or null): "
+ + "\"" + strNullOrdering + "\"" );
+ }
+ }
+ return nullOrdering;
+ }
+
+ @Override
+ public String toString() {
+ return
+ super.toString()
+ + "[ "
+ + " expr = " + expr
+ + ", direction = " + direction
+ + ", nullOrdering = " + nullOrdering
+ + "] ";
+ }
+
+ @JsonIgnore
+ public Direction getDirection() {
+ return direction;
+ }
+
+ public LogicalExpression getExpr() {
+ return expr;
+ }
+
+ public String getOrder() {
+ switch (direction) {
+ case ASCENDING:
+ return Direction.ASCENDING.shortString;
+ case DESCENDING:
+ return Direction.DESCENDING.shortString;
+ default:
+ throw new DrillRuntimeException(
+ "Unexpected " + Direction.class.getName() + " value other than "
+ + Direction.ASCENDING + " or " + Direction.DESCENDING + ": "
+ + direction );
+ }
+ }
+
+ public NullDirection getNullDirection() {
+ return nullOrdering;
+ }
+
+ /**
+ * Reports whether NULL sorts high or low in this ordering.
+ *
+ * @return
+ * {@code true} if NULL sorts higher than any other value;
+ * {@code false} if NULL sorts lower than any other value
+ */
+ public boolean nullsSortHigh() {
+ final boolean nullsHigh;
+
+ switch (nullOrdering) {
+
+ case UNSPECIFIED:
+ // Default: NULL sorts high: like NULLS LAST if ASC, FIRST if DESC.
+ nullsHigh = true;
+ break;
+
+ case FIRST:
+ // FIRST: NULL sorts low with ASC, high with DESC.
+ nullsHigh = Direction.DESCENDING == getDirection();
+ break;
+
+ case LAST:
+ // LAST: NULL sorts high with ASC, low with DESC.
+ nullsHigh = Direction.ASCENDING == getDirection();
+ break;
+
+ default:
+ throw new DrillRuntimeException(
+ "Unexpected " + NullDirection.class.getName() + " value other than "
+ + NullDirection.FIRST + ", " + NullDirection.LAST + " or " + NullDirection.UNSPECIFIED + ": "
+ + nullOrdering );
+ }
+
+ return nullsHigh;
+ }
+
+ }
+
+ public static Builder builder(){
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractSingleBuilder<Order, Builder>{
+ private List<Ordering> orderings = Lists.newArrayList();
+ private FieldReference within;
+
+ public Builder setWithin(FieldReference within){
+ this.within = within;
+ return this;
+ }
+
+ public Builder addOrdering(Direction direction, LogicalExpression e, NullDirection collation){
+ orderings.add(new Ordering(direction, e, collation));
+ return this;
+ }
+
+ @Override
+ public Order internalBuild() {
+ return new Order(within, orderings.toArray(new Ordering[orderings.size()]));
+ }
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Project.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Project.java b/logical/src/main/java/org/apache/drill/common/logical/data/Project.java
new file mode 100644
index 0000000..4345c80
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Project.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExpressionParsingException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Lists;
+
+@JsonTypeName("project")
+public class Project extends SingleInputOperator {
+
+ private final NamedExpression[] selections;
+
+ @JsonCreator
+ public Project(@JsonProperty("projections") NamedExpression[] selections) {
+ this.selections = selections;
+ if (selections == null || selections.length == 0) {
+ throw new ExpressionParsingException(
+ "Project did not provide any projection selections. At least one projection must be provided.");
+// for (int i = 0; i < selections.length; i++) {
+// PathSegment segment = selections[i].getRef().getRootSegment();
+// CharSequence path = segment.getNameSegment().getPath();
+// if (!segment.isNamed() || !path.equals("output"))
+// throw new ExpressionParsingException(
+// String
+// .format(
+// "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]",
+// path, segment.isNamed()));
+//
+// }
+ }
+ }
+
+ @JsonProperty("projections")
+ public NamedExpression[] getSelections() {
+ return selections;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitProject(this, value);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractSingleBuilder<Project, Builder> {
+
+ private List<NamedExpression> exprs = Lists.newArrayList();
+
+ public Builder addExpr(NamedExpression expr) {
+ exprs.add(expr);
+ return this;
+ }
+
+ public Builder addExpr(FieldReference ref, LogicalExpression expr) {
+ exprs.add(new NamedExpression(expr, ref));
+ return this;
+ }
+
+ @Override
+ public Project internalBuild() {
+ return new Project(aN(exprs));
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/RunningAggregate.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/RunningAggregate.java b/logical/src/main/java/org/apache/drill/common/logical/data/RunningAggregate.java
new file mode 100644
index 0000000..e280627
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/RunningAggregate.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+
+@JsonTypeName("runningaggregate")
+public class RunningAggregate extends SingleInputOperator{
+
+ private final FieldReference within;
+ private final NamedExpression[] aggregations;
+
+ @JsonCreator
+ public RunningAggregate(@JsonProperty("within") FieldReference within, @JsonProperty("aggregations") NamedExpression[] aggregations) {
+ super();
+ this.within = within;
+ this.aggregations = aggregations;
+ }
+
+ public FieldReference getWithin() {
+ return within;
+ }
+
+ public NamedExpression[] getAggregations() {
+ return aggregations;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitRunningAggregate(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Scan.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Scan.java b/logical/src/main/java/org/apache/drill/common/logical/data/Scan.java
new file mode 100644
index 0000000..68eabbb
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Scan.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import org.apache.drill.common.JSONOptions;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("scan")
+public class Scan extends SourceOperator {
+ private final String storageEngine;
+ private final JSONOptions selection;
+
+ @JsonCreator
+ public Scan(@JsonProperty("storageengine") String storageEngine, @JsonProperty("selection") JSONOptions selection) {
+ super();
+ this.storageEngine = storageEngine;
+ this.selection = selection;
+ }
+
+ @JsonProperty("storageengine")
+ public String getStorageEngine() {
+ return storageEngine;
+ }
+
+ public JSONOptions getSelection() {
+ return selection;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitScan(this, value);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractBuilder<Scan>{
+ private String storageEngine;
+ private JSONOptions selection;
+
+ public Builder storageEngine(String storageEngine) {
+ this.storageEngine = storageEngine;
+ return this;
+ }
+
+ public Builder selection(JSONOptions selection) {
+ this.selection = selection;
+ return this;
+ }
+
+ @Override
+ public Scan build() {
+ return new Scan(storageEngine, selection);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Sequence.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Sequence.java b/logical/src/main/java/org/apache/drill/common/logical/data/Sequence.java
new file mode 100644
index 0000000..f04b09c
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Sequence.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.drill.common.logical.data.Sequence.De;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.annotation.ObjectIdGenerator;
+import com.fasterxml.jackson.annotation.ObjectIdGenerators;
+import com.fasterxml.jackson.core.JsonLocation;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.deser.impl.ReadableObjectId;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.google.common.collect.Iterators;
+
+/**
+ * Describes a list of operators where each operator only has one input and that
+ * input is the operator that came before.
+ *
+ */
+@JsonDeserialize(using = De.class)
+@JsonTypeName("sequence")
+public class Sequence extends LogicalOperatorBase {
+ static final Logger logger = LoggerFactory.getLogger(Sequence.class);
+
+ private Sequence() {}
+
+ public boolean openTop;
+ public LogicalOperator input;
+ @JsonProperty("do")
+ public LogicalOperator[] stream;
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitSequence(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(stream[stream.length - 1]);
+ }
+
+ public static class De extends StdDeserializer<LogicalOperator> {
+
+ protected De() {
+ super(Sequence.class);
+ }
+
+ @Override
+ public LogicalOperator deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
+ JsonProcessingException {
+ ObjectIdGenerator<Integer> idGenerator = new ObjectIdGenerators.IntSequenceGenerator();
+ JsonLocation start = jp.getCurrentLocation();
+ JsonToken t = jp.getCurrentToken();
+ LogicalOperator parent = null;
+ LogicalOperator first = null;
+ LogicalOperator prev = null;
+ Integer id = null;
+
+ while (true) {
+ String fieldName = jp.getText();
+ t = jp.nextToken();
+ switch (fieldName) { // switch on field names.
+ case "@id":
+ id = _parseIntPrimitive(jp, ctxt);
+ break;
+ case "input":
+ JavaType tp = ctxt.constructType(LogicalOperator.class);
+ JsonDeserializer<Object> d = ctxt.findRootValueDeserializer(tp);
+ parent = (LogicalOperator) d.deserialize(jp, ctxt);
+ break;
+
+ case "do":
+ if (!jp.isExpectedStartArrayToken()) {
+ throwE(
+ jp,
+ "The do parameter of sequence should be an array of SimpleOperators. Expected a JsonToken.START_ARRAY token but received a "
+ + t.name() + "token.");
+ }
+
+ int pos = 0;
+ while ((t = jp.nextToken()) != JsonToken.END_ARRAY) {
+ // logger.debug("Reading sequence child {}.", pos);
+ JsonLocation l = jp.getCurrentLocation(); // get current location
+ // first so we can
+ // correctly reference the
+ // start of the object in
+ // the case that the type
+ // is wrong.
+ LogicalOperator o = jp.readValueAs(LogicalOperator.class);
+
+ if (pos == 0) {
+ if (!(o instanceof SingleInputOperator) && !(o instanceof SourceOperator)) {
+ throwE(
+ l,
+ "The first operator in a sequence must be either a ZeroInput or SingleInput operator. The provided first operator was not. It was of type "
+ + o.getClass().getName());
+ }
+ first = o;
+ } else {
+ if (!(o instanceof SingleInputOperator)) {
+ throwE(l, "All operators after the first must be single input operators. The operator at position "
+ + pos + " was not. It was of type " + o.getClass().getName());
+ }
+ SingleInputOperator now = (SingleInputOperator) o;
+ now.setInput(prev);
+ }
+ prev = o;
+
+ pos++;
+ }
+ break;
+ default:
+ throwE(jp, "Unknown field name provided for Sequence: " + jp.getText());
+ }
+
+ t = jp.nextToken();
+ if (t == JsonToken.END_OBJECT) {
+ break;
+ }
+ }
+
+ if (first == null) {
+ throwE(start, "A sequence must include at least one operator.");
+ }
+ if ((parent == null && first instanceof SingleInputOperator)
+ || (parent != null && first instanceof SourceOperator)) {
+ throwE(start,
+ "A sequence must either start with a ZeroInputOperator or have a provided input. It cannot have both or neither.");
+ }
+
+ if (parent != null && first instanceof SingleInputOperator) {
+ ((SingleInputOperator) first).setInput(parent);
+ }
+
+ // set input reference.
+ if (id != null) {
+
+ ReadableObjectId rid = ctxt.findObjectId(id, idGenerator);
+ rid.bindItem(prev);
+ // logger.debug("Binding id {} to item {}.", rid.id, rid.item);
+
+ }
+
+ return first;
+ }
+
+ }
+
+ private static void throwE(JsonLocation l, String e) throws JsonParseException {
+ throw new JsonParseException(e, l);
+ }
+
+ private static void throwE(JsonParser jp, String e) throws JsonParseException {
+ throw new JsonParseException(e, jp.getCurrentLocation());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java b/logical/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
new file mode 100644
index 0000000..e46cb6b
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.logical.UnexpectedOperatorType;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Iterators;
+
+/**
+ * SimpleOperator is an operator that has one inputs at most.
+ */
+public abstract class SingleInputOperator extends LogicalOperatorBase {
+
+ private LogicalOperator input;
+
+ @JsonProperty("input")
+ public LogicalOperator getInput() {
+ return input;
+ }
+
+ @JsonProperty(value="input", required=true)
+ public void setInput(LogicalOperator input) {
+ if (input instanceof SinkOperator) {
+ throw new UnexpectedOperatorType("You have set the input of a sink node of type ["+input.getClass().getSimpleName()+ "] as the input for another node of type ["+this.getClass().getSimpleName()+ "]. This is invalid.");
+ }
+ this.input = input;
+ input.registerAsSubscriber(this);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(input);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/SinkOperator.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/SinkOperator.java b/logical/src/main/java/org/apache/drill/common/logical/data/SinkOperator.java
new file mode 100644
index 0000000..2ca4006
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/SinkOperator.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+
+/**
+ * An operator that cannot be subscribed to.
+ */
+public abstract class SinkOperator extends SingleInputOperator{
+
+ @Override
+ public void registerAsSubscriber(LogicalOperator operator) {
+ throw new UnsupportedOperationException("You can not subscribe to a sink operator such as " + this.getClass().getSimpleName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/SourceOperator.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/SourceOperator.java b/logical/src/main/java/org/apache/drill/common/logical/data/SourceOperator.java
new file mode 100644
index 0000000..d4f8246
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/SourceOperator.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+
+import com.google.common.collect.Iterators;
+
+
+/**
+ * An operator that produces data without any parents. (zero input operator)
+ */
+public abstract class SourceOperator extends LogicalOperatorBase{
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Store.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Store.java b/logical/src/main/java/org/apache/drill/common/logical/data/Store.java
new file mode 100644
index 0000000..8f99eed
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Store.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.JSONOptions;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+
+@JsonTypeName("store")
+public class Store extends SinkOperator{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Store.class);
+
+ private final String storageEngine;
+ private final JSONOptions target;
+
+ @JsonCreator
+ public Store(@JsonProperty("storageengine") String storageEngine, @JsonProperty("target") JSONOptions target) {
+ super();
+ this.storageEngine = storageEngine;
+ this.target = target;
+ }
+
+ public String getStorageEngine() {
+ return storageEngine;
+ }
+
+ public JSONOptions getTarget() {
+ return target;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitStore(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractSingleBuilder<Store, Builder>{
+ private String storageEngine;
+ private JSONOptions target;
+
+ public Builder storageEngine(String storageEngine) {
+ this.storageEngine = storageEngine;
+ return this;
+ }
+
+ public Builder target(JSONOptions target) {
+ this.target = target;
+ return this;
+ }
+
+ @Override
+ public Store internalBuild() {
+ return new Store(storageEngine, target);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Transform.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Transform.java b/logical/src/main/java/org/apache/drill/common/logical/data/Transform.java
new file mode 100644
index 0000000..129ae2c
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Transform.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+
+@JsonTypeName("transform")
+public class Transform extends SingleInputOperator {
+
+ private final NamedExpression[] transforms;
+
+ @JsonCreator
+ public Transform(@JsonProperty("transforms") NamedExpression[] transforms) {
+ super();
+ this.transforms = transforms;
+ }
+
+ public NamedExpression[] getTransforms() {
+ return transforms;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitTransform(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Union.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Union.java b/logical/src/main/java/org/apache/drill/common/logical/data/Union.java
new file mode 100644
index 0000000..113ffe0
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Union.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+
+@JsonTypeName("union")
+public class Union extends LogicalOperatorBase {
+ private final LogicalOperator[] inputs;
+ private final boolean distinct;
+
+// @JsonCreator
+// public Union(@JsonProperty("inputs") LogicalOperator[] inputs){
+// this(inputs, false);
+// }
+
+ @JsonCreator
+ public Union(@JsonProperty("inputs") LogicalOperator[] inputs, @JsonProperty("distinct") Boolean distinct){
+ this.inputs = inputs;
+ for (LogicalOperator o : inputs) {
+ o.registerAsSubscriber(this);
+ }
+ this.distinct = distinct == null ? false : distinct;
+ }
+
+ public LogicalOperator[] getInputs() {
+ return inputs;
+ }
+
+ public boolean isDistinct() {
+ return distinct;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitUnion(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.forArray(inputs);
+ }
+
+
+ public static Builder builder(){
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractBuilder<Union>{
+ private List<LogicalOperator> inputs = Lists.newArrayList();
+ private boolean distinct;
+
+ public Builder addInput(LogicalOperator o){
+ inputs.add(o);
+ return this;
+ }
+
+ public Builder setDistinct(boolean distinct){
+ this.distinct = distinct;
+ return this;
+ }
+
+ @Override
+ public Union build() {
+ return new Union(inputs.toArray(new LogicalOperator[inputs.size()]), distinct);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Values.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Values.java b/logical/src/main/java/org/apache/drill/common/logical/data/Values.java
new file mode 100644
index 0000000..9276e55
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Values.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import org.apache.drill.common.JSONOptions;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.core.JsonLocation;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Preconditions;
+
+@JsonTypeName("values")
+public class Values extends SourceOperator {
+
+ private final JSONOptions content;
+
+ @JsonCreator
+ public Values(@JsonProperty("content") JSONOptions content){
+ super();
+ this.content = content;
+ Preconditions.checkNotNull(content, "content attribute is required for source operator 'constant'.");
+ }
+
+ public JSONOptions getContent() {
+ return content;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitValues(this, value);
+ }
+
+ public static Builder builder(){
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractBuilder<Values>{
+ private JSONOptions content;
+
+ public Builder content(JsonNode n){
+ content = new JSONOptions(n, JsonLocation.NA);
+ return this;
+ }
+
+ @Override
+ public Values build() {
+ return new Values(content);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Window.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Window.java b/logical/src/main/java/org/apache/drill/common/logical/data/Window.java
new file mode 100644
index 0000000..0f550dc
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Window.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import org.apache.drill.common.expression.FieldReference;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+@JsonTypeName("window")
+public class Window extends SingleInputOperator {
+ private final NamedExpression[] withins;
+ private final NamedExpression[] aggregations;
+ private final Order.Ordering[] orderings;
+ private final long start;
+ private final long end;
+
+
+ @JsonCreator
+ public Window(@JsonProperty("withins") NamedExpression[] withins,
+ @JsonProperty("aggregations") NamedExpression[] aggregations,
+ @JsonProperty("orderings") Order.Ordering[] orderings,
+ @JsonProperty("start") Long start,
+ @JsonProperty("end") Long end) {
+ super();
+ this.withins = withins;
+ this.start = start == null ? Long.MIN_VALUE : start;
+ this.end = end == null ? Long.MIN_VALUE : end;
+ this.aggregations = aggregations;
+ this.orderings = orderings;
+ }
+
+ public NamedExpression[] getWithins() {
+ return withins;
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public long getEnd() {
+ return end;
+ }
+
+ public NamedExpression[] getAggregations() {
+ return aggregations;
+ }
+
+ public Order.Ordering[] getOrderings() {
+ return orderings;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitWindow(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+ public static class Builder extends AbstractSingleBuilder<Window, Builder>{
+ private List<NamedExpression> aggregations = Lists.newArrayList();
+ private List<NamedExpression> withins = Lists.newArrayList();
+ private List<Order.Ordering> orderings = Lists.newArrayList();
+ private long start = Long.MIN_VALUE;
+ private long end = Long.MIN_VALUE;
+
+
+ public Builder addAggregation(FieldReference ref, LogicalExpression expr){
+ aggregations.add(new NamedExpression(expr, ref));
+ return this;
+ }
+
+ public Builder addWithin(FieldReference within, LogicalExpression expr) {
+ withins.add(new NamedExpression(expr, within));
+ return this;
+ }
+
+ public Window internalBuild() {
+ //TODO withins can actually be empty: over(), over(order by <expression>), ...
+ checkState(!withins.isEmpty(), "Withins in window must not be empty.");
+ checkState(!aggregations.isEmpty(), "Aggregations in window must not be empty.");
+ return new Window(aN(withins), aN(aggregations), aO(orderings), start, end);
+ }
+
+ public Builder addOrdering(Order.Ordering ordering) {
+ orderings.add(ordering);
+ return this;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/Writer.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/Writer.java b/logical/src/main/java/org/apache/drill/common/logical/data/Writer.java
new file mode 100644
index 0000000..6e51df7
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/Writer.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("writer")
+public class Writer extends SingleInputOperator {
+
+ // TODO: Hack - Make the type generic until "common" and "java-exec" modules are merged (DRILL-507).
+ private final Object createTableEntry;
+
+ @JsonCreator
+ public Writer(@JsonProperty("createTableEntry") Object createTableEntry) {
+ this.createTableEntry = createTableEntry;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitWriter(this, value);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractSingleBuilder<Writer, Builder> {
+
+ // TODO: Hack - Make the type generic until "common" and "java-exec" modules are merged (DRILL-507).
+ private Object createTableEntry;
+
+ public Builder setCreateTableEntry(Object createTableEntry) {
+ this.createTableEntry = createTableEntry;
+ return this;
+ }
+
+ @Override
+ public Writer internalBuild() {
+ return new Writer(createTableEntry);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/package-info.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/package-info.java b/logical/src/main/java/org/apache/drill/common/logical/data/package-info.java
new file mode 100644
index 0000000..fc76c0d
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Logical operators in a query graph.
+ */
+package org.apache.drill.common.logical.data;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java b/logical/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
new file mode 100644
index 0000000..165ee66
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data.visitors;
+
+import org.apache.drill.common.logical.data.Values;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.Flatten;
+import org.apache.drill.common.logical.data.GroupingAggregate;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.RunningAggregate;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Sequence;
+import org.apache.drill.common.logical.data.Store;
+import org.apache.drill.common.logical.data.Transform;
+import org.apache.drill.common.logical.data.Union;
+import org.apache.drill.common.logical.data.Window;
+import org.apache.drill.common.logical.data.Writer;
+
+
+public abstract class AbstractLogicalVisitor<T, X, E extends Throwable> implements LogicalVisitor<T, X, E> {
+
+ public T visitOp(LogicalOperator op, X value) throws E{
+ throw new UnsupportedOperationException(String.format(
+ "The LogicalVisitor of type %s does not currently support visiting the PhysicalOperator type %s.", this
+ .getClass().getCanonicalName(), op.getClass().getCanonicalName()));
+ }
+
+ @Override
+ public T visitScan(Scan scan, X value) throws E {
+ return visitOp(scan, value);
+ }
+
+ @Override
+ public T visitStore(Store store, X value) throws E {
+ return visitOp(store, value);
+ }
+
+ @Override
+ public T visitFilter(Filter filter, X value) throws E {
+ return visitOp(filter, value);
+ }
+
+ @Override
+ public T visitFlatten(Flatten flatten, X value) throws E {
+ return visitOp(flatten, value);
+ }
+
+ @Override
+ public T visitProject(Project project, X value) throws E {
+ return visitOp(project, value);
+ }
+
+ @Override
+ public T visitOrder(Order order, X value) throws E {
+ return visitOp(order, value);
+ }
+
+ @Override
+ public T visitJoin(Join join, X value) throws E {
+ return visitOp(join, value);
+ }
+
+ @Override
+ public T visitLimit(Limit limit, X value) throws E {
+ return visitOp(limit, value);
+ }
+
+ @Override
+ public T visitRunningAggregate(RunningAggregate runningAggregate, X value) throws E {
+ return visitOp(runningAggregate, value);
+ }
+
+ @Override
+ public T visitGroupingAggregate(GroupingAggregate groupBy, X value) throws E {
+ return visitOp(groupBy, value);
+ }
+
+ @Override
+ public T visitSequence(Sequence sequence, X value) throws E {
+ return visitOp(sequence, value);
+ }
+
+ @Override
+ public T visitTransform(Transform transform, X value) throws E {
+ return visitOp(transform, value);
+ }
+
+ @Override
+ public T visitUnion(Union union, X value) throws E {
+ return visitOp(union, value);
+ }
+
+ @Override
+ public T visitWindow(Window window, X value) throws E {
+ return visitOp(window, value);
+ }
+
+ @Override
+ public T visitValues(Values constant, X value) throws E {
+ return visitOp(constant, value);
+ }
+
+ @Override
+ public T visitWriter(Writer writer, X value) throws E {
+ return visitOp(writer, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java b/logical/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
new file mode 100644
index 0000000..1e07dc6
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data.visitors;
+
+
+import org.apache.drill.common.logical.data.Values;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.Flatten;
+import org.apache.drill.common.logical.data.GroupingAggregate;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.RunningAggregate;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Sequence;
+import org.apache.drill.common.logical.data.Store;
+import org.apache.drill.common.logical.data.Transform;
+import org.apache.drill.common.logical.data.Union;
+import org.apache.drill.common.logical.data.Window;
+import org.apache.drill.common.logical.data.Window;
+import org.apache.drill.common.logical.data.Writer;
+
+/**
+ * Visitor class designed to traversal of a operator tree. Basis for a number of operator manipulations including fragmentation and materialization.
+ * @param <RETURN> The class associated with the return of each visit method.
+ * @param <EXTRA> The class object associated with additional data required for a particular operator modification.
+ * @param <EXCEP> An optional exception class that can be thrown when a portion of a modification or traversal fails. Must extend Throwable. In the case where the visitor does not throw any caught exception, this can be set as RuntimeException.
+ */
+public interface LogicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalVisitor.class);
+
+
+ public RETURN visitScan(Scan scan, EXTRA value) throws EXCEP;
+ public RETURN visitStore(Store store, EXTRA value) throws EXCEP;
+ public RETURN visitGroupingAggregate(GroupingAggregate groupBy, EXTRA value) throws EXCEP;
+ public RETURN visitFilter(Filter filter, EXTRA value) throws EXCEP;
+ public RETURN visitFlatten(Flatten flatten, EXTRA value) throws EXCEP;
+
+ public RETURN visitProject(Project project, EXTRA value) throws EXCEP;
+ public RETURN visitValues(Values constant, EXTRA value) throws EXCEP;
+ public RETURN visitOrder(Order order, EXTRA value) throws EXCEP;
+ public RETURN visitJoin(Join join, EXTRA value) throws EXCEP;
+ public RETURN visitLimit(Limit limit, EXTRA value) throws EXCEP;
+ public RETURN visitRunningAggregate(RunningAggregate runningAggregate, EXTRA value) throws EXCEP;
+ public RETURN visitSequence(Sequence sequence, EXTRA value) throws EXCEP;
+ public RETURN visitTransform(Transform transform, EXTRA value) throws EXCEP;
+ public RETURN visitUnion(Union union, EXTRA value) throws EXCEP;
+ public RETURN visitWindow(Window window, EXTRA value) throws EXCEP;
+ public RETURN visitWriter(Writer writer, EXTRA value) throws EXCEP;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/data/visitors/package-info.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/data/visitors/package-info.java b/logical/src/main/java/org/apache/drill/common/logical/data/visitors/package-info.java
new file mode 100644
index 0000000..9b82bb3
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/data/visitors/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Visitors for Drill logical plan graphs.
+ */
+package org.apache.drill.common.logical.data.visitors;
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/logical/package-info.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/logical/package-info.java b/logical/src/main/java/org/apache/drill/common/logical/package-info.java
new file mode 100644
index 0000000..1f88f06
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/logical/package-info.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Storage plugin interfaces and logical plan representation.
+ *
+ * This package includes the interface for storage and format plugins for Drill
+ * as well as the top level class for representing a Drill logical plan and a
+ * builder for constructing plans.
+ */
+package org.apache.drill.common.logical;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/main/java/org/apache/drill/common/util/AbstractDynamicBean.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/util/AbstractDynamicBean.java b/logical/src/main/java/org/apache/drill/common/util/AbstractDynamicBean.java
new file mode 100644
index 0000000..07a3620
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/util/AbstractDynamicBean.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.util;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.drill.common.exceptions.LogicalPlanParsingException;
+
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * Captures all properties and turns them into an object node for late bind conversion.
+ */
+public abstract class AbstractDynamicBean {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractDynamicBean.class);
+
+ private static volatile ObjectMapper MAPPER;
+
+ private ObjectNode objectNode = new ObjectNode(null);
+
+ @JsonAnySetter
+ public void _anySetter(String name, JsonNode value){
+ objectNode.put(name, value);
+ }
+
+ @JsonAnyGetter
+ public Map<String,JsonNode> _anyGetter() {
+ Map<String, JsonNode> unknowns = new HashMap<String, JsonNode>();
+
+ for(Iterator<Entry<String, JsonNode>> i = objectNode.fields(); i.hasNext(); ){
+ Entry<String, JsonNode> e = i.next();
+ unknowns.put(e.getKey(), e.getValue());
+ }
+ return unknowns;
+ }
+
+ public <T> T getWith(Class<T> c){
+ try {
+ return getMapper().treeToValue(objectNode, c);
+ } catch (JsonProcessingException e) {
+ throw new LogicalPlanParsingException(String.format("Failure while trying to convert late bound json type to type of %s.", c.getCanonicalName()), e);
+ }
+ }
+
+
+ private static synchronized ObjectMapper getMapper(){
+ if(MAPPER == null){
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.enable(SerializationFeature.INDENT_OUTPUT);
+ mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+ mapper.configure(Feature.ALLOW_COMMENTS, true);
+ MAPPER = mapper;
+ }
+ return MAPPER;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/logical/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java
----------------------------------------------------------------------
diff --git a/logical/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java b/logical/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java
new file mode 100644
index 0000000..97cd578
--- /dev/null
+++ b/logical/src/test/java/org/apache/drill/common/expression/PathSegmentTests.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.drill.test.DrillTest;
+import org.junit.Test;
+
+public class PathSegmentTests extends DrillTest {
+ protected PathSegment makeArraySegment(final int len, final PathSegment tail) {
+ PathSegment node = tail;
+ for (int i = 0; i < len; i++) {
+ node = new PathSegment.ArraySegment(node);
+ }
+ return node;
+ }
+
+ @Test
+ public void testIfMultiLevelCloneWorks() {
+ final int levels = 10;
+ final PathSegment segment = new PathSegment.NameSegment("test", makeArraySegment(levels, null));
+ final PathSegment clone = segment.clone();
+ assertEquals("result of clone & original segments must be identical", segment, clone);
+
+ final PathSegment tail = new PathSegment.NameSegment("tail");
+ final PathSegment newSegment = new PathSegment.NameSegment("test", makeArraySegment(levels, tail));
+ final PathSegment newClone = segment.cloneWithNewChild(tail);
+ assertEquals("result of cloneWithChild & original segment must be identical", newSegment, newClone);
+ }
+}
[38/45] drill git commit: DRILL-3987: (MOVE) Move logical expressions
and operators out of common. Move to new drill-logical model.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
deleted file mode 100644
index eb1394c..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-import org.apache.drill.common.expression.BooleanOperator;
-import org.apache.drill.common.expression.CastExpression;
-import org.apache.drill.common.expression.ConvertExpression;
-import org.apache.drill.common.expression.ErrorCollector;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionHolderExpression;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.IfExpression.IfCondition;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.NullExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.TypedNullConstant;
-import org.apache.drill.common.expression.ValueExpressions;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DateExpression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
-import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, RuntimeException> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionValidator.class);
-
- @Override
- public Void visitFunctionCall(FunctionCall call, ErrorCollector errors) throws RuntimeException {
- // we throw an exception here because this is a fundamental operator programming problem as opposed to an expression
- // problem. At this point in an expression's lifecycle, all function calls should have been converted into
- // FunctionHolders.
- throw new UnsupportedOperationException("FunctionCall is not expected here. "
- + "It should have been converted to FunctionHolderExpression in materialization");
- }
-
- @Override
- public Void visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors)
- throws RuntimeException {
- // make sure aggregate functions are not nested inside aggregate functions
- AggregateChecker.isAggregating(holder, errors);
-
- // make sure arguments are constant if the function implementation expects constants for any arguments
- ConstantChecker.checkConstants(holder, errors);
-
- return null;
- }
-
- @Override
- public Void visitBooleanOperator(BooleanOperator op, ErrorCollector errors) throws RuntimeException {
- int i = 0;
- for (LogicalExpression arg : op.args) {
- if ( arg.getMajorType().getMinorType() != MinorType.BIT) {
- errors
- .addGeneralError(
- arg.getPosition(),
- String
- .format(
- "Failure composing boolean operator %s. All conditions must return a boolean type. Condition %d was of Type %s.",
- op.getName(), i, arg.getMajorType().getMinorType()));
- }
- i++;
- }
-
- return null;
- }
-
- @Override
- public Void visitIfExpression(IfExpression ifExpr, ErrorCollector errors) throws RuntimeException {
- // confirm that all conditions are required boolean values.
- IfCondition cond = ifExpr.ifCondition;
- MajorType majorType = cond.condition.getMajorType();
- if ( majorType
- .getMinorType() != MinorType.BIT) {
- errors
- .addGeneralError(
- cond.condition.getPosition(),
- String
- .format(
- "Failure composing If Expression. All conditions must return a boolean type. Condition was of Type %s.",
- majorType.getMinorType()));
- }
-
- // confirm that all outcomes are the same type.
- final MajorType mt = ifExpr.elseExpression.getMajorType();
- cond = ifExpr.ifCondition;
- MajorType innerT = cond.expression.getMajorType();
- if ((innerT.getMode() == DataMode.REPEATED && mt.getMode() != DataMode.REPEATED) || //
- ((innerT.getMinorType() != mt.getMinorType()) &&
- (innerT.getMode() != DataMode.OPTIONAL && mt.getMode() != DataMode.OPTIONAL &&
- (innerT.getMinorType() != MinorType.NULL && mt.getMinorType() != MinorType.NULL)))) {
- errors
- .addGeneralError(
- cond.condition.getPosition(),
- String
- .format(
- "Failure composing If Expression. All expressions must return the same MinorType as the else expression. The if condition returned type type %s but the else expression was of type %s",
- innerT, mt));
- }
- return null;
- }
-
- @Override
- public Void visitSchemaPath(SchemaPath path, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitIntConstant(ValueExpressions.IntExpression intExpr, ErrorCollector value) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitFloatConstant(ValueExpressions.FloatExpression fExpr, ErrorCollector value) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitLongConstant(LongExpression intExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitDecimal9Constant(Decimal9Expression decExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitDecimal18Constant(Decimal18Expression decExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitDecimal28Constant(Decimal28Expression decExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitDecimal38Constant(Decimal38Expression decExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitDateConstant(DateExpression intExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitTimeConstant(TimeExpression intExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitIntervalYearConstant(IntervalYearExpression intExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitIntervalDayConstant(IntervalDayExpression intExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitTimeStampConstant(TimeStampExpression intExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitBooleanConstant(BooleanExpression e, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitQuotedStringConstant(QuotedString e, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitUnknown(LogicalExpression e, ErrorCollector value) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitCastExpression(CastExpression e, ErrorCollector value) throws RuntimeException {
- return e.getInput().accept(this, value);
- }
-
- @Override
- public Void visitNullConstant(TypedNullConstant e, ErrorCollector value) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitNullExpression(NullExpression e, ErrorCollector value) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitConvertExpression(ConvertExpression e, ErrorCollector value)
- throws RuntimeException {
- return e.getInput().accept(this, value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/OpVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/OpVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/OpVisitor.java
deleted file mode 100644
index b3c7879..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/OpVisitor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-import org.apache.drill.common.graph.GraphVisitor;
-import org.apache.drill.common.logical.data.LogicalOperator;
-
-public interface OpVisitor extends GraphVisitor<LogicalOperator> {
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
deleted file mode 100644
index 5f83e93..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.visitors;
-
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionHolderExpression;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DateExpression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
-import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
-import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
-
-public abstract class SimpleExprVisitor<T> implements ExprVisitor<T, Void, RuntimeException>{
-
- @Override
- public T visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
- return visitFunctionCall(call);
- }
-
- @Override
- public T visitFunctionHolderExpression(FunctionHolderExpression holder, Void value) throws RuntimeException {
- return visitFunctionHolderExpression(holder);
- }
-
- @Override
- public T visitIfExpression(IfExpression ifExpr, Void value) throws RuntimeException {
- return visitIfExpression(ifExpr);
- }
-
- @Override
- public T visitSchemaPath(SchemaPath path, Void value) throws RuntimeException {
- return visitSchemaPath(path);
- }
-
- @Override
- public T visitIntConstant(IntExpression intExpr, Void value) throws RuntimeException {
- return visitIntConstant(intExpr);
- }
-
- @Override
- public T visitFloatConstant(FloatExpression fExpr, Void value) throws RuntimeException {
- return visitFloatConstant(fExpr);
- }
-
- @Override
- public T visitLongConstant(LongExpression intExpr, Void value) throws RuntimeException {
- return visitLongConstant(intExpr);
- }
-
- @Override
- public T visitDateConstant(DateExpression intExpr, Void value) throws RuntimeException {
- return visitDateConstant(intExpr);
- }
-
-
- @Override
- public T visitTimeConstant(TimeExpression intExpr, Void value) throws RuntimeException {
- return visitTimeConstant(intExpr);
- }
-
- @Override
- public T visitIntervalYearConstant(IntervalYearExpression intExpr, Void value) throws RuntimeException {
- return visitIntervalYearConstant(intExpr);
- }
-
- @Override
- public T visitIntervalDayConstant(IntervalDayExpression intExpr, Void value) throws RuntimeException {
- return visitIntervalDayConstant(intExpr);
- }
-
- @Override
- public T visitTimeStampConstant(TimeStampExpression intExpr, Void value) throws RuntimeException {
- return visitTimeStampConstant(intExpr);
- }
-
- @Override
- public T visitDecimal9Constant(Decimal9Expression decExpr, Void value) throws RuntimeException {
- return visitDecimal9Constant(decExpr);
- }
-
- @Override
- public T visitDecimal18Constant(Decimal18Expression decExpr, Void value) throws RuntimeException {
- return visitDecimal18Constant(decExpr);
- }
-
- @Override
- public T visitDecimal28Constant(Decimal28Expression decExpr, Void value) throws RuntimeException {
- return visitDecimal28Constant(decExpr);
- }
-
- @Override
- public T visitDecimal38Constant(Decimal38Expression decExpr, Void value) throws RuntimeException {
- return visitDecimal38Constant(decExpr);
- }
-
- @Override
- public T visitDoubleConstant(DoubleExpression dExpr, Void value) throws RuntimeException {
- return visitDoubleConstant(dExpr);
- }
-
- @Override
- public T visitBooleanConstant(BooleanExpression e, Void value) throws RuntimeException {
- return visitBooleanConstant(e);
- }
-
- @Override
- public T visitQuotedStringConstant(QuotedString e, Void value) throws RuntimeException {
- return visitQuotedStringConstant(e);
- }
-
-
- public abstract T visitFunctionCall(FunctionCall call);
- public abstract T visitFunctionHolderExpression(FunctionHolderExpression call);
- public abstract T visitIfExpression(IfExpression ifExpr);
- public abstract T visitSchemaPath(SchemaPath path);
- public abstract T visitIntConstant(IntExpression intExpr);
- public abstract T visitFloatConstant(FloatExpression fExpr);
- public abstract T visitLongConstant(LongExpression intExpr);
- public abstract T visitDateConstant(DateExpression intExpr);
- public abstract T visitTimeConstant(TimeExpression intExpr);
- public abstract T visitIntervalYearConstant(IntervalYearExpression intExpr);
- public abstract T visitIntervalDayConstant(IntervalDayExpression intExpr);
- public abstract T visitTimeStampConstant(TimeStampExpression intExpr);
- public abstract T visitDecimal9Constant(Decimal9Expression intExpr);
- public abstract T visitDecimal18Constant(Decimal18Expression intExpr);
- public abstract T visitDecimal28Constant(Decimal28Expression intExpr);
- public abstract T visitDecimal38Constant(Decimal38Expression intExpr);
- public abstract T visitDoubleConstant(DoubleExpression dExpr);
- public abstract T visitBooleanConstant(BooleanExpression e);
- public abstract T visitQuotedStringConstant(QuotedString e);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/expression/visitors/package-info.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/package-info.java b/common/src/main/java/org/apache/drill/common/expression/visitors/package-info.java
deleted file mode 100644
index b57362e..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Visitors for validating or optimizing logical expressions.
- *
- * These visitors are used to provide feedback to users on invalid expressions
- * as well as some basic optimizations applied during planning.
- */
-package org.apache.drill.common.expression.visitors;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/AdjacencyList.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/AdjacencyList.java b/common/src/main/java/org/apache/drill/common/graph/AdjacencyList.java
deleted file mode 100644
index a4edfc1..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/AdjacencyList.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.graph;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Multimaps;
-
-class AdjacencyList<V extends GraphValue<V>> {
- private Set<Node> allNodes = new HashSet<Node>();
- private ListMultimap<Node, Edge<Node>> adjacencies = ArrayListMultimap.create();
-
- void addEdge(Node source, Node target, int weight) {
- adjacencies.put(source, new Edge<Node>(source, target, weight));
- allNodes.add(source);
- allNodes.add(target);
- }
-
- void clearVisited() {
- for (Edge<Node> e : adjacencies.values()) {
- e.from.visited = false;
- e.to.visited = false;
- }
- }
-
- Node getNewNode(V value) {
- return new Node(value);
- }
-
- public List<Edge<Node>> getAdjacent(AdjacencyList<V>.Node source) {
- return adjacencies.get(source);
- }
-
- public void printEdges() {
- for (Edge<Node> e : adjacencies.values()) {
- System.out.println(e.from.index + " -> " + e.to.index);
- }
- }
-
- public AdjacencyList<V> getReversedList() {
- AdjacencyList<V> newlist = new AdjacencyList<V>();
- for (Edge<Node> e : adjacencies.values()) {
- newlist.addEdge(e.to, e.from, e.weight);
- }
- return newlist;
- }
-
- public Set<Node> getNodeSet() {
- return adjacencies.keySet();
- }
-
-
- Collection<Node> getInternalLeafNodes() {
- // we have to use the allNodes list as otherwise destination only nodes won't be found.
- List<Node> nodes = new LinkedList<Node>(allNodes);
-
- for (Iterator<Node> i = nodes.iterator(); i.hasNext();) {
- final Node n = i.next();
-
- // remove any nodes that have one or more outbound edges.
- List<Edge<Node>> adjList = this.getAdjacent(n);
- if (adjList != null && !adjList.isEmpty()) {
- i.remove();
- }
-
- }
- return nodes;
- }
-
- /**
- * Get a list of nodes that have no outbound edges.
- *
- * @return
- */
- public Collection<V> getLeafNodes() {
- return convert(getInternalLeafNodes());
- }
-
-
- Collection<Node> getInternalRootNodes() {
- Set<Node> nodes = new HashSet<Node>(getNodeSet());
- for (Edge<Node> e : adjacencies.values()) {
- nodes.remove(e.to);
- }
- return nodes;
- }
-
- /**
- * Get a list of all nodes that have no incoming edges.
- *
- * @return
- */
- public List<V> getRootNodes() {
- return convert(getInternalRootNodes());
- }
-
- public Collection<Edge<Node>> getAllEdges() {
- return adjacencies.values();
- }
-
- public void fix(boolean requireDirected) {
- adjacencies = Multimaps.unmodifiableListMultimap(adjacencies);
- allNodes = Collections.unmodifiableSet(allNodes);
-
- if (requireDirected) {
- List<List<Node>> cyclicReferences = GraphAlgos.checkDirected(this);
- if (cyclicReferences.size() > 0) {
- throw new IllegalArgumentException(
- "A logical plan must be a valid DAG. You have cyclic references in your graph. " + cyclicReferences);
- }
- }
- }
-
- List<V> convert(Collection<Node> nodes) {
- List<V> out = new ArrayList<V>(nodes.size());
- for (Node o : nodes) {
- out.add(o.getNodeValue());
- }
- return out;
- }
-
- class Node implements Comparable<Node> {
- final V nodeValue;
- boolean visited = false; // used for Kosaraju's algorithm and Edmonds's
- // algorithm
- int lowlink = -1; // used for Tarjan's algorithm
- int index = -1; // used for Tarjan's algorithm
-
- public Node(final V operator) {
- if (operator == null) {
- throw new IllegalArgumentException("Operator node was null.");
- }
- this.nodeValue = operator;
- }
-
- public int compareTo(final Node argNode) {
- // just do an identity compare since elsewhere you should ensure that only one node exists for each nodeValue.
- return argNode == this ? 0 : -1;
- }
-
- @Override
- public int hashCode() {
- return nodeValue.hashCode();
- }
-
- public V getNodeValue() {
- return nodeValue;
- }
-
- @Override
- public String toString() {
- return "Node [val=" + nodeValue + "]";
- }
-
- }
-
- public static <V extends GraphValue<V>> AdjacencyList<V> newInstance(Collection<V> nodes) {
- AdjacencyList<V> list = new AdjacencyList<V>();
- AdjacencyListBuilder<V> builder = new AdjacencyListBuilder<V>(list);
- for (V v : nodes) {
- v.accept(builder);
- }
- return builder.getAdjacencyList();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/AdjacencyListBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/AdjacencyListBuilder.java b/common/src/main/java/org/apache/drill/common/graph/AdjacencyListBuilder.java
deleted file mode 100644
index e90d70a..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/AdjacencyListBuilder.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.graph;
-
-import java.util.HashMap;
-import java.util.Map;
-
- class AdjacencyListBuilder<V extends GraphValue<V>> implements GraphVisitor<V> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AdjacencyListBuilder.class);
-
- private Map<V, AdjacencyList<V>.Node> ops = new HashMap<V, AdjacencyList<V>.Node>();
- private final AdjacencyList<V> parent;
-
- public AdjacencyListBuilder(AdjacencyList<V> parent) {
- this.parent = parent;
- }
-
- protected boolean requireDirected() {
- return true;
- }
-
- public boolean enter(V o) {
- visit(o);
- return true;
- }
-
- @Override
- public void leave(V o) {
- }
-
- @Override
- public boolean visit(V o) {
- if (o == null) {
- throw new IllegalArgumentException("Null operator.");
- }
-
- if (!ops.containsKey(o)) {
- ops.put(o, parent.getNewNode(o));
- return true;
- }
-
- return true;
- }
-
- public AdjacencyList<V> getAdjacencyList() {
-// logger.debug("Values; {}", ops.values().toArray());
- AdjacencyList<V> a = new AdjacencyList<V>();
-
- for (AdjacencyList<V>.Node from : ops.values()) {
- for (V t : from.getNodeValue()) {
- AdjacencyList<V>.Node to = ops.get(t);
- a.addEdge(from, to, 0);
- }
-
- }
- a.fix(true);
- return a;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/Edge.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/Edge.java b/common/src/main/java/org/apache/drill/common/graph/Edge.java
deleted file mode 100644
index 7b9ca61..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/Edge.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.graph;
-
-
-class Edge<N> implements Comparable<Edge<N>> {
-
- final N from, to;
- final int weight;
-
- public Edge(final N argFrom, final N argTo, final int argWeight) {
- from = argFrom;
- to = argTo;
- weight = argWeight;
- }
-
- public int compareTo(final Edge<N> argEdge) {
- return weight - argEdge.weight;
- }
-
- @Override
- public String toString() {
- return "Edge [from=" + from + ", to=" + to + "]";
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/Graph.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/Graph.java b/common/src/main/java/org/apache/drill/common/graph/Graph.java
deleted file mode 100644
index 3cf4e7a..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/Graph.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.graph;
-
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.drill.common.logical.UnexpectedOperatorType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class Graph<G extends GraphValue<G>, R extends G, T extends G> {
-
- static final Logger logger = LoggerFactory.getLogger(Graph.class);
-
- private AdjacencyList<G> adjList;
- private final List<R> roots;
- private final List<T> leaves;
-
- public Graph(List<G> operators, Class<R> root, Class<T> leaf) {
- adjList = AdjacencyList.newInstance(operators);
- roots = checkOperatorType(adjList.getRootNodes(), root, String.format("Root nodes must be a subclass of %s.", root.getSimpleName()));
- leaves = checkOperatorType(adjList.getLeafNodes(), leaf, String.format("Leaf nodes must be a subclass of %s.", leaf.getSimpleName()));
- }
-
- @SuppressWarnings("unchecked")
- private <O extends G> List<O> checkOperatorType(Collection<G> ops, Class<O> classIdentifier, String error){
- for(G o : ops){
- if(!classIdentifier.isAssignableFrom(o.getClass())){
- throw new UnexpectedOperatorType(o, error);
- }
- }
- return (List<O>) ops;
- }
-
- public AdjacencyList<G> getAdjList() {
- return adjList;
- }
-
- public Collection<R> getRoots() {
- return roots;
- }
-
- public Collection<T> getLeaves() {
- return leaves;
- }
-
- public static <G extends GraphValue<G>, R extends G, T extends G> Graph<G, R, T> newGraph(List<G> operators, Class<R> root, Class<T> leaf){
- return new Graph<G, R, T>(operators, root, leaf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/GraphAlgos.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/GraphAlgos.java b/common/src/main/java/org/apache/drill/common/graph/GraphAlgos.java
deleted file mode 100644
index 0b82c7b..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/GraphAlgos.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.graph;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class GraphAlgos {
- static final Logger logger = LoggerFactory.getLogger(GraphAlgos.class);
-
- public static class TopoSorter<V extends GraphValue<V>> {
- final List<AdjacencyList<V>.Node> sorted = new LinkedList<AdjacencyList<V>.Node>();
- final AdjacencyList<V> rGraph;
-
- private TopoSorter(AdjacencyList<V> graph, boolean reverse) {
- graph.clearVisited();
-
- if (reverse) {
- this.rGraph = graph.getReversedList();
- } else {
- this.rGraph = graph;
- }
- Collection<AdjacencyList<V>.Node> sourceNodes = rGraph.getInternalRootNodes();
-
- for (AdjacencyList<V>.Node n : sourceNodes) {
- visit(n);
- }
- }
-
- private void visit(AdjacencyList<V>.Node n) {
- if (n.visited) {
- return;
- }
-
- n.visited = true;
- List<Edge<AdjacencyList<V>.Node>> edges = rGraph.getAdjacent(n);
- if (edges != null) {
- for (Edge<AdjacencyList<V>.Node> e : edges) {
- visit(e.to);
- }
- }
-
- sorted.add(n);
-
- }
-
- /**
- * Execute a depth-first sort on the reversed DAG.
- *
- * @param graph
- * The adjacency list for the DAG.
- * @param sourceNodes
- * List of nodes that
- * @return
- */
- static <V extends GraphValue<V>> List<AdjacencyList<V>.Node> sortInternal(AdjacencyList<V> graph, boolean reverse) {
- TopoSorter<V> ts = new TopoSorter<V>(graph, reverse);
- return ts.sorted;
- }
-
- public static <V extends GraphValue<V>> List<V> sort(Graph<V, ?, ?> graph) {
- AdjacencyList<V> l = graph.getAdjList();
- return l.convert(sortInternal(l, true));
- }
-
- public static <V extends GraphValue<V>> List<V> sortLogical(Graph<V, ?, ?> graph) {
- AdjacencyList<V> l = graph.getAdjList();
- return l.convert(sortInternal(l, false));
- }
- }
-
- static <V extends GraphValue<V>> List<List<AdjacencyList<V>.Node>> checkDirected(AdjacencyList<V> graph) {
- Tarjan<V> t = new Tarjan<V>();
- List<List<AdjacencyList<V>.Node>> subgraphs = t.executeTarjan(graph);
- for (Iterator<List<AdjacencyList<V>.Node>> i = subgraphs.iterator(); i.hasNext();) {
- List<AdjacencyList<V>.Node> l = i.next();
- if (l.size() == 1) {
- i.remove();
- }
- }
- return subgraphs;
- }
-
- public static <V extends GraphValue<V>> List<List<AdjacencyList<V>.Node>> checkDirected(Graph<V, ?, ?> graph) {
- return checkDirected(graph.getAdjList());
- }
-
- public static class Tarjan<V extends GraphValue<V>> {
-
- private int index = 0;
- private List<AdjacencyList<V>.Node> stack = new LinkedList<AdjacencyList<V>.Node>();
- private List<List<AdjacencyList<V>.Node>> SCC = new LinkedList<List<AdjacencyList<V>.Node>>();
-
- public List<List<AdjacencyList<V>.Node>> executeTarjan(AdjacencyList<V> graph) {
- SCC.clear();
- index = 0;
- stack.clear();
- if (graph != null) {
- List<AdjacencyList<V>.Node> nodeList = new LinkedList<AdjacencyList<V>.Node>(graph.getNodeSet());
- for (AdjacencyList<V>.Node node : nodeList) {
- if (node.index == -1) {
- tarjan(node, graph);
- }
- }
- }
- return SCC;
- }
-
- private List<List<AdjacencyList<V>.Node>> tarjan(AdjacencyList<V>.Node v, AdjacencyList<V> list) {
- v.index = index;
- v.lowlink = index;
- index++;
- stack.add(0, v);
- List<Edge<AdjacencyList<V>.Node>> l = list.getAdjacent(v);
- if (l != null) {
- for (Edge<AdjacencyList<V>.Node> e : l) {
- AdjacencyList<V>.Node n = e.to;
- if (n.index == -1) {
- tarjan(n, list);
- v.lowlink = Math.min(v.lowlink, n.lowlink);
- } else if (stack.contains(n)) {
- v.lowlink = Math.min(v.lowlink, n.index);
- }
- }
- }
- if (v.lowlink == v.index) {
- AdjacencyList<V>.Node n;
- List<AdjacencyList<V>.Node> component = new LinkedList<AdjacencyList<V>.Node>();
- do {
- n = stack.remove(0);
- component.add(n);
- } while (n != v);
- SCC.add(component);
- }
- return SCC;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/GraphValue.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/GraphValue.java b/common/src/main/java/org/apache/drill/common/graph/GraphValue.java
deleted file mode 100644
index a919008..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/GraphValue.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.graph;
-
-
-public interface GraphValue<T> extends Iterable<T>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GraphValue.class);
-
- public void accept(GraphVisitor<T> visitor);
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/GraphVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/GraphVisitor.java b/common/src/main/java/org/apache/drill/common/graph/GraphVisitor.java
deleted file mode 100644
index 810b738..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/GraphVisitor.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.graph;
-
-
-public interface GraphVisitor<T> {
- public boolean enter(T o);
- public void leave(T o);
- public boolean visit(T o);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/Visitable.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/Visitable.java b/common/src/main/java/org/apache/drill/common/graph/Visitable.java
deleted file mode 100644
index c97062e..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/Visitable.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.graph;
-
-public interface Visitable<T extends Visitable<?>> {
- public void accept(T node);
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/graph/package-info.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/graph/package-info.java b/common/src/main/java/org/apache/drill/common/graph/package-info.java
deleted file mode 100644
index 38138d0..0000000
--- a/common/src/main/java/org/apache/drill/common/graph/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Connected-graph representation shared by logical and physical query graphs.
- */
-package org.apache.drill.common.graph;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
deleted file mode 100644
index e7d3d1f..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-/**
- * Interface for defining a Drill format plugin.
- *
- * Format plugins are contained within storage plugins to describe particular
- * data formats available in a given storage system. These formats are not
- * necessarily tied to how the data is stored. One current use of this
- * abstraction is describing different file formats like CSV and JSON that
- * can be stored in a filesystem. Some storage systems like databases may only
- * have a single format in which the data is actually stored. This interface
- * enables flexibility for configuring the different data formats that will
- * live under one storage system. The storage systems themselves are described
- * in {@see StoragePluginConfig}s.
- */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
-public interface FormatPluginConfig {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfig.class);
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
deleted file mode 100644
index 50de963..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-import java.util.Set;
-
-import org.apache.drill.common.scanner.persistence.ScanResult;
-
-
-public abstract class FormatPluginConfigBase implements FormatPluginConfig{
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfigBase.class);
-
-
- /**
- * scan for implementations of {@see FormatPlugin}.
- *
- * @param classpathScan - Drill configuration object, used to find the packages to scan
- * @return - list of classes that implement the interface.
- */
- public static Set<Class<? extends FormatPluginConfig>> getSubTypes(final ScanResult classpathScan) {
- final Set<Class<? extends FormatPluginConfig>> pluginClasses = classpathScan.getImplementations(FormatPluginConfig.class);
- logger.debug("Found {} format plugin configuration classes: {}.", pluginClasses.size(), pluginClasses);
- return pluginClasses;
- }
-
- @Override
- public abstract boolean equals(Object o);
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
deleted file mode 100644
index 8a4a5ab..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.config.LogicalPlanPersistence;
-import org.apache.drill.common.graph.Graph;
-import org.apache.drill.common.graph.GraphAlgos;
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.common.logical.data.SinkOperator;
-import org.apache.drill.common.logical.data.SourceOperator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-@JsonPropertyOrder({ "head", "storage", "query" })
-public class LogicalPlan {
- static final Logger logger = LoggerFactory.getLogger(LogicalPlan.class);
-
- private final PlanProperties properties;
- private final Map<String, StoragePluginConfig> storageEngineMap;
- private final Graph<LogicalOperator, SinkOperator, SourceOperator> graph;
-
-
- @JsonCreator
- public LogicalPlan(@JsonProperty("head") PlanProperties head,
- @JsonProperty("storage") Map<String, StoragePluginConfig> storageEngineMap,
- @JsonProperty("query") List<LogicalOperator> operators) {
- this.storageEngineMap = storageEngineMap != null ? storageEngineMap : new HashMap<String, StoragePluginConfig>();
- this.properties = head;
- this.graph = Graph.newGraph(operators, SinkOperator.class, SourceOperator.class);
- }
-
- @JsonProperty("query")
- public List<LogicalOperator> getSortedOperators() {
- return GraphAlgos.TopoSorter.sortLogical(graph);
- }
-
- public StoragePluginConfig getStorageEngineConfig(String name) {
- return storageEngineMap.get(name);
- }
-
- @JsonIgnore
- public Graph<LogicalOperator, SinkOperator, SourceOperator> getGraph() {
- return graph;
- }
-
- @JsonProperty("head")
- public PlanProperties getProperties() {
- return properties;
- }
-
- @JsonProperty("storage")
- public Map<String, StoragePluginConfig> getStorageEngines() {
- return storageEngineMap;
- }
-
- public String toJsonString(LogicalPlanPersistence config) throws JsonProcessingException {
- return config.getMapper().writeValueAsString(this);
- }
-
- public String toJsonStringSafe(LogicalPlanPersistence config){
- try{
- return toJsonString(config);
- }catch(JsonProcessingException e){
- logger.error("Failure while trying to get JSON representation of plan.", e);
- return "Unable to generate plan.";
- }
- }
-
- /** Parses a logical plan. */
- public static LogicalPlan parse(LogicalPlanPersistence config, String planString) {
- ObjectMapper mapper = config.getMapper();
- try {
- LogicalPlan plan = mapper.readValue(planString, LogicalPlan.class);
- return plan;
- } catch (IOException e) {
-
- throw new RuntimeException(String.format("Failure while parsing plan: \n %s}", planString), e);
- }
- }
-
- /** Converts a logical plan to a string. (Opposite of {@link #parse}.) */
- public String unparse(LogicalPlanPersistence config) {
- try {
- return config.getMapper().writeValueAsString(this);
- } catch (JsonProcessingException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static LogicalPlanBuilder builder() {
- return new LogicalPlanBuilder();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
deleted file mode 100644
index 28f1402..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.common.logical;
-
-import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
-import org.apache.drill.common.logical.PlanProperties.PlanType;
-import org.apache.drill.common.logical.data.LogicalOperator;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-
-/**
- * A programmatic builder for logical plans.
- */
-public class LogicalPlanBuilder {
- private PlanProperties planProperties;
- private ImmutableMap.Builder<String, StoragePluginConfig> storageEngines = ImmutableMap.builder();
- private ImmutableList.Builder<LogicalOperator> operators = ImmutableList.builder();
-
- public LogicalPlanBuilder planProperties(PlanProperties planProperties) {
- this.planProperties = planProperties;
- return this;
- }
-
- public LogicalPlanBuilder planProperties(PlanType type, int version, String generatorType, String generatorInfo, ResultMode mode){
- this.planProperties = PlanProperties.builder().generator(generatorType, generatorInfo).type(type).version(version).resultMode(mode).build();
- return this;
- }
-
- public LogicalPlanBuilder addStorageEngine(String name, StoragePluginConfig config) {
- this.storageEngines.put(name, config);
- return this;
- }
- public LogicalPlanBuilder addLogicalOperator(LogicalOperator operator) {
- this.operators.add(operator);
- return this;
- }
- public LogicalPlan build() {
- return new LogicalPlan(this.planProperties, this.storageEngines.build(), this.operators.build());
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java b/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java
deleted file mode 100644
index ce9603e..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.common.logical;
-
-import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Logical plan meta properties.
- */
-public class PlanProperties {
- public static enum PlanType {APACHE_DRILL_LOGICAL, APACHE_DRILL_PHYSICAL}
-
- public PlanType type;
- public int version;
- public Generator generator;
- public ResultMode resultMode;
- public JSONOptions options;
- public int queue;
-
-// @JsonInclude(Include.NON_NULL)
- public static class Generator {
- public String type;
- public String info;
-
- public static enum ResultMode {
- EXEC, LOGICAL, PHYSICAL;
- }
-
- private Generator(@JsonProperty("type") String type, @JsonProperty("info") String info) {
- this.type = type;
- this.info = info;
- }
- }
-
- private PlanProperties(@JsonProperty("version") int version,
- @JsonProperty("generator") Generator generator,
- @JsonProperty("type") PlanType type,
- @JsonProperty("mode") ResultMode resultMode,
- @JsonProperty("options") JSONOptions options,
- @JsonProperty("queue") int queue
- ) {
- this.version = version;
- this.queue = queue;
- this.generator = generator;
- this.type = type;
- this.resultMode = resultMode == null ? ResultMode.EXEC : resultMode;
- this.options = options;
- }
-
- public static PlanPropertiesBuilder builder() {
- return new PlanPropertiesBuilder();
- }
-
- public static class PlanPropertiesBuilder {
- private int version;
- private Generator generator;
- private PlanType type;
- private ResultMode mode = ResultMode.EXEC;
- private JSONOptions options;
- private int queueNumber = 0;
-
- public PlanPropertiesBuilder type(PlanType type) {
- this.type = type;
- return this;
- }
-
- public PlanPropertiesBuilder version(int version) {
- this.version = version;
- return this;
- }
-
- public PlanPropertiesBuilder generator(String type, String info) {
- this.generator = new Generator(type, info);
- return this;
- }
-
- public PlanPropertiesBuilder resultMode(ResultMode mode) {
- this.mode = mode;
- return this;
- }
-
- public PlanPropertiesBuilder queue(int queueNumber) {
- this.queueNumber = queueNumber;
- return this;
- }
-
- public PlanPropertiesBuilder options(JSONOptions options) {
- this.options = options;
- return this;
- }
-
- public PlanPropertiesBuilder generator(Generator generator) {
- this.generator = generator;
- return this;
- }
-
- public PlanProperties build() {
- return new PlanProperties(version, generator, type, mode, options, queueNumber);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
deleted file mode 100644
index a4a0f2b..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
-public abstract class StoragePluginConfig{
-
- private boolean enabled = true;
-
- public boolean isEnabled() {
- return enabled;
- }
-
- public void setEnabled(boolean enabled) {
- this.enabled = enabled;
- }
-
- @Override
- public abstract boolean equals(Object o);
-
- @Override
- public abstract int hashCode();
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
deleted file mode 100644
index 6075483..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-import java.util.Set;
-
-import org.apache.drill.common.scanner.persistence.ScanResult;
-
-
-public abstract class StoragePluginConfigBase extends StoragePluginConfig {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginConfigBase.class);
-
- public static Set<Class<? extends StoragePluginConfig>> getSubTypes(final ScanResult classpathScan) {
- final Set<Class<? extends StoragePluginConfig>> packages = classpathScan.getImplementations(StoragePluginConfig.class);
- logger.debug("Found {} logical operator classes: {}.", packages.size(), packages);
- return packages;
- }
-
- @Override
- public abstract boolean equals(Object o);
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/UnexpectedOperatorType.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/UnexpectedOperatorType.java b/common/src/main/java/org/apache/drill/common/logical/UnexpectedOperatorType.java
deleted file mode 100644
index ea3f4b2..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/UnexpectedOperatorType.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-
-public class UnexpectedOperatorType extends ValidationError{
-
- public UnexpectedOperatorType(String message){
- super(message);
- }
-
- public UnexpectedOperatorType(Object operator, String message) {
- super(message + " Received node of type " + operator.getClass().getCanonicalName());
- }
-
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/ValidationError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/ValidationError.java b/common/src/main/java/org/apache/drill/common/logical/ValidationError.java
deleted file mode 100644
index 54c9c82..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/ValidationError.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-public class ValidationError extends RuntimeException{
-
- public ValidationError() {
- super();
- }
-
-
- public ValidationError(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public ValidationError(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ValidationError(String message) {
- super(message);
- }
-
- public ValidationError(Throwable cause) {
- super(cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java b/common/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
deleted file mode 100644
index b178cd9..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-
-public abstract class AbstractBuilder<T extends LogicalOperator> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBuilder.class);
-
- public abstract T build();
-
- protected LogicalExpression[] aL(List<LogicalExpression> exprs){
- return exprs.toArray(new LogicalExpression[exprs.size()]);
- }
-
- protected FieldReference[] aF(List<FieldReference> exprs){
- return exprs.toArray(new FieldReference[exprs.size()]);
- }
-
- protected NamedExpression[] aN(List<NamedExpression> exprs){
- return exprs.toArray(new NamedExpression[exprs.size()]);
- }
-
- protected Order.Ordering[] aO(List<Order.Ordering> orderings) {
- return orderings.toArray(new Order.Ordering[orderings.size()]);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java b/common/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
deleted file mode 100644
index 3ed947e..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import com.google.common.base.Preconditions;
-
-public abstract class AbstractSingleBuilder<T extends SingleInputOperator, X extends AbstractSingleBuilder<T, X>> extends AbstractBuilder<T> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSingleBuilder.class);
-
- private LogicalOperator input;
-
- @Override
- public final T build(){
- Preconditions.checkNotNull(input);
- T out = internalBuild();
- out.setInput(input);
- return out;
- }
-
- public X setInput(LogicalOperator input){
- this.input = input;
- return (X) this;
- }
-
- public abstract T internalBuild();
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Filter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Filter.java b/common/src/main/java/org/apache/drill/common/logical/data/Filter.java
deleted file mode 100644
index 02737f8..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Filter.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-
-@JsonTypeName("filter")
-public class Filter extends SingleInputOperator {
- private final LogicalExpression expr;
-
- @JsonCreator
- public Filter(@JsonProperty("expr") LogicalExpression expr) {
- this.expr = expr;
- }
-
- public LogicalExpression getExpr() {
- return expr;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitFilter(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java b/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java
deleted file mode 100644
index 803a4b3..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Flatten.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-
-@JsonTypeName("flatten")
-public class Flatten extends SingleInputOperator {
- private final FieldReference name;
- private final LogicalExpression expr;
- private final boolean drop;
-
- @JsonCreator
- public Flatten(@JsonProperty("name") FieldReference name, @JsonProperty("expr") LogicalExpression expr,
- @JsonProperty("drop") boolean drop) {
- this.name = name;
- this.expr = expr;
- this.drop = drop;
- }
-
- public FieldReference getName() {
- return name;
- }
-
- public LogicalExpression getExpr() {
- return expr;
- }
-
- public boolean isDrop() {
- return drop;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitFlatten(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java b/common/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
deleted file mode 100644
index 2913a7e..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-
-@JsonTypeName("groupingaggregate")
-public class GroupingAggregate extends SingleInputOperator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GroupingAggregate.class);
-
- private final NamedExpression[] keys;
- private final NamedExpression[] exprs;
-
- public GroupingAggregate(@JsonProperty("keys") NamedExpression[] keys, @JsonProperty("exprs") NamedExpression[] exprs) {
- super();
- this.keys = keys;
- this.exprs = exprs;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitGroupingAggregate(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
- }
-
- public static Builder builder(){
- return new Builder();
- }
-
- public NamedExpression[] getKeys(){
- return keys;
- }
-
- public NamedExpression[] getExprs(){
- return exprs;
- }
-
- public static class Builder extends AbstractSingleBuilder<GroupingAggregate, Builder>{
- private List<NamedExpression> keys = Lists.newArrayList();
- private List<NamedExpression> exprs = Lists.newArrayList();
-
- public Builder addKey(FieldReference ref, LogicalExpression expr){
- keys.add(new NamedExpression(expr, ref));
- return this;
- }
-
- public Builder addKey(NamedExpression expr){
- keys.add(expr);
- return this;
- }
-
- public Builder addExpr(NamedExpression expr){
- exprs.add(expr);
- return this;
- }
-
- public Builder addExpr(FieldReference ref, LogicalExpression expr){
- exprs.add(new NamedExpression(expr, ref));
- return this;
- }
-
- @Override
- public GroupingAggregate internalBuild(){
- GroupingAggregate ga = new GroupingAggregate(aN(keys), aN(exprs));
- return ga;
- }
-
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/44dea433/common/src/main/java/org/apache/drill/common/logical/data/Join.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Join.java b/common/src/main/java/org/apache/drill/common/logical/data/Join.java
deleted file mode 100644
index 2d7e6e1..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Join.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.exceptions.ExpressionParsingException;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-import org.apache.calcite.rel.core.JoinRelType;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-
-@JsonTypeName("join")
-public class Join extends LogicalOperatorBase {
- private final LogicalOperator left;
- private final LogicalOperator right;
- private final JoinRelType type;
- private final JoinCondition[] conditions;
-
- public static JoinRelType resolve(String val) {
- for (JoinRelType jt : JoinRelType.values()) {
- if (jt.name().equalsIgnoreCase(val)) {
- return jt;
- }
- }
- throw new ExpressionParsingException(String.format("Unable to determine join type for value '%s'.", val));
- }
-
- @JsonCreator
- public Join(@JsonProperty("left") LogicalOperator left, @JsonProperty("right") LogicalOperator right,
- @JsonProperty("conditions") JoinCondition[] conditions, @JsonProperty("type") String type) {
- this(left, right, conditions, resolve(type));
- }
-
- @JsonCreator
- public Join(@JsonProperty("left") LogicalOperator left, @JsonProperty("right") LogicalOperator right, @JsonProperty("conditions")JoinCondition[] conditions, @JsonProperty("type") JoinRelType type) {
- super();
- this.conditions = conditions;
- this.left = left;
- this.right = right;
- left.registerAsSubscriber(this);
- right.registerAsSubscriber(this);
- this.type = type;
-
- }
-
- public LogicalOperator getLeft() {
- return left;
- }
-
- public LogicalOperator getRight() {
- return right;
- }
-
- public JoinCondition[] getConditions() {
- return conditions;
- }
-
- @JsonIgnore
- public JoinRelType getJoinType() {
- return type;
- }
-
- public String getType() {
- return type.name();
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitJoin(this, value);
- }
-
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.forArray(getLeft(), getRight());
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static class Builder extends AbstractBuilder<Join>{
- private LogicalOperator left;
- private LogicalOperator right;
- private JoinRelType type;
- private List<JoinCondition> conditions = Lists.newArrayList();
-
- public Builder type(JoinRelType type) {
- this.type = type;
- return this;
- }
-
- public Builder left(LogicalOperator left) {
- this.left = left;
- return this;
- }
- public Builder right(LogicalOperator right) {
- this.right = right;
- return this;
- }
-
- public Builder addCondition(String relationship, LogicalExpression left, LogicalExpression right) {
- conditions.add(new JoinCondition(relationship, left, right));
- return this;
- }
-
- @Override
- public Join build() {
- Preconditions.checkNotNull(left);
- Preconditions.checkNotNull(right);
- Preconditions.checkNotNull(type);
- return new Join(left, right, conditions.toArray(new JoinCondition[conditions.size()]), type);
- }
-
- }
-
-}