You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2015/11/13 03:37:48 UTC
[18/45] drill git commit: DRILL-3987: (REFACTOR) Common and Vector
modules building.
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index 49a64cf..0e5b436 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -26,6 +26,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.apache.calcite.rel.RelFieldCollation.Direction;
import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
@@ -37,6 +38,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
@@ -45,8 +47,6 @@ import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.ExternalSort;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
@@ -71,7 +71,6 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.AbstractContainerVector;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.calcite.rel.RelFieldCollation.Direction;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
@@ -145,7 +144,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
SPILL_THRESHOLD = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD);
dirs = Iterators.cycle(config.getStringList(ExecConstants.EXTERNAL_SORT_SPILL_DIRS));
copierAllocator = oContext.getAllocator().getChildAllocator(
- context, PriorityQueueCopier.INITIAL_ALLOCATION, PriorityQueueCopier.MAX_ALLOCATION, true);
+ context.asLimitConsumer(), PriorityQueueCopier.INITIAL_ALLOCATION, PriorityQueueCopier.MAX_ALLOCATION, true);
FragmentHandle handle = context.getHandle();
fileName = String.format("%s/major_fragment_%s/minor_fragment_%s/operator_%s", QueryIdHelper.getQueryId(handle.getQueryId()),
handle.getMajorFragmentId(), handle.getMinorFragmentId(), popConfig.getOperatorId());
@@ -321,7 +320,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
} catch(InterruptedException e) {
return IterOutcome.STOP;
} catch (OutOfMemoryException e) {
- throw new OutOfMemoryRuntimeException(e);
+ throw new OutOfMemoryException(e);
}
}
int count = sv2.getCount();
@@ -343,7 +342,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
// current memory used is more than 95% of memory usage limit of this operator
(totalSizeInMemory > .95 * popConfig.getMaxAllocation()) ||
// current memory used is more than 95% of memory usage limit of this fragment
- (totalSizeInMemory > .95 * oContext.getAllocator().getFragmentLimit()) ||
+ (totalSizeInMemory > .95 * oContext.getAllocator().getLimit()) ||
// Number of incoming batches (BatchGroups) exceed the limit and number of incoming batches accumulated
// since the last spill exceed the defined limit
(batchGroups.size() > SPILL_THRESHOLD && batchesSinceLastSpill >= SPILL_BATCH_GROUP_SIZE)) {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
index f06c397..229fe65 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
@@ -23,8 +23,8 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.ops.OperatorStats;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
index 4f91317..65d164d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.record;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
index 3dab51f..dcf9a7d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.record.selection;
import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.record.DeadBuf;
/**
@@ -90,7 +90,7 @@ public class SelectionVector2 implements AutoCloseable {
public boolean allocateNewSafe(int size) {
try {
allocateNew(size);
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
index 3e2adaf..3dfe03f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ProtobufLengthDecoder.java
@@ -27,7 +27,7 @@ import java.util.List;
import org.apache.drill.exec.memory.BufferAllocator;
import com.google.protobuf.CodedInputStream;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
/**
* Modified version of {@link io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder} that avoids bytebuf copy.
@@ -85,7 +85,7 @@ public class ProtobufLengthDecoder extends ByteToMessageDecoder {
ByteBuf outBuf;
try {
outBuf = allocator.buffer(length);
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
logger.warn("Failure allocating buffer on incoming stream due to memory limits. Current Allocation: {}.", allocator.getAllocatedMemory());
in.resetReaderIndex();
outOfMemoryHandler.handle();
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
index 41285c7..2f1ca56 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
@@ -21,7 +21,7 @@ import java.util.Collection;
import java.util.Map;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.planner.logical.ColumnList;
import org.apache.drill.exec.record.MaterializedField.Key;
import org.apache.drill.exec.vector.ValueVector;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
index f1b55e7..4df1772 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.store;
import java.util.Map;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField.Key;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
index 8160f1c..1d22fed 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
@@ -25,7 +25,7 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
index c742690..d2c2425 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
@@ -22,9 +22,9 @@ import java.util.Map;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index c22392a..91cd112 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -27,7 +27,7 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.AbstractWriter;
import org.apache.drill.exec.physical.base.PhysicalOperator;
@@ -57,7 +57,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-
import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.ParquetFileWriter;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index f784566..a2cd1fa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -30,7 +30,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.planner.physical.WriterPrel;
@@ -47,7 +47,6 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.parquet.column.ColumnWriteStore;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
import org.apache.parquet.column.impl.ColumnWriteStoreV1;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
index f42996b..47d502f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
@@ -31,8 +31,8 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
@@ -45,7 +45,6 @@ import org.apache.drill.exec.vector.complex.RepeatedValueVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.format.FileMetaData;
import org.apache.parquet.format.SchemaElement;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 9e6919b..27536d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -33,8 +33,8 @@ import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
@@ -50,7 +50,6 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
import org.apache.hadoop.fs.Path;
-
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.hadoop.CodecFactory;
import org.apache.parquet.hadoop.metadata.ColumnPath;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
index 32fa31d..22f6bc3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoRecordReader.java
@@ -25,8 +25,8 @@ import java.util.List;
import java.util.Map;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField.Key;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
deleted file mode 100644
index 2348e62..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-public class AssertionUtil {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class);
-
-
- public static final boolean ASSERT_ENABLED;
- public static final boolean BOUNDS_CHECKING_ENABLED;
-
- static{
- boolean isAssertEnabled = false;
- assert isAssertEnabled = true;
- ASSERT_ENABLED = isAssertEnabled;
- BOUNDS_CHECKING_ENABLED = ASSERT_ENABLED || !"true".equals(System.getProperty("drill.enable_unsafe_memory_access"));
- }
-
- public static boolean isAssertionsEnabled(){
- return ASSERT_ENABLED;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
deleted file mode 100644
index 6966357..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
+++ /dev/null
@@ -1,730 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import io.netty.buffer.DrillBuf;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.drill.common.util.CoreDecimalUtility;
-import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
-import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
-
-public class DecimalUtility extends CoreDecimalUtility{
-
- public final static int MAX_DIGITS = 9;
- public final static int DIGITS_BASE = 1000000000;
- public final static int DIGITS_MAX = 999999999;
- public final static int integerSize = (Integer.SIZE/8);
-
- public final static String[] decimalToString = {"",
- "0",
- "00",
- "000",
- "0000",
- "00000",
- "000000",
- "0000000",
- "00000000",
- "000000000"};
-
- public final static long[] scale_long_constants = {
- 1,
- 10,
- 100,
- 1000,
- 10000,
- 100000,
- 1000000,
- 10000000,
- 100000000,
- 1000000000,
- 10000000000l,
- 100000000000l,
- 1000000000000l,
- 10000000000000l,
- 100000000000000l,
- 1000000000000000l,
- 10000000000000000l,
- 100000000000000000l,
- 1000000000000000000l};
-
- /*
- * Simple function that returns the static precomputed
- * power of ten, instead of using Math.pow
- */
- public static long getPowerOfTen(int power) {
- assert power >= 0 && power < scale_long_constants.length;
- return scale_long_constants[(power)];
- }
-
- /*
- * Math.pow returns a double and while multiplying with large digits
- * in the decimal data type we encounter noise. So instead of multiplying
- * with Math.pow we use the static constants to perform the multiplication
- */
- public static long adjustScaleMultiply(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input * scale_long_constants[index];
- } else {
- return input / scale_long_constants[index];
- }
- }
-
- public static long adjustScaleDivide(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input / scale_long_constants[index];
- } else {
- return input * scale_long_constants[index];
- }
- }
-
- /* Given the number of actual digits this function returns the
- * number of indexes it will occupy in the array of integers
- * which are stored in base 1 billion
- */
- public static int roundUp(int ndigits) {
- return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
- }
-
- /* Returns a string representation of the given integer
- * If the length of the given integer is less than the
- * passed length, this function will prepend zeroes to the string
- */
- public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
- String value = ((Integer) number).toString();
- int length = value.length();
-
- StringBuilder str = new StringBuilder();
- str.append(decimalToString[desiredLength - length]);
- str.append(value);
-
- return str;
- }
-
- public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
- String value = ((Long) number).toString();
- int length = value.length();
-
- StringBuilder str = new StringBuilder();
-
- // Desired length can be > MAX_DIGITS
- int zeroesLength = desiredLength - length;
- while (zeroesLength > MAX_DIGITS) {
- str.append(decimalToString[MAX_DIGITS]);
- zeroesLength -= MAX_DIGITS;
- }
- str.append(decimalToString[zeroesLength]);
- str.append(value);
-
- return str;
- }
-
- public static BigDecimal getBigDecimalFromIntermediate(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
-
- // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
- return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false);
- }
-
- public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
-
- // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
- return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true);
- }
-
- public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) {
- byte[] value = new byte[length];
- bytebuf.getBytes(start, value, 0, length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
- byte[] value = new byte[length];
- bytebuf.get(value);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- /* Create a BigDecimal object using the data in the DrillBuf.
- * This function assumes that data is provided in a non-dense format
- * It works on both sparse and intermediate representations.
- */
- public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) {
-
- // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
- int actualDigits;
-
- // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out
- BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
-
- BigInteger base = BigInteger.valueOf(DIGITS_BASE);
-
- for (int i = 1; i < nDecimalDigits; i++) {
-
- BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * integerSize)));
- decimalDigits = decimalDigits.multiply(base);
- decimalDigits = decimalDigits.add(temp);
- }
-
- // Truncate any additional padding we might have added
- if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
- BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
- decimalDigits = decimalDigits.divide(truncate);
- }
-
- // set the sign
- if ((data.getInt(startIndex) & 0x80000000) != 0) {
- decimalDigits = decimalDigits.negate();
- }
-
- BigDecimal decimal = new BigDecimal(decimalDigits, scale);
-
- return decimal;
- }
-
- /* This function returns a BigDecimal object from the dense decimal representation.
- * First step is to convert the dense representation into an intermediate representation
- * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object
- */
- public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
-
- /* This method converts the dense representation to
- * an intermediate representation. The intermediate
- * representation has one more integer than the dense
- * representation.
- */
- byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * integerSize)];
-
- // Start storing from the least significant byte of the first integer
- int intermediateIndex = 3;
-
- int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
- int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
-
- int maskIndex;
- int shiftOrder;
- byte shiftBits;
-
- // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
- if (maxPrecision == 38) {
- maskIndex = 0;
- shiftOrder = 6;
- shiftBits = 0x00;
- intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
- } else if (maxPrecision == 28) {
- maskIndex = 1;
- shiftOrder = 4;
- shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
- intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
- } else {
- throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
- }
-
- int inputIndex = 1;
- boolean sign = false;
-
- if ((data.getByte(startIndex) & 0x80) != 0) {
- sign = true;
- }
-
- while (inputIndex < width) {
-
- intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
-
- shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
-
- inputIndex++;
- intermediateIndex++;
-
- if (((inputIndex - 1) % integerSize) == 0) {
- shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
- maskIndex++;
- shiftOrder -= 2;
- }
-
- }
- /* copy the last byte */
- intermediateBytes[intermediateIndex] = shiftBits;
-
- if (sign == true) {
- intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
- }
- DrillBuf intermediate = data.getAllocator().buffer(intermediateBytes.length);
- intermediate.setBytes(0, intermediateBytes);
-
- BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
- intermediate.release();
- return ret;
- }
-
- /*
- * Function converts the BigDecimal and stores it in out internal sparse representation
- */
- public static void getSparseFromBigDecimal(BigDecimal input, DrillBuf data, int startIndex, int scale, int precision, int nDecimalDigits) {
-
- // Initialize the buffer
- for (int i = 0; i < nDecimalDigits; i++) {
- data.setInt(startIndex + (i * integerSize), 0);
- }
-
- boolean sign = false;
-
- if (input.signum() == -1) {
- // negative input
- sign = true;
- input = input.abs();
- }
-
- // Truncate the input as per the scale provided
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- // Separate out the integer part
- BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
-
- int destIndex = nDecimalDigits - roundUp(scale) - 1;
-
- // we use base 1 billion integer digits for out integernal representation
- BigDecimal base = new BigDecimal(DIGITS_BASE);
-
- while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
- // store the modulo as the integer value
- data.setInt(startIndex + (destIndex * integerSize), (integerPart.remainder(base)).intValue());
- destIndex--;
- // Divide by base 1 billion
- integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
- }
-
- /* Sparse representation contains padding of additional zeroes
- * so each digit contains MAX_DIGITS for ease of arithmetic
- */
- int actualDigits;
- if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
- // Pad additional zeroes
- scale = scale + (MAX_DIGITS - actualDigits);
- input = input.setScale(scale, BigDecimal.ROUND_DOWN);
- }
-
- //separate out the fractional part
- BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
-
- destIndex = nDecimalDigits - 1;
-
- while (scale > 0) {
- // Get next set of MAX_DIGITS (9) store it in the DrillBuf
- fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
- BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
-
- data.setInt(startIndex + (destIndex * integerSize), (temp.unscaledValue().intValue()));
- destIndex--;
-
- fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
- scale -= MAX_DIGITS;
- }
-
- // Set the negative sign
- if (sign == true) {
- data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
- }
-
- }
-
-
- public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- return (input.unscaledValue().longValue());
- }
-
- public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
- return BigDecimal.valueOf(input, scale);
- }
-
- public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
- return BigDecimal.valueOf(input, scale);
- }
-
-
- public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) {
-
- int invert = 1;
-
- /* If signs are different then simply look at the
- * sign of the two inputs and determine which is greater
- */
- if (leftSign != rightSign) {
-
- return((leftSign == true) ? -1 : 1);
- } else if(leftSign == true) {
- /* Both inputs are negative, at the end we will
- * have to invert the comparison
- */
- invert = -1;
- }
-
- int cmp = 0;
-
- for (int i = 0; i < width; i++) {
- byte leftByte = left.getByte(leftStart + i);
- byte rightByte = right.getByte(rightStart + i);
- // Unsigned byte comparison
- if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
- cmp = 1;
- break;
- } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
- cmp = -1;
- break;
- }
- }
- cmp *= invert; // invert the comparison if both were negative values
-
- return cmp;
- }
-
- public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) {
- int value = buffer.getInt(start + (index * 4));
-
- if (index == 0) {
- /* the first byte contains sign bit, return value without it */
- value = (value & 0x7FFFFFFF);
- }
- return value;
- }
-
- public static void setInteger(DrillBuf buffer, int start, int index, int value) {
- buffer.setInt(start + (index * 4), value);
- }
-
- public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
-
- int invert = 1;
-
- if (absCompare == false) {
- if (leftSign != rightSign) {
- return (leftSign == true) ? -1 : 1;
- }
-
- // Both values are negative invert the outcome of the comparison
- if (leftSign == true) {
- invert = -1;
- }
- }
-
- int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
- return cmp * invert;
- }
- public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
- /* compute the number of integer digits in each decimal */
- int leftInt = leftPrecision - leftScale;
- int rightInt = rightPrecision - rightScale;
-
- /* compute the number of indexes required for storing integer digits */
- int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt);
- int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt);
-
- /* compute number of indexes required for storing scale */
- int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale);
- int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale);
-
- /* compute index of the most significant integer digits */
- int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
- int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
- int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
-
- /* Discard the zeroes in the integer part */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- break;
- }
-
- /* Digit in this location is zero, decrement the actual number
- * of integer digits
- */
- leftIntRoundedUp--;
- leftIndex1++;
- }
-
- /* If we reached the stop index then the number of integers is zero */
- if (leftIndex1 == leftStopIndex) {
- leftIntRoundedUp = 0;
- }
-
- while (rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- break;
- }
-
- /* Digit in this location is zero, decrement the actual number
- * of integer digits
- */
- rightIntRoundedUp--;
- rightIndex1++;
- }
-
- if (rightIndex1 == rightStopIndex) {
- rightIntRoundedUp = 0;
- }
-
- /* We have the accurate number of non-zero integer digits,
- * if the number of integer digits are different then we can determine
- * which decimal is larger and needn't go down to comparing individual values
- */
- if (leftIntRoundedUp > rightIntRoundedUp) {
- return 1;
- }
- else if (rightIntRoundedUp > leftIntRoundedUp) {
- return -1;
- }
-
- /* The number of integer digits are the same, set the each index
- * to the first non-zero integer and compare each digit
- */
- leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
-
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
- }
-
- /* The integer part of both the decimal's are equal, now compare
- * each individual fractional part. Set the index to be at the
- * beginning of the fractional part
- */
- leftIndex1 = leftStopIndex;
- rightIndex1 = rightStopIndex;
-
- /* Stop indexes will be the end of the array */
- leftStopIndex = nDecimalDigits;
- rightStopIndex = nDecimalDigits;
-
- /* compare the two fractional parts of the decimal */
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
- }
-
- /* Till now the fractional part of the decimals are equal, check
- * if one of the decimal has fractional part that is remaining
- * and is non-zero
- */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- return 1;
- }
- leftIndex1++;
- }
-
- while(rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- return -1;
- }
- rightIndex1++;
- }
-
- /* Both decimal values are equal */
- return 0;
- }
-
- public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
- byte[] value = Arrays.copyOfRange(bytes, start, start + length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
-
- public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
- int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale);
- int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale);
-
- if (desiredScale < currentScale) {
-
- boolean roundUp = false;
-
- //Extract the first digit to be truncated to check if we need to round up
- int truncatedScaleIndex = desiredScale + 1;
- if (truncatedScaleIndex <= currentScale) {
- int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
- extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex);
- int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
- int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
- if (temp != 0) {
- extractDigit = extractDigit / (int) (Math.pow(10, temp));
- }
- if ((extractDigit % 10) > 4) {
- roundUp = true;
- }
- }
-
- // Get the source index beyond which we will truncate
- int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
- int srcIndex = srcIntIndex + newScaleRoundedUp;
-
- // Truncate the remaining fractional part, move the integer part
- int destIndex = nDecimalDigits - 1;
- if (srcIndex != destIndex) {
- while (srcIndex >= 0) {
- setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
- }
-
- // Set the remaining portion of the decimal to be zeroes
- while (destIndex >= 0) {
- setInteger(result, start, destIndex--, 0);
- }
- srcIndex = nDecimalDigits - 1;
- }
-
- // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
- int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
- if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
- truncateFactor = (int) Math.pow(10, truncateFactor);
- int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
- fractionalDigits /= truncateFactor;
- setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
- }
-
- // Finally round up the digit if needed
- if (roundUp == true) {
- srcIndex = nDecimalDigits - 1;
- int carry;
- if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
- carry = truncateFactor;
- } else {
- carry = 1;
- }
-
- while (srcIndex >= 0) {
- int value = getIntegerFromSparseBuffer(result, start, srcIndex);
- value += carry;
-
- if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
- setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
- carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
- } else {
- setInteger(result, start, srcIndex--, value);
- carry = 0;
- break;
- }
- }
- }
- } else if (desiredScale > currentScale) {
- // Add fractional digits to the decimal
-
- // Check if we need to shift the decimal digits to the left
- if (newScaleRoundedUp > origScaleRoundedUp) {
- int srcIndex = 0;
- int destIndex = newScaleRoundedUp - origScaleRoundedUp;
-
- // Check while extending scale, we are not overwriting integer part
- while (srcIndex < destIndex) {
- if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
- throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
- }
- }
-
- srcIndex = 0;
- while (destIndex < nDecimalDigits) {
- setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
- }
-
- // Clear the remaining part
- while (srcIndex < nDecimalDigits) {
- setInteger(result, start, srcIndex++, 0);
- }
- }
- }
- }
-
- public static int getFirstFractionalDigit(int decimal, int scale) {
- if (scale == 0) {
- return 0;
- }
- int temp = (int) adjustScaleDivide(decimal, scale - 1);
- return Math.abs(temp % 10);
- }
-
- public static int getFirstFractionalDigit(long decimal, int scale) {
- if (scale == 0) {
- return 0;
- }
- long temp = adjustScaleDivide(decimal, scale - 1);
- return (int) (Math.abs(temp % 10));
- }
-
- public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) {
- if (scale == 0) {
- return 0;
- }
-
- int index = nDecimalDigits - roundUp(scale);
- return (int) (adjustScaleDivide(data.getInt(start + (index * integerSize)), MAX_DIGITS - 1));
- }
-
- public static int compareSparseSamePrecScale(DrillBuf left, int lStart, byte[] right, int length) {
- // check the sign first
- boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
- boolean rSign = ByteFunctionHelpers.getSign(right);
- int cmp = 0;
-
- if (lSign != rSign) {
- return (lSign == false) ? 1 : -1;
- }
-
- // invert the comparison if we are comparing negative numbers
- int invert = (lSign == true) ? -1 : 1;
-
- // compare byte by byte
- int n = 0;
- int lPos = lStart;
- int rPos = 0;
- while (n < length/4) {
- int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
- int rightInt = ByteFunctionHelpers.getInteger(right, n);
- if (leftInt != rightInt) {
- cmp = (leftInt - rightInt ) > 0 ? 1 : -1;
- break;
- }
- n++;
- }
- return cmp * invert;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
index d52cb5d..3f2e3f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
import org.apache.drill.common.AutoCloseables;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.BitControl.Collector;
import org.apache.drill.exec.record.RawFragmentBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
index cfe5b6b..753c75d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
@@ -29,8 +29,8 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.BitData;
import org.apache.drill.exec.proto.BitData.FragmentRecordBatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index e17eba5..6adbfc0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -43,8 +43,7 @@ import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.DistributedSemaphore;
import org.apache.drill.exec.coord.DistributedSemaphore.DistributedLease;
import org.apache.drill.exec.exception.OptimizerException;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.opt.BasicOptimizer;
@@ -247,7 +246,7 @@ public class Foreman implements Runnable {
throw new IllegalStateException();
}
injector.injectChecked(queryContext.getExecutionControls(), "run-try-end", ForemanException.class);
- } catch (final OutOfMemoryException | OutOfMemoryRuntimeException e) {
+ } catch (final OutOfMemoryException e) {
moveToState(QueryState.FAILED, UserException.memoryError(e).build(logger));
} catch (final ForemanException e) {
moveToState(QueryState.FAILED, e);
@@ -407,7 +406,7 @@ public class Foreman implements Runnable {
setupRootFragment(rootPlanFragment, work.getRootOperator());
setupNonRootFragments(planFragments);
- drillbitContext.getAllocator().resetFragmentLimits(); // TODO a global effect for this query?!?
+ drillbitContext.getAllocator().resetLimits(); // TODO a global effect for this query?!?
moveToState(QueryState.RUNNING, null);
logger.debug("Fragments running.");
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
index 094316e..9101c13 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
@@ -29,7 +29,7 @@ import org.apache.drill.common.SerializedExecutor;
import org.apache.drill.common.concurrent.ExtendedLatch;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.coord.ClusterCoordinator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.FragmentContext.ExecutorState;
import org.apache.drill.exec.physical.base.FragmentRoot;
@@ -261,7 +261,7 @@ public class FragmentExecutor implements Runnable {
}
});
- } catch (OutOfMemoryError | OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryError | OutOfMemoryException e) {
if (!(e instanceof OutOfMemoryError) || "Direct buffer memory".equals(e.getMessage())) {
fail(UserException.memoryError(e).build(logger));
} else {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java b/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java
index 28f6390..735aaa2 100644
--- a/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java
+++ b/exec/java-exec/src/main/java/org/apache/parquet/hadoop/ColumnChunkIncReadStore.java
@@ -28,7 +28,7 @@ import java.util.Map;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.store.parquet.ColumnDataReader;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -217,7 +217,7 @@ public class ColumnChunkIncReadStore implements PageReadStore {
}
in.close();
return null;
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
throw e; // throw as it is
} catch (Exception e) {
throw new DrillRuntimeException("Error reading page." +
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
index 8f16c44..1844c32 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.client.PrintingResultsListener;
import org.apache.drill.exec.client.QuerySubmitter.Format;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.user.AwaitableUserResultsListener;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
index 10fd5da..67cabff 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
@@ -18,7 +18,7 @@
package org.apache.drill;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import org.apache.drill.exec.testing.Controls;
@@ -42,7 +42,7 @@ public class TestAllocationException extends BaseTestQuery {
final String controls = Controls.newBuilder()
.addException(TopLevelAllocator.class,
TopLevelAllocator.CHILD_BUFFER_INJECTION_SITE,
- OutOfMemoryRuntimeException.class,
+ OutOfMemoryException.class,
200,
1
).build();
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
index d522663..65adc02 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
@@ -170,7 +170,7 @@ public class TestAllocators {
try {
oContext31.getAllocator().buffer(4400000);
fail("Fragment 3 should fail to allocate buffer");
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
// expected
}
@@ -179,7 +179,7 @@ public class TestAllocators {
try {
DrillBuf b32 = oContext32.getAllocator().buffer(4400000);
b32.release();
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
fail("Fragment 3 failed to allocate buffer");
} finally {
closeOp(oContext32);
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
index f269bd7..0124f9e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
@@ -29,8 +29,8 @@ import java.util.List;
import java.util.Random;
import org.apache.drill.PlanTestBase;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.physical.MinorFragmentEndpoint;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
index a32322b..a10a76d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.server;
import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import org.glassfish.hk2.api.Factory;
public class DrillClientFactory implements Factory<DrillClient> {
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/pom.xml
----------------------------------------------------------------------
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 4bd968e..6001944 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -56,6 +56,12 @@
<artifactId>hppc</artifactId>
<version>0.4.2</version>
</dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+
</dependencies>
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/includes/vv_imports.ftl b/exec/vector/src/main/codegen/includes/vv_imports.ftl
index 0954dcf..e6dc973 100644
--- a/exec/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/vector/src/main/codegen/includes/vv_imports.ftl
@@ -45,7 +45,7 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
import java.util.Arrays;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/BasicTypeHelper.java b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
new file mode 100644
index 0000000..723687a
--- /dev/null
+++ b/exec/vector/src/main/codegen/templates/BasicTypeHelper.java
@@ -0,0 +1,544 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/BasicTypeHelper.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr;
+
+<#include "/@includes/vv_imports.ftl" />
+import org.apache.drill.exec.vector.complex.UnionVector;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.complex.RepeatedMapVector;
+import org.apache.drill.exec.util.CallBack;
+
+public class BasicTypeHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicTypeHelper.class);
+
+ private static final int WIDTH_ESTIMATE = 50;
+
+ // Default length when casting to varchar : 65536 = 2^16
+ // This only defines an absolute maximum for values, setting
+ // a high value like this will not inflate the size for small values
+ public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
+
+ private static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
+ return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
+ }
+
+ private static String buildErrorMessage(final String operation, final MajorType type) {
+ return buildErrorMessage(operation, type.getMinorType(), type.getMode());
+ }
+
+ public static int getSize(MajorType major) {
+ switch (major.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
+ minor.class?substring(0, 3) == "PRO" ||
+ minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
+ </#list>
+</#list>
+ case FIXEDCHAR: return major.getWidth();
+ case FIXED16CHAR: return major.getWidth();
+ case FIXEDBINARY: return major.getWidth();
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get size", major));
+ }
+
+ public static ValueVector getNewVector(SchemaPath parentPath, String name, BufferAllocator allocator, MajorType type, CallBack callback){
+ SchemaPath child = parentPath.getChild(name);
+ MaterializedField field = MaterializedField.create(child, type);
+ return getNewVector(field, allocator, callback);
+ }
+
+
+ public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
+ switch (type) {
+ case UNION:
+ return UnionVector.class;
+ case MAP:
+ switch (mode) {
+ case OPTIONAL:
+ case REQUIRED:
+ return MapVector.class;
+ case REPEATED:
+ return RepeatedMapVector.class;
+ }
+
+ case LIST:
+ switch (mode) {
+ case REPEATED:
+ return RepeatedListVector.class;
+ case REQUIRED:
+ case OPTIONAL:
+ return ListVector.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}Vector.class;
+ case OPTIONAL:
+ return Nullable${minor.class}Vector.class;
+ case REPEATED:
+ return Repeated${minor.class}Vector.class;
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT :
+ return ObjectVector.class ;
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
+ }
+ public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
+ switch (type) {
+ case MAP:
+ switch (mode) {
+ case REQUIRED:
+ if (!isSingularRepeated)
+ return SingleMapReaderImpl.class;
+ else
+ return SingleLikeRepeatedMapReaderImpl.class;
+ case REPEATED:
+ return RepeatedMapReaderImpl.class;
+ }
+ case LIST:
+ switch (mode) {
+ case REQUIRED:
+ return SingleListReaderImpl.class;
+ case REPEATED:
+ return RepeatedListReaderImpl.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}ReaderImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}ReaderImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}ReaderImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
+ }
+
+ public static Class<?> getWriterInterface( MinorType type, DataMode mode){
+ switch (type) {
+ case UNION: return UnionWriter.class;
+ case MAP: return MapWriter.class;
+ case LIST: return ListWriter.class;
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}: return ${minor.class}Writer.class;
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
+ }
+
+ public static Class<?> getWriterImpl( MinorType type, DataMode mode){
+ switch (type) {
+ case UNION:
+ return UnionWriter.class;
+ case MAP:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return SingleMapWriter.class;
+ case REPEATED:
+ return RepeatedMapWriter.class;
+ }
+ case LIST:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return UnionListWriter.class;
+ case REPEATED:
+ return RepeatedListWriter.class;
+ }
+
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}WriterImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}WriterImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}WriterImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
+ }
+
+ public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
+ switch (type) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ${minor.class}HolderReaderImpl.class;
+ case OPTIONAL:
+ return Nullable${minor.class}HolderReaderImpl.class;
+ case REPEATED:
+ return Repeated${minor.class}HolderReaderImpl.class;
+ }
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
+ }
+
+ public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
+ return getNewVector(field, allocator, null);
+ }
+ public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+ MajorType type = field.getType();
+
+ switch (type.getMinorType()) {
+
+ case UNION:
+ return new UnionVector(field, allocator, callBack);
+
+ case MAP:
+ switch (type.getMode()) {
+ case REQUIRED:
+ case OPTIONAL:
+ return new MapVector(field, allocator, callBack);
+ case REPEATED:
+ return new RepeatedMapVector(field, allocator, callBack);
+ }
+ case LIST:
+ switch (type.getMode()) {
+ case REPEATED:
+ return new RepeatedListVector(field, allocator, callBack);
+ case OPTIONAL:
+ case REQUIRED:
+ return new ListVector(field, allocator, callBack);
+ }
+<#list vv. types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Vector(field, allocator);
+ case OPTIONAL:
+ return new Nullable${minor.class}Vector(field, allocator);
+ case REPEATED:
+ return new Repeated${minor.class}Vector(field, allocator);
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ return new ObjectVector(field, allocator) ;
+ default:
+ break;
+ }
+ // All ValueVector types have been handled.
+ throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
+ }
+
+ public static ValueHolder getValue(ValueVector vector, int index) {
+ MajorType type = vector.getField().getType();
+ ValueHolder holder;
+ switch(type.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
+ minor.class?starts_with("Decimal28") || minor.class?starts_with("Decimal38")>
+ switch (type.getMode()) {
+ case REQUIRED:
+ holder = new ${minor.class}Holder();
+ ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
+ return holder;
+ case OPTIONAL:
+ holder = new Nullable${minor.class}Holder();
+ ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
+ if (((Nullable${minor.class}Holder)holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
+ }
+ return holder;
+ }
+ <#else>
+ switch (type.getMode()) {
+ case REQUIRED:
+ holder = new ${minor.class}Holder();
+ ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
+ return holder;
+ case OPTIONAL:
+ holder = new Nullable${minor.class}Holder();
+ ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
+ if (((Nullable${minor.class}Holder)holder).isSet == 1) {
+ ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
+ }
+ return holder;
+ }
+ </#if>
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ holder = new ObjectHolder();
+ ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index) ;
+ break;
+ }
+
+ throw new UnsupportedOperationException(buildErrorMessage("get value", type));
+ }
+
+ public static void setValue(ValueVector vector, int index, ValueHolder holder) {
+ MajorType type = vector.getField().getType();
+
+ switch(type.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
+ return;
+ case OPTIONAL:
+ if (((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
+ }
+ return;
+ }
+ </#list>
+</#list>
+ case GENERIC_OBJECT:
+ ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
+ return;
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("set value", type));
+ }
+ }
+
+ public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
+ MajorType type = vector.getField().getType();
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
+ return;
+ case OPTIONAL:
+ if (((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
+ } else {
+ ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
+ }
+ return;
+ }
+ </#list>
+ </#list>
+ case GENERIC_OBJECT:
+ ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
+ }
+ }
+
+ public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
+ MajorType type1 = v1.getField().getType();
+ MajorType type2 = v2.getField().getType();
+
+ if (type1.getMinorType() != type2.getMinorType()) {
+ return false;
+ }
+
+ switch(type1.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) ==
+ ((${minor.class}Vector) v2).getAccessor().get(v2index) )
+ return true;
+ break;
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ return false;
+ }
+
+ /**
+ * Create a ValueHolder of MajorType.
+ * @param type
+ * @return
+ */
+ public static ValueHolder createValueHolder(MajorType type) {
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Holder();
+ case OPTIONAL:
+ return new Nullable${minor.class}Holder();
+ case REPEATED:
+ return new Repeated${minor.class}Holder();
+ }
+ </#list>
+ </#list>
+ case GENERIC_OBJECT:
+ return new ObjectHolder();
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
+ }
+ }
+
+ public static boolean isNull(ValueHolder holder) {
+ MajorType type = getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return true;
+ case OPTIONAL:
+ return ((Nullable${minor.class}Holder) holder).isSet == 0;
+ case REPEATED:
+ return true;
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
+ }
+ }
+
+ public static ValueHolder deNullify(ValueHolder holder) {
+ MajorType type = getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+
+ switch (type.getMode()) {
+ case REQUIRED:
+ return holder;
+ case OPTIONAL:
+ if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
+ ${minor.class}Holder newHolder = new ${minor.class}Holder();
+
+ <#assign fields = minor.fields!type.fields />
+ <#list fields as field>
+ newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
+ </#list>
+
+ return newHolder;
+ } else {
+ throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
+ }
+ case REPEATED:
+ return holder;
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
+ }
+ }
+
+ public static ValueHolder nullify(ValueHolder holder) {
+ MajorType type = getValueHolderType(holder);
+
+ switch(type.getMinorType()) {
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ switch (type.getMode()) {
+ case REQUIRED:
+ Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
+ newHolder.isSet = 1;
+ <#assign fields = minor.fields!type.fields />
+ <#list fields as field>
+ newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
+ </#list>
+ return newHolder;
+ case OPTIONAL:
+ return holder;
+ case REPEATED:
+ throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
+ }
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
+ }
+ }
+
+ public static MajorType getValueHolderType(ValueHolder holder) {
+
+ if (0 == 1) {
+ return null;
+ }
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ else if (holder instanceof ${minor.class}Holder) {
+ return ((${minor.class}Holder) holder).TYPE;
+ } else if (holder instanceof Nullable${minor.class}Holder) {
+ return ((Nullable${minor.class}Holder) holder).TYPE;
+ }
+ </#list>
+ </#list>
+
+ throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index d9d5fae..f67614a 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -94,7 +94,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
@Override
public void allocateNew() {
if(!allocateNewSafe()){
- throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
+ throw new OutOfMemoryException("Failure while allocating buffer.");
}
}
@@ -124,7 +124,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
* Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
*
* @param valueCount
- * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer
+ * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer
*/
@Override
public void allocateNew(final int valueCount) {
@@ -154,7 +154,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
/**
* Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
*
- * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer
+ * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer
*/
public void reAlloc() {
final long newAllocationSize = allocationSizeInBytes * 2L;
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
index fe91fe6..13bdd4f 100644
--- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -138,7 +138,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public void allocateNew() {
if(!allocateNewSafe()){
- throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
+ throw new OutOfMemoryException("Failure while allocating buffer.");
}
}
@@ -201,7 +201,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
try {
values.allocateNew(valueCount);
bits.allocateNew(valueCount+1);
- } catch(OutOfMemoryRuntimeException e) {
+ } catch(OutOfMemoryException e) {
clear();
throw e;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4524fdbe/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
index 2f5ec18..21f5616 100644
--- a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
@@ -204,7 +204,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
try {
offsets.allocateNew();
values.allocateNew();
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
clear();
throw e;
}
@@ -223,7 +223,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
try {
offsets.allocateNew(valueCount + 1);
values.allocateNew(totalBytes, innerValueCount);
- } catch (OutOfMemoryRuntimeException e) {
+ } catch (OutOfMemoryException e) {
clear();
throw e;
}
@@ -249,7 +249,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
try {
offsets.allocateNew(valueCount + 1);
values.allocateNew(innerValueCount);
- } catch(OutOfMemoryRuntimeException e){
+ } catch(OutOfMemoryException e){
clear();
throw e;
}