You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/08/29 01:34:06 UTC
[17/47] Switch to DrillBuf Add @Inject DrillBuf Move comparison
functions to memory sensitive ones Add scalar replacement functionality for
value holders Simplify date parsing function Add local compiled code caching
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
new file mode 100644
index 0000000..8e752eb
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
@@ -0,0 +1,692 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import io.netty.buffer.DrillBuf;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.util.CoreDecimalUtility;
+
+public class DecimalUtility extends CoreDecimalUtility{
+
+ public final static int MAX_DIGITS = 9;
+ public final static int DIGITS_BASE = 1000000000;
+ public final static int DIGITS_MAX = 999999999;
+ public final static int integerSize = (Integer.SIZE/8);
+
+ public final static String[] decimalToString = {"",
+ "0",
+ "00",
+ "000",
+ "0000",
+ "00000",
+ "000000",
+ "0000000",
+ "00000000",
+ "000000000"};
+
+ public final static long[] scale_long_constants = {
+ 1,
+ 10,
+ 100,
+ 1000,
+ 10000,
+ 100000,
+ 1000000,
+ 10000000,
+ 100000000,
+ 1000000000,
+ 10000000000l,
+ 100000000000l,
+ 1000000000000l,
+ 10000000000000l,
+ 100000000000000l,
+ 1000000000000000l,
+ 10000000000000000l,
+ 100000000000000000l,
+ 1000000000000000000l};
+
+ /*
+ * Simple function that returns the static precomputed
+ * power of ten, instead of using Math.pow
+ */
+ public static long getPowerOfTen(int power) {
+ assert power >= 0 && power < scale_long_constants.length;
+ return scale_long_constants[(power)];
+ }
+
+ /*
+ * Math.pow returns a double and while multiplying with large digits
+ * in the decimal data type we encounter noise. So instead of multiplying
+ * with Math.pow we use the static constants to perform the multiplication
+ */
+ public static long adjustScaleMultiply(long input, int factor) {
+ int index = Math.abs(factor);
+ assert index >= 0 && index < scale_long_constants.length;
+ if (factor >= 0) {
+ return input * scale_long_constants[index];
+ } else {
+ return input / scale_long_constants[index];
+ }
+ }
+
+ public static long adjustScaleDivide(long input, int factor) {
+ int index = Math.abs(factor);
+ assert index >= 0 && index < scale_long_constants.length;
+ if (factor >= 0) {
+ return input / scale_long_constants[index];
+ } else {
+ return input * scale_long_constants[index];
+ }
+ }
+
+ /* Given the number of actual digits this function returns the
+ * number of indexes it will occupy in the array of integers
+ * which are stored in base 1 billion
+ */
+ public static int roundUp(int ndigits) {
+ return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+ }
+
+ /* Returns a string representation of the given integer
+ * If the length of the given integer is less than the
+ * passed length, this function will prepend zeroes to the string
+ */
+ public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
+ String value = ((Integer) number).toString();
+ int length = value.length();
+
+ StringBuilder str = new StringBuilder();
+ str.append(decimalToString[desiredLength - length]);
+ str.append(value);
+
+ return str;
+ }
+
+ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
+ String value = ((Long) number).toString();
+ int length = value.length();
+
+ StringBuilder str = new StringBuilder();
+
+ // Desired length can be > MAX_DIGITS
+ int zeroesLength = desiredLength - length;
+ while (zeroesLength > MAX_DIGITS) {
+ str.append(decimalToString[MAX_DIGITS]);
+ zeroesLength -= MAX_DIGITS;
+ }
+ str.append(decimalToString[zeroesLength]);
+ str.append(value);
+
+ return str;
+ }
+
+ public static BigDecimal getBigDecimalFromIntermediate(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
+
+ // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
+ return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false);
+ }
+
+ public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) {
+
+ // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
+ return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true);
+ }
+
+ public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) {
+ byte[] value = new byte[length];
+ bytebuf.getBytes(start, value, 0, length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
+
+ /* Create a BigDecimal object using the data in the DrillBuf.
+ * This function assumes that data is provided in a non-dense format
+ * It works on both sparse and intermediate representations.
+ */
+ public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) {
+
+ // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
+ int actualDigits;
+
+ // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out
+ BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
+
+ BigInteger base = BigInteger.valueOf(DIGITS_BASE);
+
+ for (int i = 1; i < nDecimalDigits; i++) {
+
+ BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * integerSize)));
+ decimalDigits = decimalDigits.multiply(base);
+ decimalDigits = decimalDigits.add(temp);
+ }
+
+ // Truncate any additional padding we might have added
+ if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
+ BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
+ decimalDigits = decimalDigits.divide(truncate);
+ }
+
+ // set the sign
+ if ((data.getInt(startIndex) & 0x80000000) != 0) {
+ decimalDigits = decimalDigits.negate();
+ }
+
+ BigDecimal decimal = new BigDecimal(decimalDigits, scale);
+
+ return decimal;
+ }
+
+ /* This function returns a BigDecimal object from the dense decimal representation.
+ * First step is to convert the dense representation into an intermediate representation
+ * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object
+ */
+ public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
+
+ /* This method converts the dense representation to
+ * an intermediate representation. The intermediate
+ * representation has one more integer than the dense
+ * representation.
+ */
+ byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * integerSize)];
+
+ // Start storing from the least significant byte of the first integer
+ int intermediateIndex = 3;
+
+ int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
+ int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
+
+ int maskIndex;
+ int shiftOrder;
+ byte shiftBits;
+
+ // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
+ if (maxPrecision == 38) {
+ maskIndex = 0;
+ shiftOrder = 6;
+ shiftBits = 0x00;
+ intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
+ } else if (maxPrecision == 28) {
+ maskIndex = 1;
+ shiftOrder = 4;
+ shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
+ intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
+ } else {
+ throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
+ }
+
+ int inputIndex = 1;
+ boolean sign = false;
+
+ if ((data.getByte(startIndex) & 0x80) != 0) {
+ sign = true;
+ }
+
+ while (inputIndex < width) {
+
+ intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
+
+ shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
+
+ inputIndex++;
+ intermediateIndex++;
+
+ if (((inputIndex - 1) % integerSize) == 0) {
+ shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
+ maskIndex++;
+ shiftOrder -= 2;
+ }
+
+ }
+ /* copy the last byte */
+ intermediateBytes[intermediateIndex] = shiftBits;
+
+ if (sign == true) {
+ intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
+ }
+ DrillBuf intermediate = data.getAllocator().buffer(intermediateBytes.length);
+ intermediate.setBytes(0, intermediateBytes);
+
+ BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
+ intermediate.release();
+ return ret;
+ }
+
+ /*
+ * Function converts the BigDecimal and stores it in out internal sparse representation
+ */
+ public static void getSparseFromBigDecimal(BigDecimal input, DrillBuf data, int startIndex, int scale, int precision, int nDecimalDigits) {
+
+ // Initialize the buffer
+ for (int i = 0; i < nDecimalDigits; i++) {
+ data.setInt(startIndex + (i * integerSize), 0);
+ }
+
+ boolean sign = false;
+
+ if (input.signum() == -1) {
+ // negative input
+ sign = true;
+ input = input.abs();
+ }
+
+ // Truncate the input as per the scale provided
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+ // Separate out the integer part
+ BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
+
+ int destIndex = nDecimalDigits - roundUp(scale) - 1;
+
+ // we use base 1 billion integer digits for out integernal representation
+ BigDecimal base = new BigDecimal(DIGITS_BASE);
+
+ while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
+ // store the modulo as the integer value
+ data.setInt(startIndex + (destIndex * integerSize), (integerPart.remainder(base)).intValue());
+ destIndex--;
+ // Divide by base 1 billion
+ integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
+ }
+
+ /* Sparse representation contains padding of additional zeroes
+ * so each digit contains MAX_DIGITS for ease of arithmetic
+ */
+ int actualDigits;
+ if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
+ // Pad additional zeroes
+ scale = scale + (MAX_DIGITS - actualDigits);
+ input = input.setScale(scale, BigDecimal.ROUND_DOWN);
+ }
+
+ //separate out the fractional part
+ BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
+
+ destIndex = nDecimalDigits - 1;
+
+ while (scale > 0) {
+ // Get next set of MAX_DIGITS (9) store it in the DrillBuf
+ fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
+ BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
+
+ data.setInt(startIndex + (destIndex * integerSize), (temp.unscaledValue().intValue()));
+ destIndex--;
+
+ fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
+ scale -= MAX_DIGITS;
+ }
+
+ // Set the negative sign
+ if (sign == true) {
+ data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
+ }
+
+ }
+
+
+ public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+ // Truncate or pad to set the input to the correct scale
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+
+ return (input.unscaledValue().longValue());
+ }
+
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
+ return BigDecimal.valueOf(input, scale);
+ }
+
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
+ return BigDecimal.valueOf(input, scale);
+ }
+
+
+ public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) {
+
+ int invert = 1;
+
+ /* If signs are different then simply look at the
+ * sign of the two inputs and determine which is greater
+ */
+ if (leftSign != rightSign) {
+
+ return((leftSign == true) ? -1 : 1);
+ } else if(leftSign == true) {
+ /* Both inputs are negative, at the end we will
+ * have to invert the comparison
+ */
+ invert = -1;
+ }
+
+ int cmp = 0;
+
+ for (int i = 0; i < width; i++) {
+ byte leftByte = left.getByte(leftStart + i);
+ byte rightByte = right.getByte(rightStart + i);
+ // Unsigned byte comparison
+ if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
+ cmp = 1;
+ break;
+ } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
+ cmp = -1;
+ break;
+ }
+ }
+ cmp *= invert; // invert the comparison if both were negative values
+
+ return cmp;
+ }
+
+ public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) {
+ int value = buffer.getInt(start + (index * 4));
+
+ if (index == 0) {
+ /* the first byte contains sign bit, return value without it */
+ value = (value & 0x7FFFFFFF);
+ }
+ return value;
+ }
+
+ public static void setInteger(DrillBuf buffer, int start, int index, int value) {
+ buffer.setInt(start + (index * 4), value);
+ }
+
+ public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
+
+ int invert = 1;
+
+ if (absCompare == false) {
+ if (leftSign != rightSign) {
+ return (leftSign == true) ? -1 : 1;
+ }
+
+ // Both values are negative invert the outcome of the comparison
+ if (leftSign == true) {
+ invert = -1;
+ }
+ }
+
+ int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
+ return cmp * invert;
+ }
+ public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
+ /* compute the number of integer digits in each decimal */
+ int leftInt = leftPrecision - leftScale;
+ int rightInt = rightPrecision - rightScale;
+
+ /* compute the number of indexes required for storing integer digits */
+ int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt);
+ int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt);
+
+ /* compute number of indexes required for storing scale */
+ int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale);
+ int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale);
+
+ /* compute index of the most significant integer digits */
+ int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+ int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+
+ int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
+ int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
+
+ /* Discard the zeroes in the integer part */
+ while (leftIndex1 < leftStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+ break;
+ }
+
+ /* Digit in this location is zero, decrement the actual number
+ * of integer digits
+ */
+ leftIntRoundedUp--;
+ leftIndex1++;
+ }
+
+ /* If we reached the stop index then the number of integers is zero */
+ if (leftIndex1 == leftStopIndex) {
+ leftIntRoundedUp = 0;
+ }
+
+ while (rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+ break;
+ }
+
+ /* Digit in this location is zero, decrement the actual number
+ * of integer digits
+ */
+ rightIntRoundedUp--;
+ rightIndex1++;
+ }
+
+ if (rightIndex1 == rightStopIndex) {
+ rightIntRoundedUp = 0;
+ }
+
+ /* We have the accurate number of non-zero integer digits,
+ * if the number of integer digits are different then we can determine
+ * which decimal is larger and needn't go down to comparing individual values
+ */
+ if (leftIntRoundedUp > rightIntRoundedUp) {
+ return 1;
+ }
+ else if (rightIntRoundedUp > leftIntRoundedUp) {
+ return -1;
+ }
+
+ /* The number of integer digits are the same, set the each index
+ * to the first non-zero integer and compare each digit
+ */
+ leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+ rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+
+ while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+ return 1;
+ }
+ else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+ return -1;
+ }
+
+ leftIndex1++;
+ rightIndex1++;
+ }
+
+ /* The integer part of both the decimal's are equal, now compare
+ * each individual fractional part. Set the index to be at the
+ * beginning of the fractional part
+ */
+ leftIndex1 = leftStopIndex;
+ rightIndex1 = rightStopIndex;
+
+ /* Stop indexes will be the end of the array */
+ leftStopIndex = nDecimalDigits;
+ rightStopIndex = nDecimalDigits;
+
+ /* compare the two fractional parts of the decimal */
+ while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+ return 1;
+ }
+ else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+ return -1;
+ }
+
+ leftIndex1++;
+ rightIndex1++;
+ }
+
+ /* Till now the fractional part of the decimals are equal, check
+ * if one of the decimal has fractional part that is remaining
+ * and is non-zero
+ */
+ while (leftIndex1 < leftStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+ return 1;
+ }
+ leftIndex1++;
+ }
+
+ while(rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+ return -1;
+ }
+ rightIndex1++;
+ }
+
+ /* Both decimal values are equal */
+ return 0;
+ }
+
+ public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
+ byte[] value = Arrays.copyOfRange(bytes, start, start + length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
+
+ public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
+ int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale);
+ int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale);
+
+ if (desiredScale < currentScale) {
+
+ boolean roundUp = false;
+
+ //Extract the first digit to be truncated to check if we need to round up
+ int truncatedScaleIndex = desiredScale + 1;
+ if (truncatedScaleIndex <= currentScale) {
+ int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1;
+ extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex);
+ int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex);
+ int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
+ if (temp != 0) {
+ extractDigit = extractDigit / (int) (Math.pow(10, temp));
+ }
+ if ((extractDigit % 10) > 4) {
+ roundUp = true;
+ }
+ }
+
+ // Get the source index beyond which we will truncate
+ int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1;
+ int srcIndex = srcIntIndex + newScaleRoundedUp;
+
+ // Truncate the remaining fractional part, move the integer part
+ int destIndex = nDecimalDigits - 1;
+ if (srcIndex != destIndex) {
+ while (srcIndex >= 0) {
+ setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--));
+ }
+
+ // Set the remaining portion of the decimal to be zeroes
+ while (destIndex >= 0) {
+ setInteger(result, start, destIndex--, 0);
+ }
+ srcIndex = nDecimalDigits - 1;
+ }
+
+ // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit
+ int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS);
+ if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
+ truncateFactor = (int) Math.pow(10, truncateFactor);
+ int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1);
+ fractionalDigits /= truncateFactor;
+ setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor);
+ }
+
+ // Finally round up the digit if needed
+ if (roundUp == true) {
+ srcIndex = nDecimalDigits - 1;
+ int carry;
+ if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) {
+ carry = truncateFactor;
+ } else {
+ carry = 1;
+ }
+
+ while (srcIndex >= 0) {
+ int value = getIntegerFromSparseBuffer(result, start, srcIndex);
+ value += carry;
+
+ if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) {
+ setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE);
+ carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE;
+ } else {
+ setInteger(result, start, srcIndex--, value);
+ carry = 0;
+ break;
+ }
+ }
+ }
+ } else if (desiredScale > currentScale) {
+ // Add fractional digits to the decimal
+
+ // Check if we need to shift the decimal digits to the left
+ if (newScaleRoundedUp > origScaleRoundedUp) {
+ int srcIndex = 0;
+ int destIndex = newScaleRoundedUp - origScaleRoundedUp;
+
+ // Check while extending scale, we are not overwriting integer part
+ while (srcIndex < destIndex) {
+ if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) {
+ throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified");
+ }
+ }
+
+ srcIndex = 0;
+ while (destIndex < nDecimalDigits) {
+ setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++));
+ }
+
+ // Clear the remaining part
+ while (srcIndex < nDecimalDigits) {
+ setInteger(result, start, srcIndex++, 0);
+ }
+ }
+ }
+ }
+
+ public static int getFirstFractionalDigit(int decimal, int scale) {
+ if (scale == 0) {
+ return 0;
+ }
+ int temp = (int) adjustScaleDivide(decimal, scale - 1);
+ return Math.abs(temp % 10);
+ }
+
+ public static int getFirstFractionalDigit(long decimal, int scale) {
+ if (scale == 0) {
+ return 0;
+ }
+ long temp = adjustScaleDivide(decimal, scale - 1);
+ return (int) (Math.abs(temp % 10));
+ }
+
+ public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) {
+ if (scale == 0) {
+ return 0;
+ }
+
+ int index = nDecimalDigits - roundUp(scale);
+ return (int) (adjustScaleDivide(data.getInt(start + (index * integerSize)), MAX_DIGITS - 1));
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
index 7d0fbc7..b711c66 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
@@ -17,9 +17,10 @@
*/
package org.apache.drill.exec.vector;
-import java.util.Iterator;
+import io.netty.buffer.DrillBuf;
+import io.netty.buffer.DrillBuf;
-import io.netty.buffer.ByteBuf;
+import java.util.Iterator;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
@@ -30,7 +31,7 @@ import com.google.common.collect.Iterators;
public abstract class BaseDataValueVector extends BaseValueVector{
- protected ByteBuf data = DeadBuf.DEAD_BUFFER;
+ protected DrillBuf data;
protected int valueCount;
protected int currentValueCount;
@@ -40,13 +41,13 @@ public abstract class BaseDataValueVector extends BaseValueVector{
}
/**
- * Release the underlying ByteBuf and reset the ValueVector
+ * Release the underlying DrillBuf and reset the ValueVector
*/
@Override
public void clear() {
- if (data != DeadBuf.DEAD_BUFFER) {
+ if (data != null) {
data.release();
- data = DeadBuf.DEAD_BUFFER;
+ data = data.getAllocator().getEmpty();
valueCount = 0;
}
}
@@ -61,12 +62,12 @@ public abstract class BaseDataValueVector extends BaseValueVector{
@Override
- public ByteBuf[] getBuffers(){
- ByteBuf[] out;
+ public DrillBuf[] getBuffers(){
+ DrillBuf[] out;
if(valueCount == 0){
- out = new ByteBuf[0];
+ out = new DrillBuf[0];
}else{
- out = new ByteBuf[]{data};
+ out = new DrillBuf[]{data};
data.readerIndex(0);
data.retain();
}
@@ -82,10 +83,14 @@ public abstract class BaseDataValueVector extends BaseValueVector{
@Override
public abstract SerializedField getMetadata();
- public ByteBuf getData(){
+ public DrillBuf getData(){
return data;
}
+ public long getDataAddr(){
+ return data.memoryAddress();
+ }
+
@Override
public Iterator<ValueVector> iterator() {
return Iterators.emptyIterator();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index e310b81..cf62d54 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -63,7 +63,7 @@ public abstract class BaseValueVector implements ValueVector{
abstract public ByteBuf getData();
- abstract class BaseAccessor implements ValueVector.Accessor{
+ abstract static class BaseAccessor implements ValueVector.Accessor{
public abstract int getValueCount();
public void reset(){}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 01fcfe0..0076a1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -17,8 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.AccountingByteBuf;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.exec.expr.holders.BitHolder;
@@ -113,11 +112,11 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
}
@Override
- public int load(int valueCount, ByteBuf buf) {
+ public int load(int valueCount, DrillBuf buf) {
clear();
this.valueCount = valueCount;
int len = getSizeFromCount(valueCount);
- data = buf.slice(0, len);
+ data = (DrillBuf) buf.slice(0, len);
data.retain();
return len;
}
@@ -136,7 +135,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
}
@Override
- public void load(SerializedField metadata, ByteBuf buffer) {
+ public void load(SerializedField metadata, DrillBuf buffer) {
assert this.field.matches(metadata);
int loaded = load(metadata.getValueCount(), buffer);
assert metadata.getBufferLength() == loaded;
@@ -181,7 +180,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
int offset = startIndex % 8;
if (offset == 0) {
// slice
- target.data = this.data.slice(firstByte, byteSize);
+ target.data = (DrillBuf) this.data.slice(firstByte, byteSize);
target.data.retain();
} else {
// Copy data
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
index c5e1ae7..b546b9e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
@@ -17,26 +17,26 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
public interface FixedWidthVector extends ValueVector{
-
+
/**
* Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
*
* @param valueCount Number of values in the vector.
*/
public void allocateNew(int valueCount);
-
+
/**
* Load the records in the provided buffer based on the given number of values.
* @param valueCount Number of values the buffer contains.
* @param buf Incoming buffer.
* @return The number of bytes of the buffer that were consumed.
*/
- public int load(int valueCount, ByteBuf buf);
-
-
+ public int load(int valueCount, DrillBuf buf);
+
+
public abstract Mutator getMutator();
/**
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
index 650f5ae..2c215ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -18,6 +18,8 @@
package org.apache.drill.exec.vector;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
+
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.exec.expr.holders.ObjectHolder;
import org.apache.drill.exec.memory.BufferAllocator;
@@ -164,12 +166,12 @@ public class ObjectVector extends BaseValueVector{
}
@Override
- public ByteBuf[] getBuffers() {
+ public DrillBuf[] getBuffers() {
throw new UnsupportedOperationException("ObjectVector does not support this");
}
@Override
- public void load(UserBitShared.SerializedField metadata, ByteBuf buffer) {
+ public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
throw new UnsupportedOperationException("ObjectVector does not support this");
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
index 6d03541..a28caf1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
public interface RepeatedFixedWidthVector extends ValueVector{
/**
@@ -27,7 +27,7 @@ public interface RepeatedFixedWidthVector extends ValueVector{
* @param childValueCount Number of supported values in the vector.
*/
public void allocateNew(int parentValueCount, int childValueCount);
-
+
/**
* Load the records in the provided buffer based on the given number of values.
* @param parentValueCount Number of separate repeating groupings.
@@ -35,7 +35,7 @@ public interface RepeatedFixedWidthVector extends ValueVector{
* @param buf Incoming buffer.
* @return The number of bytes of the buffer that were consumed.
*/
- public int load(int parentValueCount, int childValueCount, ByteBuf buf);
+ public int load(int parentValueCount, int childValueCount, DrillBuf buf);
public abstract RepeatedAccessor getAccessor();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
index a2c884e..2e7689c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
public interface RepeatedVariableWidthVector extends ValueVector{
/**
@@ -45,5 +45,5 @@ public interface RepeatedVariableWidthVector extends ValueVector{
* @param buf Incoming buffer.
* @return The number of bytes of the buffer that were consumed.
*/
- public int load(int dataBytes, int parentValueCount, int childValueCount, ByteBuf buf);
+ public int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
index 6968c21..31afb6f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
@@ -17,20 +17,18 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.SwappedByteBuf;
-import io.netty.buffer.Unpooled;
-import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.buffer.DrillBuf;
import java.math.BigDecimal;
-import java.nio.ByteOrder;
-import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.holders.Decimal18Holder;
import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
import org.apache.drill.exec.expr.holders.Decimal9Holder;
import org.apache.drill.exec.expr.holders.IntervalDayHolder;
import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.util.DecimalUtility;
import com.google.common.base.Charsets;
@@ -38,13 +36,24 @@ import com.google.common.base.Charsets;
public class ValueHolderHelper {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
- public static VarCharHolder getVarCharHolder(String s){
+ public static VarCharHolder getVarCharHolder(DrillBuf buf, String s){
VarCharHolder vch = new VarCharHolder();
byte[] b = s.getBytes(Charsets.UTF_8);
vch.start = 0;
vch.end = b.length;
- vch.buffer = UnpooledByteBufAllocator.DEFAULT.buffer(b.length).order(ByteOrder.LITTLE_ENDIAN); // use the length of input string to allocate buffer.
+ vch.buffer = buf.reallocIfNeeded(b.length);
+ vch.buffer.setBytes(0, b);
+ return vch;
+ }
+
+ public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
+ VarCharHolder vch = new VarCharHolder();
+
+ byte[] b = s.getBytes(Charsets.UTF_8);
+ vch.start = 0;
+ vch.end = b.length;
+ vch.buffer = a.buffer(b.length); //
vch.buffer.setBytes(0, b);
return vch;
}
@@ -53,7 +62,7 @@ public class ValueHolderHelper {
IntervalDayHolder dch = new IntervalDayHolder();
dch.days = days;
- dch.milliSeconds = millis;
+ dch.milliseconds = millis;
return dch;
}
@@ -77,7 +86,7 @@ public class ValueHolderHelper {
return dch;
}
- public static Decimal28SparseHolder getDecimal28Holder(String decimal) {
+ public static Decimal28SparseHolder getDecimal28Holder(DrillBuf buf, String decimal) {
Decimal28SparseHolder dch = new Decimal28SparseHolder();
@@ -85,18 +94,15 @@ public class ValueHolderHelper {
dch.scale = bigDecimal.scale();
dch.precision = bigDecimal.precision();
- dch.setSign(bigDecimal.signum() == -1);
+ Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
dch.start = 0;
-
- dch.buffer = Unpooled.wrappedBuffer(new byte[5 * DecimalUtility.integerSize]);
- dch.buffer = new SwappedByteBuf(dch.buffer);
+ dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.integerSize);
DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
return dch;
}
- public static Decimal38SparseHolder getDecimal38Holder(String decimal) {
-
+ public static Decimal38SparseHolder getDecimal38Holder(DrillBuf buf, String decimal) {
Decimal38SparseHolder dch = new Decimal38SparseHolder();
@@ -104,12 +110,9 @@ public class ValueHolderHelper {
dch.scale = bigDecimal.scale();
dch.precision = bigDecimal.precision();
- dch.setSign(bigDecimal.signum() == -1);
+ Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
dch.start = 0;
-
-
- dch.buffer = Unpooled.wrappedBuffer(new byte[dch.maxPrecision * DecimalUtility.integerSize]);
- dch.buffer = new SwappedByteBuf(dch.buffer);
+ dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.integerSize);
DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
return dch;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 8b871fc..f7f010a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.io.Closeable;
@@ -56,7 +56,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
public void close();
/**
- * Release the underlying ByteBuf and reset the ValueVector to empty.
+ * Release the underlying DrillBuf and reset the ValueVector to empty.
*/
public void clear();
@@ -99,9 +99,9 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus
* external classes shouldn't hold a reference to it (unless they change it).
*
- * @return The underlying ByteBuf.
+ * @return The underlying DrillBuf.
*/
- public abstract ByteBuf[] getBuffers();
+ public abstract DrillBuf[] getBuffers();
/**
* Load the data provided in the buffer. Typically used when deserializing from the wire.
@@ -111,7 +111,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param buffer
* The buffer that contains the ValueVector.
*/
- public void load(SerializedField metadata, ByteBuf buffer);
+ public void load(SerializedField metadata, DrillBuf buffer);
/**
* Get the metadata for this field. Used in serialization
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
index 5c1d3ab..ff3ee63 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
public interface VariableWidthVector extends ValueVector{
@@ -28,15 +28,13 @@ public interface VariableWidthVector extends ValueVector{
* @param valueCount Number of values in the vector.
*/
public void allocateNew(int totalBytes, int valueCount);
-
+
/**
* Provide the maximum amount of variable width bytes that can be stored int his vector.
* @return
*/
public int getByteCapacity();
- public int getCurrentSizeInBytes();
-
/**
* Load the records in the provided buffer based on the given number of values.
* @param dataBytes The number of bytes associated with the data array.
@@ -44,8 +42,8 @@ public interface VariableWidthVector extends ValueVector{
* @param buf Incoming buffer.
* @return The number of bytes of the buffer that were consumed.
*/
- public int load(int dataBytes, int valueCount, ByteBuf buf);
-
+ public int load(int dataBytes, int valueCount, DrillBuf buf);
+
public abstract VariableWidthMutator getMutator();
public abstract VariableWidthAccessor getAccessor();
@@ -54,6 +52,8 @@ public interface VariableWidthVector extends ValueVector{
public int getValueLength(int index);
}
+ public int getCurrentSizeInBytes();
+
public interface VariableWidthMutator extends Mutator {
public boolean setValueLengthSafe(int index, int length);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
index e946afe..619dd95 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector;
-import io.netty.buffer.AccountingByteBuf;
+import io.netty.buffer.DrillBuf;
import io.netty.buffer.ByteBuf;
public class VectorTrimmer {
@@ -25,7 +25,7 @@ public class VectorTrimmer {
public static void trim(ByteBuf data, int idx) {
data.writerIndex(idx);
- if (data instanceof AccountingByteBuf) {
+ if (data instanceof DrillBuf) {
// data.capacity(idx);
data.writerIndex(idx);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 480b863..c91c397 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector.complex;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.HashMap;
import java.util.Iterator;
@@ -276,18 +276,18 @@ public class MapVector extends AbstractContainerVector {
}
@Override
- public ByteBuf[] getBuffers() {
- List<ByteBuf> bufs = Lists.newArrayList();
+ public DrillBuf[] getBuffers() {
+ List<DrillBuf> bufs = Lists.newArrayList();
for(ValueVector v : vectors.values()){
- for(ByteBuf b : v.getBuffers()){
+ for(DrillBuf b : v.getBuffers()){
bufs.add(b);
}
}
- return bufs.toArray(new ByteBuf[bufs.size()]);
+ return bufs.toArray(new DrillBuf[bufs.size()]);
}
@Override
- public void load(SerializedField metadata, ByteBuf buf) {
+ public void load(SerializedField metadata, DrillBuf buf) {
List<SerializedField> fields = metadata.getChildList();
valueCount = metadata.getValueCount();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index 57c47d4..00716cf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector.complex;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.Collections;
import java.util.Iterator;
@@ -354,7 +354,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
}
@Override
- public ByteBuf[] getBuffers() {
+ public DrillBuf[] getBuffers() {
return ArrayUtils.addAll(offsets.getBuffers(), vector.getBuffers());
}
@@ -364,7 +364,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
}
@Override
- public void load(SerializedField metadata, ByteBuf buf) {
+ public void load(SerializedField metadata, DrillBuf buf) {
SerializedField childField = metadata.getChildList().get(0);
int bufOffset = offsets.load(metadata.getValueCount()+1, buf);
@@ -429,7 +429,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
}
@Override
- public int load(int parentValueCount, int childValueCount, ByteBuf buf) {
+ public int load(int parentValueCount, int childValueCount, DrillBuf buf) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index c67c047..f7baaa9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.vector.complex;
-import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.util.Iterator;
import java.util.List;
@@ -307,20 +307,20 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
}
@Override
- public ByteBuf[] getBuffers() {
- List<ByteBuf> bufs = Lists.newArrayList(offsets.getBuffers());
+ public DrillBuf[] getBuffers() {
+ List<DrillBuf> bufs = Lists.newArrayList(offsets.getBuffers());
for(ValueVector v : vectors.values()){
- for(ByteBuf b : v.getBuffers()){
+ for(DrillBuf b : v.getBuffers()){
bufs.add(b);
}
}
- return bufs.toArray(new ByteBuf[bufs.size()]);
+ return bufs.toArray(new DrillBuf[bufs.size()]);
}
@Override
- public void load(SerializedField metadata, ByteBuf buf) {
+ public void load(SerializedField metadata, DrillBuf buf) {
List<SerializedField> fields = metadata.getChildList();
int bufOffset = offsets.load(metadata.getValueCount()+1, buf);
@@ -513,7 +513,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
}
@Override
- public int load(int parentValueCount, int childValueCount, ByteBuf buf) {
+ public int load(int parentValueCount, int childValueCount, DrillBuf buf) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index d365c6f..59035cb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -17,9 +17,7 @@
*/
package org.apache.drill.exec.vector.complex.fn;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
-import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.buffer.DrillBuf;
import java.io.IOException;
import java.io.Reader;
@@ -33,11 +31,10 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.core.JsonToken;
import com.google.common.base.Charsets;
public class JsonReader {
@@ -48,10 +45,12 @@ public class JsonReader {
private final JsonFactory factory = new JsonFactory();
private JsonParser parser;
+ private DrillBuf workBuf;
- public JsonReader() throws JsonParseException, IOException {
+ public JsonReader(DrillBuf managedBuf) throws JsonParseException, IOException {
factory.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
factory.configure(Feature.ALLOW_COMMENTS, true);
+ this.workBuf = managedBuf;
}
public boolean write(Reader reader, ComplexWriter writer) throws JsonParseException, IOException {
@@ -134,9 +133,9 @@ public class JsonReader {
VarCharHolder vh = new VarCharHolder();
String value = parser.getText();
byte[] b = value.getBytes(Charsets.UTF_8);
- ByteBuf d = UnpooledByteBufAllocator.DEFAULT.buffer(b.length);
- d.setBytes(0, b);
- vh.buffer = d;
+ ensure(b.length);
+ workBuf.setBytes(0, b);
+ vh.buffer = workBuf;
vh.start = 0;
vh.end = b.length;
map.varChar(fieldName).write(vh);
@@ -152,6 +151,10 @@ public class JsonReader {
}
+ private void ensure(int length){
+ workBuf = workBuf.reallocIfNeeded(length);
+ }
+
private void writeData(ListWriter list) throws JsonParseException, IOException {
list.start();
outside: while(true){
@@ -197,9 +200,9 @@ public class JsonReader {
VarCharHolder vh = new VarCharHolder();
String value = parser.getText();
byte[] b = value.getBytes(Charsets.UTF_8);
- ByteBuf d = UnpooledByteBufAllocator.DEFAULT.buffer(b.length);
- d.setBytes(0, b);
- vh.buffer = d;
+ ensure(b.length);
+ workBuf.setBytes(0, b);
+ vh.buffer = workBuf;
vh.start = 0;
vh.end = b.length;
list.varChar().write(vh);
@@ -209,7 +212,7 @@ public class JsonReader {
}
}
list.end();
-
-
+
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
index cf885a4..54e063b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java
@@ -18,6 +18,8 @@
package org.apache.drill.exec.vector.complex.fn;
+import io.netty.buffer.DrillBuf;
+
import java.io.IOException;
import java.io.Reader;
@@ -35,10 +37,10 @@ public class JsonReaderWithState {
private JsonRecordSplitter splitter;
private JsonReader jsonReader;
- public JsonReaderWithState(JsonRecordSplitter splitter) throws IOException{
+ public JsonReaderWithState(JsonRecordSplitter splitter, DrillBuf workspace) throws IOException{
this.splitter = splitter;
reader = splitter.getNextReader();
- jsonReader = new JsonReader();
+ jsonReader = new JsonReader(workspace);
}
public WriteState write(ComplexWriter writer) throws JsonParseException, IOException {
@@ -50,7 +52,7 @@ public class JsonReaderWithState {
}
jsonReader.write(reader, writer);
-
+
if (!writer.ok()) {
reader.reset();
return WriteState.WRITE_FAILED;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
index 76d642c..c7527d5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.work.batch;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.DrillBuf;
import java.io.IOException;
import java.util.List;
@@ -36,10 +37,6 @@ import org.apache.drill.exec.proto.BitData;
import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.record.RawFragmentBatch;
-import org.apache.drill.exec.rpc.RemoteConnection;
-import org.apache.drill.exec.rpc.ResponseSender;
-import org.apache.drill.exec.rpc.data.BitServerConnection;
-import org.apache.drill.exec.rpc.data.DataRpcConfig;
import org.apache.drill.exec.store.LocalSyncableFileSystem;
import org.apache.drill.exec.work.fragment.FragmentManager;
import org.apache.hadoop.conf.Configuration;
@@ -272,7 +269,7 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer {
Stopwatch watch = new Stopwatch();
watch.start();
BitData.FragmentRecordBatch header = BitData.FragmentRecordBatch.parseDelimitedFrom(stream);
- ByteBuf buf = allocator.buffer(bodyLength);
+ DrillBuf buf = allocator.buffer(bodyLength);
buf.writeBytes(stream, bodyLength);
batch = new RawFragmentBatch(null, header, buf, null);
buf.release();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java b/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java
new file mode 100644
index 0000000..4258e60
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec;
+
+
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.SimpleRootExec;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.vector.ValueVector;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
+import com.google.common.io.Files;
+
+public class RunRootExec {
+ public static DrillConfig c = DrillConfig.create();
+
+ public static void main(String args[]) throws Exception {
+ String path = args[0];
+ int iterations = Integer.parseInt(args[1]);
+ Drillbit bit = new Drillbit(c, RemoteServiceSet.getLocalServiceSet());
+ bit.run();
+ DrillbitContext bitContext = bit.getContext();
+ PhysicalPlanReader reader = bitContext.getPlanReader();
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(new File(path), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = bitContext.getFunctionImplementationRegistry();
+ FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), null, registry);
+ SimpleRootExec exec;
+ for (int i = 0; i < iterations; i ++) {
+ Stopwatch w= new Stopwatch().start();
+ System.out.println("STARTITER:" + i);
+ exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ while (exec.next()) {
+ for (ValueVector v : exec) {
+ v.clear();
+ }
+ }
+ System.out.println("ENDITER: " + i);
+ System.out.println("TIME: " + w.elapsed(TimeUnit.MILLISECONDS) + "ms");
+ exec.stop();
+ }
+ context.close();
+ bit.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
index 9090685..e678c29 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -70,6 +71,7 @@ public class DumpCatTest extends ExecTest{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
@@ -87,6 +89,8 @@ public class DumpCatTest extends ExecTest{
}
assertTrue(!context.isFailed());
+ exec.stop();
+
FragmentHandle handle = context.getHandle();
/* Form the file name to which the trace output will dump the record batches */
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
index e1fefdf..896cf4d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.compile;
import java.io.IOException;
import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.cache.local.LocalCache;
import org.apache.drill.exec.compile.ClassTransformer.ClassSet;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
@@ -72,7 +73,7 @@ public class TestClassTransformation extends BaseTestQuery {
public void testCompilationNoDebug() throws CompileException, ClassNotFoundException, ClassTransformationException, IOException {
CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class);
ClassSet classSet = new ClassSet(null, cg.getDefinition().getTemplateClassName(), cg.getMaterializedClassName());
- String sourceCode = cg.generate();
+ String sourceCode = cg.generateAndGet();
sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_OPTION, QueryClassLoader.CompilerPolicy.JDK.name()));
sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_DEBUG_OPTION, false));
@@ -104,9 +105,9 @@ public class TestClassTransformation extends BaseTestQuery {
private void compilationInnerClass(QueryClassLoader loader) throws Exception{
CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class);
- ClassTransformer ct = new ClassTransformer();
- ExampleInner t = ct.getImplementationClass(loader, cg.getDefinition(), cg.generate(), cg.getMaterializedClassName());
-
+ ClassTransformer ct = new ClassTransformer(new LocalCache());
+ Class<? extends ExampleInner> c = (Class<? extends ExampleInner>) ct.getImplementationClass(loader, cg.getDefinition(), cg.generateAndGet(), cg.getMaterializedClassName());
+ ExampleInner t = (ExampleInner) c.newInstance();
t.doOutside();
t.doInsideOutside();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
index 4d0cbcd..2265347 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
@@ -69,7 +69,7 @@ public class TestEvaluationVisitor {
ValueVectorWriteExpression e2 = new ValueVectorWriteExpression(outId, e, true);
v.addExpr(e2, g.getRoot());
- System.out.println(g.generate());
+ System.out.println(g.generateAndGet());
}
private LogicalExpression getExpr(String expr) throws Exception{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
new file mode 100644
index 0000000..c7467a0
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.compile.bytecode;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.net.URL;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.compile.QueryClassLoader;
+import org.apache.drill.exec.server.options.SystemOptionManager;
+import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.util.ASMifier;
+import org.objectweb.asm.util.CheckClassAdapter;
+import org.objectweb.asm.util.Textifier;
+import org.objectweb.asm.util.TraceClassVisitor;
+
+import com.google.common.io.Files;
+import com.google.common.io.Resources;
+
+public class ReplaceMethodInvoke {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReplaceMethodInvoke.class);
+
+ @SuppressWarnings("unchecked")
+ public static void main(String[] args) throws Exception{
+ String e = "org/apache/drill/ExampleReplaceable.class";
+ String h = "org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.class";
+ String k = "org/apache/drill/Pickle$OutgoingBatch.class";
+ String k2 = "org/apache/drill/Pickle.class";
+ String r = "org/apache/drill/exec/test/generated/FiltererGen0.class";
+ String p = "org/apache/drill/exec/test/generated/PartitionerGen4.class";
+ URL url = Resources.getResource(k2);
+ byte[] clazz = Resources.toByteArray(url);
+ ClassReader cr = new ClassReader(clazz);
+
+ ClassWriter cw = writer();
+ TraceClassVisitor visitor = new TraceClassVisitor(cw, new Textifier(), new PrintWriter(System.out));
+ //getTracer(false)
+ ValueHolderReplacementVisitor v2 = new ValueHolderReplacementVisitor(visitor);
+ cr.accept(v2, ClassReader.EXPAND_FRAMES );//| ClassReader.SKIP_DEBUG);
+
+ byte[] output = cw.toByteArray();
+ Files.write(output, new File("/src/scratch/bytes/S.class"));
+ check(output);
+
+
+ DrillConfig c = DrillConfig.createClient();
+ SystemOptionManager m = new SystemOptionManager(c, new LocalPStoreProvider(c));
+ m.init();
+ QueryClassLoader ql = new QueryClassLoader(DrillConfig.create(), m);
+ ql.injectByteCode("org.apache.drill.Pickle$OutgoingBatch", output);
+ Class<?> clz = ql.loadClass("org.apache.drill.Pickle$OutgoingBatch");
+ clz.getMethod("x").invoke(null);
+
+ }
+
+
+ private static final void check(byte[] b) {
+ ClassReader cr = new ClassReader(b);
+ ClassWriter cw = writer();
+ ClassVisitor cv = new CheckClassAdapter(cw);
+ cr.accept(cv, 0);
+
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ CheckClassAdapter.verify(new ClassReader(cw.toByteArray()), false, pw);
+
+ assert sw.toString().length() == 0 : sw.toString();
+ }
+
+ private static ClassWriter writer() {
+ ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
+ return cw;
+ }
+
+ private static ClassVisitor getTracer(boolean asm) {
+ if (asm) {
+ return new TraceClassVisitor(null, new ASMifier(), new PrintWriter(System.out));
+ } else {
+ return new TraceClassVisitor(null, new Textifier(), new PrintWriter(System.out));
+ }
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
index da82998..392af18 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
@@ -126,7 +126,7 @@ public class ExpressionTest extends ExecTest {
ClassGenerator<Projector> cg = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, new FunctionImplementationRegistry(DrillConfig.create())).getRoot();
cg.addExpr(new ValueVectorWriteExpression(new TypedFieldId(materializedExpr.getMajorType(), -1), materializedExpr));
- return cg.getCodeGenerator().generate();
+ return cg.getCodeGenerator().generateAndGet();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
new file mode 100644
index 0000000..632f5a4
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.fn.impl;
+
+import static org.junit.Assert.*;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.vector.ValueHolderHelper;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestByteComparisonFunctions extends ExecTest{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestByteComparisonFunctions.class);
+
+ static TopLevelAllocator allocator;
+ static VarCharHolder hello;
+ static VarCharHolder goodbye;
+ static VarCharHolder helloLong;
+ static VarCharHolder goodbyeLong;
+
+ @BeforeClass
+ public static void setup(){
+ DrillConfig c= DrillConfig.create();
+ allocator = new TopLevelAllocator(c);
+ hello = ValueHolderHelper.getVarCharHolder(allocator, "hello");
+ goodbye = ValueHolderHelper.getVarCharHolder(allocator, "goodbye");
+ helloLong = ValueHolderHelper.getVarCharHolder(allocator, "hellomyfriend");
+ goodbyeLong = ValueHolderHelper.getVarCharHolder(allocator, "goodbyemyenemy");
+ }
+
+ @AfterClass
+ public static void teardown(){
+ hello.buffer.release();
+ helloLong.buffer.release();
+ goodbye.buffer.release();
+ goodbyeLong.buffer.release();
+ allocator.close();
+ }
+
+ @Test
+ public void testAfter(){
+ VarCharHolder left = hello;
+ VarCharHolder right = goodbye;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1);
+ }
+
+ @Test
+ public void testBefore(){
+ VarCharHolder left = goodbye;
+ VarCharHolder right = hello;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == -1);
+ }
+
+ @Test
+ public void testEqualCompare(){
+ VarCharHolder left = hello;
+ VarCharHolder right = hello;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0);
+ }
+
+ @Test
+ public void testEqual(){
+ VarCharHolder left = hello;
+ VarCharHolder right = hello;
+ assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1);
+ }
+
+ @Test
+ public void testNotEqual(){
+ VarCharHolder left = hello;
+ VarCharHolder right = goodbye;
+ assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0);
+ }
+
+ @Test
+ public void testAfterLong(){
+ VarCharHolder left = helloLong;
+ VarCharHolder right = goodbyeLong;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1);
+ }
+
+ @Test
+ public void testBeforeLong(){
+ VarCharHolder left = goodbyeLong;
+ VarCharHolder right = helloLong;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == -1);
+ }
+
+ @Test
+ public void testEqualCompareLong(){
+ VarCharHolder left = helloLong;
+ VarCharHolder right = helloLong;
+ assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0);
+ }
+
+ @Test
+ public void testEqualLong(){
+ VarCharHolder left = helloLong;
+ VarCharHolder right = helloLong;
+ assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1);
+ }
+
+ @Test
+ public void testNotEqualLong(){
+ VarCharHolder left = helloLong;
+ VarCharHolder right = goodbyeLong;
+ assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
index 104923b..4f06a9d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -64,6 +65,7 @@ public class TestMathFunctions extends ExecTest {
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
index 62a224e..3d41f0b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java
@@ -27,6 +27,7 @@ import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -83,6 +84,7 @@ public class TestNewMathFunctions {
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};
String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
index 4345560..fd71dd9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
@@ -27,6 +27,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -64,6 +65,7 @@ public class TestRepeatedFunction extends ExecTest{
bitContext.getAllocator(); result = new TopLevelAllocator();
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
bitContext.getConfig(); result = c;
+ bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
}};