You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sm...@apache.org on 2016/08/18 23:30:34 UTC
[1/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead
of MaterializedField
Repository: arrow
Updated Branches:
refs/heads/master 246a126b2 -> e7e399db5
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index a3763cd..4eb0d9f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -26,140 +26,139 @@ import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import org.apache.arrow.vector.holders.Decimal38SparseHolder;
-
-public class DecimalUtility extends CoreDecimalUtility{
-
- public final static int MAX_DIGITS = 9;
- public final static int DIGITS_BASE = 1000000000;
- public final static int DIGITS_MAX = 999999999;
- public final static int INTEGER_SIZE = (Integer.SIZE/8);
-
- public final static String[] decimalToString = {"",
- "0",
- "00",
- "000",
- "0000",
- "00000",
- "000000",
- "0000000",
- "00000000",
- "000000000"};
-
- public final static long[] scale_long_constants = {
- 1,
- 10,
- 100,
- 1000,
- 10000,
- 100000,
- 1000000,
- 10000000,
- 100000000,
- 1000000000,
- 10000000000l,
- 100000000000l,
- 1000000000000l,
- 10000000000000l,
- 100000000000000l,
- 1000000000000000l,
- 10000000000000000l,
- 100000000000000000l,
- 1000000000000000000l};
-
- /*
- * Simple function that returns the static precomputed
- * power of ten, instead of using Math.pow
- */
- public static long getPowerOfTen(int power) {
- assert power >= 0 && power < scale_long_constants.length;
- return scale_long_constants[(power)];
- }
-
- /*
- * Math.pow returns a double and while multiplying with large digits
- * in the decimal data type we encounter noise. So instead of multiplying
- * with Math.pow we use the static constants to perform the multiplication
- */
- public static long adjustScaleMultiply(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input * scale_long_constants[index];
- } else {
- return input / scale_long_constants[index];
- }
- }
- public static long adjustScaleDivide(long input, int factor) {
- int index = Math.abs(factor);
- assert index >= 0 && index < scale_long_constants.length;
- if (factor >= 0) {
- return input / scale_long_constants[index];
- } else {
- return input * scale_long_constants[index];
- }
+public class DecimalUtility {
+
+ public final static int MAX_DIGITS = 9;
+ public final static int DIGITS_BASE = 1000000000;
+ public final static int DIGITS_MAX = 999999999;
+ public final static int INTEGER_SIZE = (Integer.SIZE/8);
+
+ public final static String[] decimalToString = {"",
+ "0",
+ "00",
+ "000",
+ "0000",
+ "00000",
+ "000000",
+ "0000000",
+ "00000000",
+ "000000000"};
+
+ public final static long[] scale_long_constants = {
+ 1,
+ 10,
+ 100,
+ 1000,
+ 10000,
+ 100000,
+ 1000000,
+ 10000000,
+ 100000000,
+ 1000000000,
+ 10000000000l,
+ 100000000000l,
+ 1000000000000l,
+ 10000000000000l,
+ 100000000000000l,
+ 1000000000000000l,
+ 10000000000000000l,
+ 100000000000000000l,
+ 1000000000000000000l};
+
+ /*
+ * Simple function that returns the static precomputed
+ * power of ten, instead of using Math.pow
+ */
+ public static long getPowerOfTen(int power) {
+ assert power >= 0 && power < scale_long_constants.length;
+ return scale_long_constants[(power)];
+ }
+
+ /*
+ * Math.pow returns a double and while multiplying with large digits
+ * in the decimal data type we encounter noise. So instead of multiplying
+ * with Math.pow we use the static constants to perform the multiplication
+ */
+ public static long adjustScaleMultiply(long input, int factor) {
+ int index = Math.abs(factor);
+ assert index >= 0 && index < scale_long_constants.length;
+ if (factor >= 0) {
+ return input * scale_long_constants[index];
+ } else {
+ return input / scale_long_constants[index];
}
+ }
- /* Given the number of actual digits this function returns the
- * number of indexes it will occupy in the array of integers
- * which are stored in base 1 billion
- */
- public static int roundUp(int ndigits) {
- return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+ public static long adjustScaleDivide(long input, int factor) {
+ int index = Math.abs(factor);
+ assert index >= 0 && index < scale_long_constants.length;
+ if (factor >= 0) {
+ return input / scale_long_constants[index];
+ } else {
+ return input * scale_long_constants[index];
}
+ }
- /* Returns a string representation of the given integer
- * If the length of the given integer is less than the
- * passed length, this function will prepend zeroes to the string
- */
- public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
- String value = ((Integer) number).toString();
- int length = value.length();
+ /* Given the number of actual digits this function returns the
+ * number of indexes it will occupy in the array of integers
+ * which are stored in base 1 billion
+ */
+ public static int roundUp(int ndigits) {
+ return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+ }
- StringBuilder str = new StringBuilder();
- str.append(decimalToString[desiredLength - length]);
- str.append(value);
+ /* Returns a string representation of the given integer
+ * If the length of the given integer is less than the
+ * passed length, this function will prepend zeroes to the string
+ */
+ public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
+ String value = ((Integer) number).toString();
+ int length = value.length();
- return str;
- }
+ StringBuilder str = new StringBuilder();
+ str.append(decimalToString[desiredLength - length]);
+ str.append(value);
- public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
- String value = ((Long) number).toString();
- int length = value.length();
+ return str;
+ }
- StringBuilder str = new StringBuilder();
+ public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
+ String value = ((Long) number).toString();
+ int length = value.length();
- // Desired length can be > MAX_DIGITS
- int zeroesLength = desiredLength - length;
- while (zeroesLength > MAX_DIGITS) {
- str.append(decimalToString[MAX_DIGITS]);
- zeroesLength -= MAX_DIGITS;
- }
- str.append(decimalToString[zeroesLength]);
- str.append(value);
+ StringBuilder str = new StringBuilder();
- return str;
+ // Desired length can be > MAX_DIGITS
+ int zeroesLength = desiredLength - length;
+ while (zeroesLength > MAX_DIGITS) {
+ str.append(decimalToString[MAX_DIGITS]);
+ zeroesLength -= MAX_DIGITS;
}
+ str.append(decimalToString[zeroesLength]);
+ str.append(value);
+
+ return str;
+ }
public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
- // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
- return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false);
- }
+ // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
+ return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false);
+ }
- public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
+ public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
- // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
- return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true);
- }
+ // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
+ return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true);
+ }
- public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) {
- byte[] value = new byte[length];
- bytebuf.getBytes(start, value, 0, length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
+ public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) {
+ byte[] value = new byte[length];
+ bytebuf.getBytes(start, value, 0, length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
byte[] value = new byte[length];
@@ -168,115 +167,123 @@ public class DecimalUtility extends CoreDecimalUtility{
return new BigDecimal(unscaledValue, scale);
}
- /* Create a BigDecimal object using the data in the ArrowBuf.
- * This function assumes that data is provided in a non-dense format
- * It works on both sparse and intermediate representations.
- */
+ public static void writeBigDecimalToArrowBuf(ArrowBuf bytebuf, int startIndex, BigDecimal value) {
+ byte[] bytes = value.unscaledValue().toByteArray();
+ if (bytes.length > 16) {
+ throw new UnsupportedOperationException("Decimal size greater than 16 bytes");
+ }
+ bytebuf.setBytes(startIndex + 16 - bytes.length, bytes, 0, bytes.length);
+ }
+
+ /* Create a BigDecimal object using the data in the ArrowBuf.
+ * This function assumes that data is provided in a non-dense format
+ * It works on both sparse and intermediate representations.
+ */
public static BigDecimal getBigDecimalFromArrowBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale,
- boolean truncateScale) {
+ boolean truncateScale) {
- // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
- int actualDigits;
+ // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
+ int actualDigits;
- // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out
- BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
+ // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out
+ BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
- BigInteger base = BigInteger.valueOf(DIGITS_BASE);
+ BigInteger base = BigInteger.valueOf(DIGITS_BASE);
- for (int i = 1; i < nDecimalDigits; i++) {
+ for (int i = 1; i < nDecimalDigits; i++) {
- BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
- decimalDigits = decimalDigits.multiply(base);
- decimalDigits = decimalDigits.add(temp);
- }
+ BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
+ decimalDigits = decimalDigits.multiply(base);
+ decimalDigits = decimalDigits.add(temp);
+ }
- // Truncate any additional padding we might have added
- if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
- BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
- decimalDigits = decimalDigits.divide(truncate);
- }
+ // Truncate any additional padding we might have added
+ if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
+ BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
+ decimalDigits = decimalDigits.divide(truncate);
+ }
- // set the sign
- if ((data.getInt(startIndex) & 0x80000000) != 0) {
- decimalDigits = decimalDigits.negate();
- }
+ // set the sign
+ if ((data.getInt(startIndex) & 0x80000000) != 0) {
+ decimalDigits = decimalDigits.negate();
+ }
- BigDecimal decimal = new BigDecimal(decimalDigits, scale);
+ BigDecimal decimal = new BigDecimal(decimalDigits, scale);
- return decimal;
- }
+ return decimal;
+ }
- /* This function returns a BigDecimal object from the dense decimal representation.
- * First step is to convert the dense representation into an intermediate representation
- * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object
- */
- public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
+ /* This function returns a BigDecimal object from the dense decimal representation.
+ * First step is to convert the dense representation into an intermediate representation
+ * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object
+ */
+ public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
/* This method converts the dense representation to
* an intermediate representation. The intermediate
* representation has one more integer than the dense
* representation.
*/
- byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
-
- // Start storing from the least significant byte of the first integer
- int intermediateIndex = 3;
-
- int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
- int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
-
- int maskIndex;
- int shiftOrder;
- byte shiftBits;
-
- // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
- if (maxPrecision == 38) {
- maskIndex = 0;
- shiftOrder = 6;
- shiftBits = 0x00;
- intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
- } else if (maxPrecision == 28) {
- maskIndex = 1;
- shiftOrder = 4;
- shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
- intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
- } else {
- throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
- }
+ byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
+
+ // Start storing from the least significant byte of the first integer
+ int intermediateIndex = 3;
+
+ int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
+ int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
+
+ int maskIndex;
+ int shiftOrder;
+ byte shiftBits;
+
+ // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
+ if (maxPrecision == 38) {
+ maskIndex = 0;
+ shiftOrder = 6;
+ shiftBits = 0x00;
+ intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
+ } else if (maxPrecision == 28) {
+ maskIndex = 1;
+ shiftOrder = 4;
+ shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
+ intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
+ } else {
+ throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
+ }
- int inputIndex = 1;
- boolean sign = false;
+ int inputIndex = 1;
+ boolean sign = false;
- if ((data.getByte(startIndex) & 0x80) != 0) {
- sign = true;
- }
+ if ((data.getByte(startIndex) & 0x80) != 0) {
+ sign = true;
+ }
- while (inputIndex < width) {
+ while (inputIndex < width) {
- intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
+ intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
- shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
+ shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
- inputIndex++;
- intermediateIndex++;
+ inputIndex++;
+ intermediateIndex++;
- if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
- shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
- maskIndex++;
- shiftOrder -= 2;
- }
+ if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
+ shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
+ maskIndex++;
+ shiftOrder -= 2;
+ }
- }
+ }
/* copy the last byte */
- intermediateBytes[intermediateIndex] = shiftBits;
+ intermediateBytes[intermediateIndex] = shiftBits;
- if (sign == true) {
- intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
- }
+ if (sign == true) {
+ intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
+ }
final ByteBuf intermediate = UnpooledByteBufAllocator.DEFAULT.buffer(intermediateBytes.length);
try {
- intermediate.setBytes(0, intermediateBytes);
+ intermediate.setBytes(0, intermediateBytes);
BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
return ret;
@@ -284,299 +291,296 @@ public class DecimalUtility extends CoreDecimalUtility{
intermediate.release();
}
- }
+ }
- /*
- * Function converts the BigDecimal and stores it in out internal sparse representation
- */
- public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
- int nDecimalDigits) {
+ public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
+ int nDecimalDigits) {
- // Initialize the buffer
- for (int i = 0; i < nDecimalDigits; i++) {
- data.setInt(startIndex + (i * INTEGER_SIZE), 0);
- }
+ // Initialize the buffer
+ for (int i = 0; i < nDecimalDigits; i++) {
+ data.setInt(startIndex + (i * INTEGER_SIZE), 0);
+ }
- boolean sign = false;
+ boolean sign = false;
- if (input.signum() == -1) {
- // negative input
- sign = true;
- input = input.abs();
- }
+ if (input.signum() == -1) {
+ // negative input
+ sign = true;
+ input = input.abs();
+ }
- // Truncate the input as per the scale provided
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+ // Truncate the input as per the scale provided
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
- // Separate out the integer part
- BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
+ // Separate out the integer part
+ BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
- int destIndex = nDecimalDigits - roundUp(scale) - 1;
+ int destIndex = nDecimalDigits - roundUp(scale) - 1;
- // we use base 1 billion integer digits for out integernal representation
- BigDecimal base = new BigDecimal(DIGITS_BASE);
+ // we use base 1 billion integer digits for out integernal representation
+ BigDecimal base = new BigDecimal(DIGITS_BASE);
- while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
- // store the modulo as the integer value
- data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
- destIndex--;
- // Divide by base 1 billion
- integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
- }
+ while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
+ // store the modulo as the integer value
+ data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
+ destIndex--;
+ // Divide by base 1 billion
+ integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
+ }
/* Sparse representation contains padding of additional zeroes
* so each digit contains MAX_DIGITS for ease of arithmetic
*/
- int actualDigits;
- if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
- // Pad additional zeroes
- scale = scale + (MAX_DIGITS - actualDigits);
- input = input.setScale(scale, BigDecimal.ROUND_DOWN);
- }
-
- //separate out the fractional part
- BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
+ int actualDigits;
+ if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
+ // Pad additional zeroes
+ scale = scale + (MAX_DIGITS - actualDigits);
+ input = input.setScale(scale, BigDecimal.ROUND_DOWN);
+ }
- destIndex = nDecimalDigits - 1;
+ //separate out the fractional part
+ BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
- while (scale > 0) {
- // Get next set of MAX_DIGITS (9) store it in the ArrowBuf
- fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
- BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
+ destIndex = nDecimalDigits - 1;
- data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
- destIndex--;
+ while (scale > 0) {
+ // Get next set of MAX_DIGITS (9) store it in the ArrowBuf
+ fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
+ BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
- fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
- scale -= MAX_DIGITS;
- }
+ data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
+ destIndex--;
- // Set the negative sign
- if (sign == true) {
- data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
- }
+ fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
+ scale -= MAX_DIGITS;
+ }
+ // Set the negative sign
+ if (sign == true) {
+ data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
}
+ }
- public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
- return (input.unscaledValue().longValue());
- }
+ public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+ // Truncate or pad to set the input to the correct scale
+ input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
- public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
- return BigDecimal.valueOf(input, scale);
- }
+ return (input.unscaledValue().longValue());
+ }
- public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
- return BigDecimal.valueOf(input, scale);
- }
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
+ return BigDecimal.valueOf(input, scale);
+ }
+
+ public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
+ return BigDecimal.valueOf(input, scale);
+ }
- public static int compareDenseBytes(ArrowBuf left, int leftStart, boolean leftSign, ArrowBuf right, int rightStart, boolean rightSign, int width) {
+ public static int compareDenseBytes(ArrowBuf left, int leftStart, boolean leftSign, ArrowBuf right, int rightStart, boolean rightSign, int width) {
- int invert = 1;
+ int invert = 1;
/* If signs are different then simply look at the
* sign of the two inputs and determine which is greater
*/
- if (leftSign != rightSign) {
+ if (leftSign != rightSign) {
- return((leftSign == true) ? -1 : 1);
- } else if(leftSign == true) {
+ return((leftSign == true) ? -1 : 1);
+ } else if(leftSign == true) {
/* Both inputs are negative, at the end we will
* have to invert the comparison
*/
- invert = -1;
- }
-
- int cmp = 0;
-
- for (int i = 0; i < width; i++) {
- byte leftByte = left.getByte(leftStart + i);
- byte rightByte = right.getByte(rightStart + i);
- // Unsigned byte comparison
- if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
- cmp = 1;
- break;
- } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
- cmp = -1;
- break;
- }
- }
- cmp *= invert; // invert the comparison if both were negative values
-
- return cmp;
+ invert = -1;
}
- public static int getIntegerFromSparseBuffer(ArrowBuf buffer, int start, int index) {
- int value = buffer.getInt(start + (index * 4));
+ int cmp = 0;
- if (index == 0) {
- /* the first byte contains sign bit, return value without it */
- value = (value & 0x7FFFFFFF);
+ for (int i = 0; i < width; i++) {
+ byte leftByte = left.getByte(leftStart + i);
+ byte rightByte = right.getByte(rightStart + i);
+ // Unsigned byte comparison
+ if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
+ cmp = 1;
+ break;
+ } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
+ cmp = -1;
+ break;
}
- return value;
}
+ cmp *= invert; // invert the comparison if both were negative values
- public static void setInteger(ArrowBuf buffer, int start, int index, int value) {
- buffer.setInt(start + (index * 4), value);
+ return cmp;
+ }
+
+ public static int getIntegerFromSparseBuffer(ArrowBuf buffer, int start, int index) {
+ int value = buffer.getInt(start + (index * 4));
+
+ if (index == 0) {
+ /* the first byte contains sign bit, return value without it */
+ value = (value & 0x7FFFFFFF);
}
+ return value;
+ }
- public static int compareSparseBytes(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
+ public static void setInteger(ArrowBuf buffer, int start, int index, int value) {
+ buffer.setInt(start + (index * 4), value);
+ }
- int invert = 1;
+ public static int compareSparseBytes(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
- if (absCompare == false) {
- if (leftSign != rightSign) {
- return (leftSign == true) ? -1 : 1;
- }
+ int invert = 1;
- // Both values are negative invert the outcome of the comparison
- if (leftSign == true) {
- invert = -1;
- }
+ if (absCompare == false) {
+ if (leftSign != rightSign) {
+ return (leftSign == true) ? -1 : 1;
}
- int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
- return cmp * invert;
+ // Both values are negative invert the outcome of the comparison
+ if (leftSign == true) {
+ invert = -1;
+ }
}
- public static int compareSparseBytesInner(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
+
+ int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
+ return cmp * invert;
+ }
+ public static int compareSparseBytesInner(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
/* compute the number of integer digits in each decimal */
- int leftInt = leftPrecision - leftScale;
- int rightInt = rightPrecision - rightScale;
+ int leftInt = leftPrecision - leftScale;
+ int rightInt = rightPrecision - rightScale;
/* compute the number of indexes required for storing integer digits */
- int leftIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftInt);
- int rightIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightInt);
+ int leftIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftInt);
+ int rightIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightInt);
/* compute number of indexes required for storing scale */
- int leftScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftScale);
- int rightScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightScale);
+ int leftScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftScale);
+ int rightScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightScale);
/* compute index of the most significant integer digits */
- int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+ int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+ int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
- int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
- int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
+ int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
+ int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
/* Discard the zeroes in the integer part */
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- break;
- }
+ while (leftIndex1 < leftStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+ break;
+ }
/* Digit in this location is zero, decrement the actual number
* of integer digits
*/
- leftIntRoundedUp--;
- leftIndex1++;
- }
+ leftIntRoundedUp--;
+ leftIndex1++;
+ }
/* If we reached the stop index then the number of integers is zero */
- if (leftIndex1 == leftStopIndex) {
- leftIntRoundedUp = 0;
- }
+ if (leftIndex1 == leftStopIndex) {
+ leftIntRoundedUp = 0;
+ }
- while (rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- break;
- }
+ while (rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+ break;
+ }
/* Digit in this location is zero, decrement the actual number
* of integer digits
*/
- rightIntRoundedUp--;
- rightIndex1++;
- }
+ rightIntRoundedUp--;
+ rightIndex1++;
+ }
- if (rightIndex1 == rightStopIndex) {
- rightIntRoundedUp = 0;
- }
+ if (rightIndex1 == rightStopIndex) {
+ rightIntRoundedUp = 0;
+ }
/* We have the accurate number of non-zero integer digits,
* if the number of integer digits are different then we can determine
* which decimal is larger and needn't go down to comparing individual values
*/
- if (leftIntRoundedUp > rightIntRoundedUp) {
- return 1;
- }
- else if (rightIntRoundedUp > leftIntRoundedUp) {
- return -1;
- }
+ if (leftIntRoundedUp > rightIntRoundedUp) {
+ return 1;
+ }
+ else if (rightIntRoundedUp > leftIntRoundedUp) {
+ return -1;
+ }
/* The number of integer digits are the same, set the each index
* to the first non-zero integer and compare each digit
*/
- leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
- rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+ leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+ rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
+ while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+ return 1;
+ }
+ else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+ return -1;
}
+ leftIndex1++;
+ rightIndex1++;
+ }
+
/* The integer part of both the decimal's are equal, now compare
* each individual fractional part. Set the index to be at the
* beginning of the fractional part
*/
- leftIndex1 = leftStopIndex;
- rightIndex1 = rightStopIndex;
+ leftIndex1 = leftStopIndex;
+ rightIndex1 = rightStopIndex;
/* Stop indexes will be the end of the array */
- leftStopIndex = nDecimalDigits;
- rightStopIndex = nDecimalDigits;
+ leftStopIndex = nDecimalDigits;
+ rightStopIndex = nDecimalDigits;
/* compare the two fractional parts of the decimal */
- while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
- return 1;
- }
- else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
- return -1;
- }
-
- leftIndex1++;
- rightIndex1++;
+ while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+ return 1;
+ }
+ else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+ return -1;
}
+ leftIndex1++;
+ rightIndex1++;
+ }
+
/* Till now the fractional part of the decimals are equal, check
* if one of the decimal has fractional part that is remaining
* and is non-zero
*/
- while (leftIndex1 < leftStopIndex) {
- if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
- return 1;
- }
- leftIndex1++;
+ while (leftIndex1 < leftStopIndex) {
+ if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+ return 1;
}
+ leftIndex1++;
+ }
- while(rightIndex1 < rightStopIndex) {
- if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
- return -1;
- }
- rightIndex1++;
+ while(rightIndex1 < rightStopIndex) {
+ if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+ return -1;
}
+ rightIndex1++;
+ }
/* Both decimal values are equal */
- return 0;
- }
+ return 0;
+ }
- public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
- byte[] value = Arrays.copyOfRange(bytes, start, start + length);
- BigInteger unscaledValue = new BigInteger(value);
- return new BigDecimal(unscaledValue, scale);
- }
+ public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
+ byte[] value = Arrays.copyOfRange(bytes, start, start + length);
+ BigInteger unscaledValue = new BigInteger(value);
+ return new BigDecimal(unscaledValue, scale);
+ }
public static void roundDecimal(ArrowBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
int newScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(desiredScale);
@@ -704,34 +708,6 @@ public class DecimalUtility extends CoreDecimalUtility{
int index = nDecimalDigits - roundUp(scale);
return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1));
}
-
- public static int compareSparseSamePrecScale(ArrowBuf left, int lStart, byte[] right, int length) {
- // check the sign first
- boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
- boolean rSign = ByteFunctionHelpers.getSign(right);
- int cmp = 0;
-
- if (lSign != rSign) {
- return (lSign == false) ? 1 : -1;
- }
-
- // invert the comparison if we are comparing negative numbers
- int invert = (lSign == true) ? -1 : 1;
-
- // compare byte by byte
- int n = 0;
- int lPos = lStart;
- int rPos = 0;
- while (n < length/4) {
- int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
- int rightInt = ByteFunctionHelpers.getInteger(right, n);
- if (leftInt != rightInt) {
- cmp = (leftInt - rightInt ) > 0 ? 1 : -1;
- break;
- }
- n++;
- }
- return cmp * invert;
- }
}
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
index dea433e..d7f9d38 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -18,7 +18,9 @@
package org.apache.arrow.vector.util;
import java.util.AbstractMap;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -241,6 +243,16 @@ public class MapWithOrdinal<K, V> implements Map<K, V> {
return delegate.keySet();
}
+ public List<K> keyList() {
+ int size = size();
+ Set<K> keys = keySet();
+ List<K> children = new ArrayList<>(size);
+ for (K key : keys) {
+ children.add(getOrdinal(key), key);
+ }
+ return children;
+ }
+
@Override
public Set<Entry<K, V>> entrySet() {
return delegate.entrySet();
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
new file mode 100644
index 0000000..7ab7db3
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.util.DecimalUtility;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestDecimalVector {
+
+ private static long[] intValues;
+
+ static {
+ intValues = new long[30];
+ for (int i = 0; i < intValues.length; i++) {
+ intValues[i] = 1 << i + 1;
+ }
+ }
+ private int scale = 3;
+
+ @Test
+ public void test() {
+ BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+ NullableDecimalVector decimalVector = new NullableDecimalVector("decimal", allocator, 10, scale);
+ decimalVector.allocateNew();
+ BigDecimal[] values = new BigDecimal[intValues.length];
+ for (int i = 0; i < intValues.length; i++) {
+ BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale);
+ values[i] = decimal;
+ decimalVector.getMutator().setIndexDefined(i);
+ DecimalUtility.writeBigDecimalToArrowBuf(decimalVector.getBuffer(), i * 16, decimal);
+ }
+
+ decimalVector.getMutator().setValueCount(intValues.length);
+
+ for (int i = 0; i < intValues.length; i++) {
+ BigDecimal value = decimalVector.getAccessor().getObject(i);
+ assertEquals(values[i], value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
index 4dee86c..9baebc5 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
@@ -20,8 +20,6 @@ package org.apache.arrow.vector;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.junit.After;
import org.junit.Before;
@@ -53,8 +51,7 @@ public class TestOversizedAllocationForValueVector {
@Test(expected = OversizedAllocationException.class)
public void testFixedVectorReallocation() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
- final UInt4Vector vector = new UInt4Vector(field, allocator);
+ final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator);
// edge case 1: buffer size = max value capacity
final int expectedValueCapacity = BaseValueVector.MAX_ALLOCATION_SIZE / 4;
try {
@@ -78,8 +75,7 @@ public class TestOversizedAllocationForValueVector {
@Test(expected = OversizedAllocationException.class)
public void testBitVectorReallocation() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
- final BitVector vector = new BitVector(field, allocator);
+ final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator);
// edge case 1: buffer size ~ max value capacity
final int expectedValueCapacity = 1 << 29;
try {
@@ -109,8 +105,7 @@ public class TestOversizedAllocationForValueVector {
@Test(expected = OversizedAllocationException.class)
public void testVariableVectorReallocation() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
- final VarCharVector vector = new VarCharVector(field, allocator);
+ final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator);
// edge case 1: value count = MAX_VALUE_ALLOCATION
final int expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE;
final int expectedOffsetSize = 10;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index e4d28c3..1bb50b7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -22,8 +22,6 @@ import static org.junit.Assert.assertEquals;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.holders.NullableUInt4Holder;
-import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
import org.apache.arrow.vector.types.Types;
import org.junit.After;
import org.junit.Before;
@@ -46,13 +44,12 @@ public class TestUnionVector {
@Test
public void testUnionVector() throws Exception {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder();
uInt4Holder.value = 100;
uInt4Holder.isSet = 1;
- try (UnionVector unionVector = new UnionVector(field, allocator, null)) {
+ try (UnionVector unionVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
unionVector.allocateNew();
// write some data
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index ce091ab..21cdc4f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -19,15 +19,7 @@ package org.apache.arrow.vector;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.RepeatedListVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.holders.*;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.junit.After;
import org.junit.Before;
@@ -50,9 +42,9 @@ public class TestValueVector {
}
private final static Charset utf8Charset = Charset.forName("UTF-8");
- private final static byte[] STR1 = new String("AAAAA1").getBytes(utf8Charset);
- private final static byte[] STR2 = new String("BBBBBBBBB2").getBytes(utf8Charset);
- private final static byte[] STR3 = new String("CCCC3").getBytes(utf8Charset);
+ private final static byte[] STR1 = "AAAAA1".getBytes(utf8Charset);
+ private final static byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset);
+ private final static byte[] STR3 = "CCCC3".getBytes(utf8Charset);
@After
public void terminate() throws Exception {
@@ -61,10 +53,9 @@ public class TestValueVector {
@Test
public void testFixedType() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
// Create a new value vector for 1024 integers.
- try (final UInt4Vector vector = new UInt4Vector(field, allocator)) {
+ try (final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator)) {
final UInt4Vector.Mutator m = vector.getMutator();
vector.allocateNew(1024);
@@ -86,10 +77,9 @@ public class TestValueVector {
@Test
public void testNullableVarLen2() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
// Create a new value vector for 1024 integers.
- try (final NullableVarCharVector vector = new NullableVarCharVector(field, allocator)) {
+ try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
final NullableVarCharVector.Mutator m = vector.getMutator();
vector.allocateNew(1024 * 10, 1024);
@@ -116,44 +106,10 @@ public class TestValueVector {
}
@Test
- public void testRepeatedIntVector() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
-
- // Create a new value vector.
- try (final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator)) {
-
- // Populate the vector.
- final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
- final int nRecords = 7;
- final int nElements = values.length;
- vector1.allocateNew(nRecords, nRecords * nElements);
- final RepeatedIntVector.Mutator mutator = vector1.getMutator();
- for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
- mutator.startNewValue(recordIndex);
- for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
- mutator.add(recordIndex, recordIndex * values[elementIndex]);
- }
- }
- mutator.setValueCount(nRecords);
-
- // Verify the contents.
- final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
- assertEquals(nRecords, accessor1.getValueCount());
- for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
- for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
- final int value = accessor1.get(recordIndex, elementIndex);
- assertEquals(recordIndex * values[elementIndex], value);
- }
- }
- }
- }
-
- @Test
public void testNullableFixedType() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableUInt4Holder.TYPE);
// Create a new value vector for 1024 integers.
- try (final NullableUInt4Vector vector = new NullableUInt4Vector(field, allocator)) {
+ try (final NullableUInt4Vector vector = new NullableUInt4Vector(EMPTY_SCHEMA_PATH, allocator)) {
final NullableUInt4Vector.Mutator m = vector.getMutator();
vector.allocateNew(1024);
@@ -222,10 +178,8 @@ public class TestValueVector {
@Test
public void testNullableFloat() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
-
// Create a new value vector for 1024 integers
- try (final NullableFloat4Vector vector = (NullableFloat4Vector) BasicTypeHelper.getNewVector(field, allocator)) {
+ try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
final NullableFloat4Vector.Mutator m = vector.getMutator();
vector.allocateNew(1024);
@@ -271,10 +225,8 @@ public class TestValueVector {
@Test
public void testBitVector() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
-
// Create a new value vector for 1024 integers
- try (final BitVector vector = new BitVector(field, allocator)) {
+ try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
final BitVector.Mutator m = vector.getMutator();
vector.allocateNew(1024);
@@ -311,10 +263,8 @@ public class TestValueVector {
@Test
public void testReAllocNullableFixedWidthVector() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
-
// Create a new value vector for 1024 integers
- try (final NullableFloat4Vector vector = (NullableFloat4Vector) BasicTypeHelper.getNewVector(field, allocator)) {
+ try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
final NullableFloat4Vector.Mutator m = vector.getMutator();
vector.allocateNew(1024);
@@ -346,10 +296,8 @@ public class TestValueVector {
@Test
public void testReAllocNullableVariableWidthVector() {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
-
// Create a new value vector for 1024 integers
- try (final NullableVarCharVector vector = (NullableVarCharVector) BasicTypeHelper.getNewVector(field, allocator)) {
+ try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
final NullableVarCharVector.Mutator m = vector.getMutator();
vector.allocateNew();
@@ -376,69 +324,4 @@ public class TestValueVector {
}
}
- @Test
- public void testVVInitialCapacity() throws Exception {
- final MaterializedField[] fields = new MaterializedField[9];
- final ValueVector[] valueVectors = new ValueVector[9];
-
- fields[0] = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
- fields[1] = MaterializedField.create(EMPTY_SCHEMA_PATH, IntHolder.TYPE);
- fields[2] = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
- fields[3] = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVar16CharHolder.TYPE);
- fields[4] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedFloat4Holder.TYPE);
- fields[5] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedVarBinaryHolder.TYPE);
-
- fields[6] = MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE);
- fields[6].addChild(fields[0] /*bit*/);
- fields[6].addChild(fields[2] /*varchar*/);
-
- fields[7] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE);
- fields[7].addChild(fields[1] /*int*/);
- fields[7].addChild(fields[3] /*optional var16char*/);
-
- fields[8] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE);
- fields[8].addChild(fields[1] /*int*/);
-
- final int initialCapacity = 1024;
-
- try {
- for (int i = 0; i < valueVectors.length; i++) {
- valueVectors[i] = BasicTypeHelper.getNewVector(fields[i], allocator);
- valueVectors[i].setInitialCapacity(initialCapacity);
- valueVectors[i].allocateNew();
- }
-
- for (int i = 0; i < valueVectors.length; i++) {
- final ValueVector vv = valueVectors[i];
- final int vvCapacity = vv.getValueCapacity();
-
- // this can't be equality because Nullables will be allocated using power of two sized buffers (thus need 1025
- // spots in one vector > power of two is 2048, available capacity will be 2048 => 2047)
- assertTrue(String.format("Incorrect value capacity for %s [%d]", vv.getField(), vvCapacity),
- initialCapacity <= vvCapacity);
- }
- } finally {
- for (ValueVector v : valueVectors) {
- v.close();
- }
- }
- }
-
- @Test
- public void testListVectorShouldNotThrowOversizedAllocationException() throws Exception {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH,
- Types.optional(MinorType.LIST));
- ListVector vector = new ListVector(field, allocator, null);
- ListVector vectorFrom = new ListVector(field, allocator, null);
- vectorFrom.allocateNew();
-
- for (int i = 0; i < 10000; i++) {
- vector.allocateNew();
- vector.copyFromSafe(0, 0, vectorFrom);
- vector.clear();
- }
-
- vectorFrom.clear();
- vector.clear();
- }
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 4c24444..24f00f1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -27,7 +27,7 @@ import org.apache.arrow.vector.complex.AbstractMapVector;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MinorType;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -49,10 +49,9 @@ public class TestPromotableWriter {
@Test
public void testPromoteToUnion() throws Exception {
- final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
- try (final AbstractMapVector container = new MapVector(field, allocator, null);
- final MapVector v = container.addOrGet("test", MapVector.TYPE, MapVector.class);
+ try (final AbstractMapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
+ final MapVector v = container.addOrGet("test", MinorType.MAP, MapVector.class);
final PromotableWriter writer = new PromotableWriter(v, container)) {
container.allocateNew();
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
new file mode 100644
index 0000000..bc17a2b
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.writer;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.impl.UnionReader;
+import org.apache.arrow.vector.complex.impl.UnionWriter;
+import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestComplexWriter {
+
+ static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+
+ private static final int COUNT = 100;
+
+ @Test
+ public void simpleNestedTypes() {
+ MapVector parent = new MapVector("parent", allocator, null);
+ ComplexWriter writer = new ComplexWriterImpl("root", parent);
+ MapWriter rootWriter = writer.rootAsMap();
+ IntWriter intWriter = rootWriter.integer("int");
+ BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+ for (int i = 0; i < COUNT; i++) {
+ intWriter.setPosition(i);
+ intWriter.writeInt(i);
+ bigIntWriter.setPosition(i);
+ bigIntWriter.writeBigInt(i);
+ }
+ writer.setValueCount(COUNT);
+ MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+ for (int i = 0; i < COUNT; i++) {
+ rootReader.setPosition(i);
+ Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
+ Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+ }
+
+ parent.close();
+ }
+
+ @Test
+ public void listScalarType() {
+ ListVector listVector = new ListVector("list", allocator, null);
+ listVector.allocateNew();
+ UnionListWriter listWriter = new UnionListWriter(listVector);
+ for (int i = 0; i < COUNT; i++) {
+ listWriter.setPosition(i);
+ listWriter.startList();
+ for (int j = 0; j < i % 7; j++) {
+ listWriter.writeInt(j);
+ }
+ listWriter.endList();
+ }
+ listWriter.setValueCount(COUNT);
+ UnionListReader listReader = new UnionListReader(listVector);
+ for (int i = 0; i < COUNT; i++) {
+ listReader.setPosition(i);
+ for (int j = 0; j < i % 7; j++) {
+ listReader.next();
+ Assert.assertEquals(j, listReader.reader().readInteger().intValue());
+ }
+ }
+ }
+
+
+ @Test
+ public void listMapType() {
+ ListVector listVector = new ListVector("list", allocator, null);
+ listVector.allocateNew();
+ UnionListWriter listWriter = new UnionListWriter(listVector);
+ MapWriter mapWriter = listWriter.map();
+ for (int i = 0; i < COUNT; i++) {
+ listWriter.setPosition(i);
+ listWriter.startList();
+ for (int j = 0; j < i % 7; j++) {
+ mapWriter.start();
+ mapWriter.integer("int").writeInt(j);
+ mapWriter.bigInt("bigInt").writeBigInt(j);
+ mapWriter.end();
+ }
+ listWriter.endList();
+ }
+ listWriter.setValueCount(COUNT);
+ UnionListReader listReader = new UnionListReader(listVector);
+ for (int i = 0; i < COUNT; i++) {
+ listReader.setPosition(i);
+ for (int j = 0; j < i % 7; j++) {
+ listReader.next();
+ Assert.assertEquals("record: " + i, j, listReader.reader().reader("int").readInteger().intValue());
+ Assert.assertEquals(j, listReader.reader().reader("bigInt").readLong().longValue());
+ }
+ }
+ }
+
+ @Test
+ public void listListType() {
+ ListVector listVector = new ListVector("list", allocator, null);
+ listVector.allocateNew();
+ UnionListWriter listWriter = new UnionListWriter(listVector);
+ for (int i = 0; i < COUNT; i++) {
+ listWriter.setPosition(i);
+ listWriter.startList();
+ for (int j = 0; j < i % 7; j++) {
+ ListWriter innerListWriter = listWriter.list();
+ innerListWriter.startList();
+ for (int k = 0; k < i % 13; k++) {
+ innerListWriter.integer().writeInt(k);
+ }
+ innerListWriter.endList();
+ }
+ listWriter.endList();
+ }
+ listWriter.setValueCount(COUNT);
+ UnionListReader listReader = new UnionListReader(listVector);
+ for (int i = 0; i < COUNT; i++) {
+ listReader.setPosition(i);
+ for (int j = 0; j < i % 7; j++) {
+ listReader.next();
+ FieldReader innerListReader = listReader.reader();
+ for (int k = 0; k < i % 13; k++) {
+ innerListReader.next();
+ Assert.assertEquals("record: " + i, k, innerListReader.reader().readInteger().intValue());
+ }
+ }
+ }
+ listVector.clear();
+ }
+
+ @Test
+ public void unionListListType() {
+ ListVector listVector = new ListVector("list", allocator, null);
+ listVector.allocateNew();
+ UnionListWriter listWriter = new UnionListWriter(listVector);
+ for (int i = 0; i < COUNT; i++) {
+ listWriter.setPosition(i);
+ listWriter.startList();
+ for (int j = 0; j < i % 7; j++) {
+ ListWriter innerListWriter = listWriter.list();
+ innerListWriter.startList();
+ for (int k = 0; k < i % 13; k++) {
+ if (k % 2 == 0) {
+ innerListWriter.integer().writeInt(k);
+ } else {
+ innerListWriter.bigInt().writeBigInt(k);
+ }
+ }
+ innerListWriter.endList();
+ }
+ listWriter.endList();
+ }
+ listWriter.setValueCount(COUNT);
+ UnionListReader listReader = new UnionListReader(listVector);
+ for (int i = 0; i < COUNT; i++) {
+ listReader.setPosition(i);
+ for (int j = 0; j < i % 7; j++) {
+ listReader.next();
+ FieldReader innerListReader = listReader.reader();
+ for (int k = 0; k < i % 13; k++) {
+ innerListReader.next();
+ if (k % 2 == 0) {
+ Assert.assertEquals("record: " + i, k, innerListReader.reader().readInteger().intValue());
+ } else {
+ Assert.assertEquals("record: " + i, k, innerListReader.reader().readLong().longValue());
+ }
+ }
+ }
+ }
+ listVector.clear();
+ }
+
+ @Test
+ public void simpleUnion() {
+ UnionVector vector = new UnionVector("union", allocator, null);
+ UnionWriter unionWriter = new UnionWriter(vector);
+ unionWriter.allocate();
+ for (int i = 0; i < COUNT; i++) {
+ unionWriter.setPosition(i);
+ if (i % 2 == 0) {
+ unionWriter.writeInt(i);
+ } else {
+ unionWriter.writeFloat4((float) i);
+ }
+ }
+ vector.getMutator().setValueCount(COUNT);
+ UnionReader unionReader = new UnionReader(vector);
+ for (int i = 0; i < COUNT; i++) {
+ unionReader.setPosition(i);
+ if (i % 2 == 0) {
+ Assert.assertEquals(i, i, unionReader.readInteger());
+ } else {
+ Assert.assertEquals((float) i, unionReader.readFloat(), 1e-12);
+ }
+ }
+ vector.close();
+ }
+
+ @Test
+ public void promotableWriter() {
+ MapVector parent = new MapVector("parent", allocator, null);
+ ComplexWriter writer = new ComplexWriterImpl("root", parent);
+ MapWriter rootWriter = writer.rootAsMap();
+ for (int i = 0; i < 100; i++) {
+ BigIntWriter bigIntWriter = rootWriter.bigInt("a");
+ bigIntWriter.setPosition(i);
+ bigIntWriter.writeBigInt(i);
+ }
+ Field field = parent.getField().getChildren().get(0).getChildren().get(0);
+ Assert.assertEquals("a", field.getName());
+ Assert.assertEquals(Int.TYPE_TYPE, field.getType().getTypeType());
+ Int intType = (Int) field.getType();
+
+ Assert.assertEquals(64, intType.getBitWidth());
+ Assert.assertTrue(intType.getIsSigned());
+ for (int i = 100; i < 200; i++) {
+ VarCharWriter varCharWriter = rootWriter.varChar("a");
+ varCharWriter.setPosition(i);
+ byte[] bytes = Integer.toString(i).getBytes();
+ ArrowBuf tempBuf = allocator.buffer(bytes.length);
+ tempBuf.setBytes(0, bytes);
+ varCharWriter.writeVarChar(0, bytes.length, tempBuf);
+ }
+ field = parent.getField().getChildren().get(0).getChildren().get(0);
+ Assert.assertEquals("a", field.getName());
+ Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeType());
+ Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeType());
+ Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeType());
+ MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+ for (int i = 0; i < 100; i++) {
+ rootReader.setPosition(i);
+ Assert.assertEquals(i, rootReader.reader("a").readLong().intValue());
+ }
+ for (int i = 100; i < 200; i++) {
+ rootReader.setPosition(i);
+ Assert.assertEquals(Integer.toString(i), rootReader.reader("a").readText().toString());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
new file mode 100644
index 0000000..06a1149
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.pojo;
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test conversion between Flatbuf and Pojo field representations
+ */
+public class TestConvert {
+
+ @Test
+ public void simple() {
+ Field initialField = new Field("a", true, new Int(32, true), null);
+ run(initialField);
+ }
+
+ @Test
+ public void complex() {
+ ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+ childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
+ childrenBuilder.add(new Field("child2", true, new FloatingPoint(0), ImmutableList.<Field>of()));
+
+ Field initialField = new Field("a", true, Tuple.INSTANCE, childrenBuilder.build());
+ run(initialField);
+ }
+
+ @Test
+ public void schema() {
+ ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+ childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
+ childrenBuilder.add(new Field("child2", true, new FloatingPoint(0), ImmutableList.<Field>of()));
+ Schema initialSchema = new Schema(childrenBuilder.build());
+ run(initialSchema);
+
+ }
+
+ private void run(Field initialField) {
+ FlatBufferBuilder builder = new FlatBufferBuilder();
+ builder.finish(initialField.getField(builder));
+ org.apache.arrow.flatbuf.Field flatBufField = org.apache.arrow.flatbuf.Field.getRootAsField(builder.dataBuffer());
+ Field finalField = Field.convertField(flatBufField);
+ assertEquals(initialField, finalField);
+ }
+
+ private void run(Schema initialSchema) {
+ FlatBufferBuilder builder = new FlatBufferBuilder();
+ builder.finish(initialSchema.getSchema(builder));
+ org.apache.arrow.flatbuf.Schema flatBufSchema = org.apache.arrow.flatbuf.Schema.getRootAsSchema(builder.dataBuffer());
+ Schema finalSchema = Schema.convertSchema(flatBufSchema);
+ assertEquals(initialSchema, finalSchema);
+ }
+}
[4/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead
of MaterializedField
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/MapWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 42f3982..af29228 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -17,7 +17,7 @@
*/
<@pp.dropOutputFile />
-<#list ["Single", "Repeated"] as mode>
+<#list ["Single"] as mode>
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" />
<#if mode == "Single">
<#assign containerClass = "MapVector" />
@@ -51,16 +51,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
private final Map<String, FieldWriter> fields = Maps.newHashMap();
<#if mode == "Repeated">private int currentChildIndex = 0;</#if>
- private final boolean unionEnabled;
-
- public ${mode}MapWriter(${containerClass} container, FieldWriter parent, boolean unionEnabled) {
- super(parent);
+ public ${mode}MapWriter(${containerClass} container) {
this.container = container;
- this.unionEnabled = unionEnabled;
- }
-
- public ${mode}MapWriter(${containerClass} container, FieldWriter parent) {
- this(container, parent, false);
}
@Override
@@ -74,7 +66,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return container.getField();
}
@@ -83,12 +75,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
FieldWriter writer = fields.get(name.toLowerCase());
if(writer == null){
int vectorCount=container.size();
- MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class);
- if(!unionEnabled){
- writer = new SingleMapWriter(vector, this);
- } else {
- writer = new PromotableWriter(vector, container);
- }
+ MapVector vector = container.addOrGet(name, MinorType.MAP, MapVector.class);
+ writer = new PromotableWriter(vector, container);
if(vectorCount != container.size()) {
writer.allocate();
}
@@ -125,11 +113,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
FieldWriter writer = fields.get(name.toLowerCase());
int vectorCount = container.size();
if(writer == null) {
- if (!unionEnabled){
- writer = new SingleListWriter(name,container,this);
- } else{
- writer = new PromotableWriter(container.addOrGet(name, Types.optional(MinorType.LIST), ListVector.class), container);
- }
+ writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class), container);
if (container.size() > vectorCount) {
writer.allocate();
}
@@ -206,9 +190,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
}
public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) {
- final MajorType ${upperName}_TYPE = new MajorType(MinorType.${upperName}, DataMode.OPTIONAL, precision, scale, 0, null);
<#else>
- private static final MajorType ${upperName}_TYPE = Types.optional(MinorType.${upperName});
@Override
public ${minor.class}Writer ${lowerName}(String name) {
</#if>
@@ -216,15 +198,9 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
if(writer == null) {
ValueVector vector;
ValueVector currentVector = container.getChild(name);
- if (unionEnabled){
- ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
- writer = new PromotableWriter(v, container);
- vector = v;
- } else {
- ${vectName}Vector v = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class);
- writer = new ${vectName}WriterImpl(v, this);
- vector = v;
- }
+ ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class);
+ writer = new PromotableWriter(v, container);
+ vector = v;
if (currentVector == null || currentVector != vector) {
vector.allocateNewSafe();
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/NullReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java
index 3ef6c7d..ba0c088 100644
--- a/java/vector/src/main/codegen/templates/NullReader.java
+++ b/java/vector/src/main/codegen/templates/NullReader.java
@@ -16,6 +16,9 @@
* limitations under the License.
*/
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.Field;
+
<@pp.dropOutputFile />
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/NullReader.java" />
@@ -31,25 +34,31 @@ package org.apache.arrow.vector.complex.impl;
public class NullReader extends AbstractBaseReader implements FieldReader{
public static final NullReader INSTANCE = new NullReader();
- public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(Types.repeated(MinorType.NULL));
- public static final NullReader EMPTY_MAP_INSTANCE = new NullReader(Types.required(MinorType.MAP));
- private MajorType type;
+ public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(MinorType.NULL);
+ public static final NullReader EMPTY_MAP_INSTANCE = new NullReader(MinorType.MAP);
+ private MinorType type;
private NullReader(){
super();
- type = Types.required(MinorType.NULL);
+ type = MinorType.NULL;
}
- private NullReader(MajorType type){
+ private NullReader(MinorType type){
super();
this.type = type;
}
@Override
- public MajorType getType() {
+ public MinorType getMinorType() {
return type;
}
-
+
+
+ @Override
+ public Field getField() {
+ return new Field("", true, new Null(), null);
+ }
+
public void copyAsValue(MapWriter writer) {}
public void copyAsValue(ListWriter writer) {}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index b0029f7..df50897 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -42,19 +42,79 @@ package org.apache.arrow.vector;
public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector{
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
- private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
+ private final FieldReader reader = new ${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
- private final MaterializedField bitsField = MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED));
- private final MaterializedField valuesField = MaterializedField.create("$values$", new MajorType(field.getType().getMinorType(), DataMode.REQUIRED, field.getPrecision(), field.getScale()));
+ private final String bitsField = "$bits$";
+ private final String valuesField = "$values$";
+ private final Field field;
final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
- final ${valuesName} values = new ${minor.class}Vector(valuesField, allocator);
+ final ${valuesName} values;
- private final Mutator mutator = new Mutator();
- private final Accessor accessor = new Accessor();
+ private final Mutator mutator;
+ private final Accessor accessor;
- public ${className}(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
+ <#if minor.class == "Decimal">
+ private final int precision;
+ private final int scale;
+
+ public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
+ super(name, allocator);
+ values = new ${minor.class}Vector(valuesField, allocator, precision, scale);
+ this.precision = precision;
+ this.scale = scale;
+ mutator = new Mutator();
+ accessor = new Accessor();
+ field = new Field(name, true, new Decimal(precision, scale), null);
+ }
+ <#else>
+ public ${className}(String name, BufferAllocator allocator) {
+ super(name, allocator);
+ values = new ${minor.class}Vector(valuesField, allocator);
+ mutator = new Mutator();
+ accessor = new Accessor();
+ <#if minor.class == "TinyInt" ||
+ minor.class == "SmallInt" ||
+ minor.class == "Int" ||
+ minor.class == "BigInt">
+ field = new Field(name, true, new Int(${type.width} * 8, true), null);
+ <#elseif minor.class == "UInt1" ||
+ minor.class == "UInt2" ||
+ minor.class == "UInt4" ||
+ minor.class == "UInt8">
+ field = new Field(name, true, new Int(${type.width} * 8, false), null);
+ <#elseif minor.class == "Date">
+ field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Date(), null);
+ <#elseif minor.class == "Time">
+ field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), null);
+ <#elseif minor.class == "Float4">
+ field = new Field(name, true, new FloatingPoint(0), null);
+ <#elseif minor.class == "Float8">
+ field = new Field(name, true, new FloatingPoint(1), null);
+ <#elseif minor.class == "TimeStamp">
+ field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(""), null);
+ <#elseif minor.class == "IntervalDay">
+ field = new Field(name, true, new IntervalDay(), null);
+ <#elseif minor.class == "IntervalYear">
+ field = new Field(name, true, new IntervalYear(), null);
+ <#elseif minor.class == "VarChar">
+ field = new Field(name, true, new Utf8(), null);
+ <#elseif minor.class == "VarBinary">
+ field = new Field(name, true, new Binary(), null);
+ <#elseif minor.class == "Bit">
+ field = new Field(name, true, new Bool(), null);
+ </#if>
+ }
+ </#if>
+
+ @Override
+ public Field getField() {
+ return field;
+ }
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.${minor.class?upper_case};
}
@Override
@@ -240,12 +300,13 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public TransferPair getTransferPair(BufferAllocator allocator){
- return new TransferImpl(getField(), allocator);
+ return new TransferImpl(name, allocator);
+
}
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator){
- return new TransferImpl(getField().withPath(ref), allocator);
+ return new TransferImpl(ref, allocator);
}
@Override
@@ -273,8 +334,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
private class TransferImpl implements TransferPair {
Nullable${minor.class}Vector to;
- public TransferImpl(MaterializedField field, BufferAllocator allocator){
- to = new Nullable${minor.class}Vector(field, allocator);
+ public TransferImpl(String name, BufferAllocator allocator){
+ <#if minor.class == "Decimal">
+ to = new Nullable${minor.class}Vector(name, allocator, precision, scale);
+ <#else>
+ to = new Nullable${minor.class}Vector(name, allocator);
+ </#if>
}
public TransferImpl(Nullable${minor.class}Vector to){
@@ -312,17 +377,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
return mutator;
}
- public ${minor.class}Vector convertToRequiredVector(){
- ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
- if (v.data != null) {
- v.data.release(1);
- }
- v.data = values.data;
- v.data.retain(1);
- clear();
- return v;
- }
-
public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
final Accessor fromAccessor = from.getAccessor();
if (!fromAccessor.isNull(fromIndex)) {
@@ -389,8 +443,8 @@ public final class ${className} extends BaseDataValueVector implements <#if type
holder.isSet = bAccessor.get(index);
<#if minor.class.startsWith("Decimal")>
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
+ holder.scale = scale;
+ holder.precision = precision;
</#if>
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/RepeatedValueVectors.java b/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
deleted file mode 100644
index ceae53b..0000000
--- a/java/vector/src/main/codegen/templates/RepeatedValueVectors.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<#list vv.types as type>
-<#list type.minor as minor>
-<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
-<#assign fields = minor.fields!type.fields />
-
-<@pp.changeOutputFile name="/org/apache/arrow/vector/Repeated${minor.class}Vector.java" />
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector;
-
-<#include "/@includes/vv_imports.ftl" />
-
-/**
- * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
- * repeated protobuf field). The implementation uses two additional value vectors; one to convert
- * the index offset to the underlying element offset, and another to store the number of values
- * in the vector.
- *
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
- */
-
-public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>VectorLike {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Repeated${minor.class}Vector.class);
-
- // we maintain local reference to concrete vector type for performance reasons.
- ${minor.class}Vector values;
- private final FieldReader reader = new Repeated${minor.class}ReaderImpl(Repeated${minor.class}Vector.this);
- private final Mutator mutator = new Mutator();
- private final Accessor accessor = new Accessor();
-
- public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- addOrGetVector(VectorDescriptor.create(new MajorType(field.getType().getMinorType(), DataMode.REQUIRED)));
- }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- @Override
- public Accessor getAccessor() {
- return accessor;
- }
-
- @Override
- public FieldReader getReader() {
- return reader;
- }
-
- @Override
- public ${minor.class}Vector getDataVector() {
- return values;
- }
-
- @Override
- public TransferPair getTransferPair(BufferAllocator allocator) {
- return new TransferImpl(getField(), allocator);
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator){
- return new TransferImpl(getField().withPath(ref), allocator);
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new TransferImpl((Repeated${minor.class}Vector) to);
- }
-
- @Override
- public AddOrGetResult<${minor.class}Vector> addOrGetVector(VectorDescriptor descriptor) {
- final AddOrGetResult<${minor.class}Vector> result = super.addOrGetVector(descriptor);
- if (result.isCreated()) {
- values = result.getVector();
- }
- return result;
- }
-
- public void transferTo(Repeated${minor.class}Vector target) {
- target.clear();
- offsets.transferTo(target.offsets);
- values.transferTo(target.values);
- clear();
- }
-
- public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) {
- final UInt4Vector.Accessor a = offsets.getAccessor();
- final UInt4Vector.Mutator m = to.offsets.getMutator();
-
- final int startPos = a.get(startIndex);
- final int endPos = a.get(startIndex + groups);
- final int valuesToCopy = endPos - startPos;
-
- values.splitAndTransferTo(startPos, valuesToCopy, to.values);
- to.offsets.clear();
- to.offsets.allocateNew(groups + 1);
- int normalizedPos = 0;
- for (int i=0; i < groups + 1;i++ ) {
- normalizedPos = a.get(startIndex+i) - startPos;
- m.set(i, normalizedPos);
- }
- m.setValueCount(groups == 0 ? 0 : groups + 1);
- }
-
- private class TransferImpl implements TransferPair {
- final Repeated${minor.class}Vector to;
-
- public TransferImpl(MaterializedField field, BufferAllocator allocator) {
- this.to = new Repeated${minor.class}Vector(field, allocator);
- }
-
- public TransferImpl(Repeated${minor.class}Vector to) {
- this.to = to;
- }
-
- @Override
- public Repeated${minor.class}Vector getTo() {
- return to;
- }
-
- @Override
- public void transfer() {
- transferTo(to);
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- splitAndTransferTo(startIndex, length, to);
- }
-
- @Override
- public void copyValueSafe(int fromIndex, int toIndex) {
- to.copyFromSafe(fromIndex, toIndex, Repeated${minor.class}Vector.this);
- }
- }
-
- public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
- final Accessor vAccessor = v.getAccessor();
- final int count = vAccessor.getInnerValueCountAt(inIndex);
- mutator.startNewValue(outIndex);
- for (int i = 0; i < count; i++) {
- mutator.add(outIndex, vAccessor.get(inIndex, i));
- }
- }
-
- public void copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v) {
- final Accessor vAccessor = v.getAccessor();
- final int count = vAccessor.getInnerValueCountAt(inIndex);
- mutator.startNewValue(outIndex);
- for (int i = 0; i < count; i++) {
- mutator.addSafe(outIndex, vAccessor.get(inIndex, i));
- }
- }
-
- public boolean allocateNewSafe() {
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- if(!offsets.allocateNewSafe()) return false;
- if(!values.allocateNewSafe()) return false;
- success = true;
- } finally {
- if (!success) {
- clear();
- }
- }
- offsets.zeroVector();
- mutator.reset();
- return true;
- }
-
- @Override
- public void allocateNew() {
- try {
- offsets.allocateNew();
- values.allocateNew();
- } catch (OutOfMemoryException e) {
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- <#if type.major == "VarLen">
-// @Override
-// protected SerializedField.Builder getMetadataBuilder() {
-// return super.getMetadataBuilder()
-// .setVarByteLength(values.getVarByteLength());
-// }
-
- public void allocateNew(int totalBytes, int valueCount, int innerValueCount) {
- try {
- offsets.allocateNew(valueCount + 1);
- values.allocateNew(totalBytes, innerValueCount);
- } catch (OutOfMemoryException e) {
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- public int getByteCapacity(){
- return values.getByteCapacity();
- }
-
- <#else>
-
- @Override
- public void allocateNew(int valueCount, int innerValueCount) {
- clear();
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to//
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- offsets.allocateNew(valueCount + 1);
- values.allocateNew(innerValueCount);
- } catch(OutOfMemoryException e){
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- </#if>
-
- // This is declared a subclass of the accessor declared inside of FixedWidthVector, this is also used for
- // variable length vectors, as they should ahve consistent interface as much as possible, if they need to diverge
- // in the future, the interface shold be declared in the respective value vector superclasses for fixed and variable
- // and we should refer to each in the generation template
- public final class Accessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
- @Override
- public List<${friendlyType}> getObject(int index) {
- final List<${friendlyType}> vals = new JsonStringArrayList<>();
- final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
- final int start = offsetsAccessor.get(index);
- final int end = offsetsAccessor.get(index + 1);
- final ${minor.class}Vector.Accessor valuesAccessor = values.getAccessor();
- for(int i = start; i < end; i++) {
- vals.add(valuesAccessor.getObject(i));
- }
- return vals;
- }
-
- public ${friendlyType} getSingleObject(int index, int arrayIndex) {
- final int start = offsets.getAccessor().get(index);
- return values.getAccessor().getObject(start + arrayIndex);
- }
-
- /**
- * Get a value for the given record. Each element in the repeated field is accessed by
- * the positionIndex param.
- *
- * @param index record containing the repeated field
- * @param positionIndex position within the repeated field
- * @return element at the given position in the given record
- */
- public <#if type.major == "VarLen">byte[]
- <#else>${minor.javaType!type.javaType}
- </#if> get(int index, int positionIndex) {
- return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
- }
-
- public void get(int index, Repeated${minor.class}Holder holder) {
- holder.start = offsets.getAccessor().get(index);
- holder.end = offsets.getAccessor().get(index+1);
- holder.vector = values;
- }
-
- public void get(int index, int positionIndex, ${minor.class}Holder holder) {
- final int offset = offsets.getAccessor().get(index);
- assert offset >= 0;
- assert positionIndex < getInnerValueCountAt(index);
- values.getAccessor().get(offset + positionIndex, holder);
- }
-
- public void get(int index, int positionIndex, Nullable${minor.class}Holder holder) {
- final int offset = offsets.getAccessor().get(index);
- assert offset >= 0;
- if (positionIndex >= getInnerValueCountAt(index)) {
- holder.isSet = 0;
- return;
- }
- values.getAccessor().get(offset + positionIndex, holder);
- }
- }
-
- public final class Mutator extends BaseRepeatedValueVector.BaseRepeatedMutator implements RepeatedMutator {
- private Mutator() {}
-
- /**
- * Add an element to the given record index. This is similar to the set() method in other
- * value vectors, except that it permits setting multiple values for a single record.
- *
- * @param index record of the element to add
- * @param value value to add to the given row
- */
- public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
- int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().set(nextOffset, value);
- offsets.getMutator().set(index+1, nextOffset+1);
- }
-
- <#if type.major == "VarLen">
- public void addSafe(int index, byte[] bytes) {
- addSafe(index, bytes, 0, bytes.length);
- }
-
- public void addSafe(int index, byte[] bytes, int start, int length) {
- final int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().setSafe(nextOffset, bytes, start, length);
- offsets.getMutator().setSafe(index+1, nextOffset+1);
- }
-
- <#else>
-
- public void addSafe(int index, ${minor.javaType!type.javaType} srcValue) {
- final int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().setSafe(nextOffset, srcValue);
- offsets.getMutator().setSafe(index+1, nextOffset+1);
- }
-
- </#if>
-
- public void setSafe(int index, Repeated${minor.class}Holder h) {
- final ${minor.class}Holder ih = new ${minor.class}Holder();
- final ${minor.class}Vector.Accessor hVectorAccessor = h.vector.getAccessor();
- mutator.startNewValue(index);
- for(int i = h.start; i < h.end; i++){
- hVectorAccessor.get(i, ih);
- mutator.addSafe(index, ih);
- }
- }
-
- public void addSafe(int index, ${minor.class}Holder holder) {
- int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().setSafe(nextOffset, holder);
- offsets.getMutator().setSafe(index+1, nextOffset+1);
- }
-
- public void addSafe(int index, Nullable${minor.class}Holder holder) {
- final int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().setSafe(nextOffset, holder);
- offsets.getMutator().setSafe(index+1, nextOffset+1);
- }
-
- <#if (fields?size > 1) && !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
- public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- int nextOffset = offsets.getAccessor().get(arrayIndex+1);
- values.getMutator().setSafe(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
- offsets.getMutator().setSafe(arrayIndex+1, nextOffset+1);
- }
- </#if>
-
- protected void add(int index, ${minor.class}Holder holder) {
- int nextOffset = offsets.getAccessor().get(index+1);
- values.getMutator().set(nextOffset, holder);
- offsets.getMutator().set(index+1, nextOffset+1);
- }
-
- public void add(int index, Repeated${minor.class}Holder holder) {
-
- ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
- ${minor.class}Holder innerHolder = new ${minor.class}Holder();
-
- for(int i = holder.start; i < holder.end; i++) {
- accessor.get(i, innerHolder);
- add(index, innerHolder);
- }
- }
-
- @Override
- public void generateTestData(final int valCount) {
- final int[] sizes = {1, 2, 0, 6};
- int size = 0;
- int runningOffset = 0;
- final UInt4Vector.Mutator offsetsMutator = offsets.getMutator();
- for(int i = 1; i < valCount + 1; i++, size++) {
- runningOffset += sizes[size % sizes.length];
- offsetsMutator.set(i, runningOffset);
- }
- values.getMutator().generateTestData(valCount * 9);
- setValueCount(size);
- }
-
- @Override
- public void reset() {
- }
- }
-}
-</#list>
-</#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/UnionListWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index 9a6b08f..49d57e7 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -43,7 +43,6 @@ public class UnionListWriter extends AbstractFieldWriter {
private int lastIndex = 0;
public UnionListWriter(ListVector vector) {
- super(null);
this.vector = vector;
this.writer = new PromotableWriter(vector.getDataVector(), vector);
this.offsets = vector.getOffsetVector();
@@ -64,10 +63,14 @@ public class UnionListWriter extends AbstractFieldWriter {
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return null;
}
+ public void setValueCount(int count) {
+ vector.getMutator().setValueCount(count);
+ }
+
@Override
public int getValueCapacity() {
return vector.getValueCapacity();
@@ -78,6 +81,12 @@ public class UnionListWriter extends AbstractFieldWriter {
}
+ @Override
+ public void setPosition(int index) {
+ super.setPosition(index);
+ startList();
+ }
+
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
@@ -91,7 +100,7 @@ public class UnionListWriter extends AbstractFieldWriter {
@Override
public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
- assert inMap;
+// assert inMap;
mapName = name;
final int nextOffset = offsets.getAccessor().get(idx() + 1);
vector.getMutator().setNotNull(idx());
@@ -146,7 +155,7 @@ public class UnionListWriter extends AbstractFieldWriter {
@Override
public void start() {
- assert inMap;
+// assert inMap;
final int nextOffset = offsets.getAccessor().get(idx() + 1);
vector.getMutator().setNotNull(idx());
offsets.getMutator().setSafe(idx() + 1, nextOffset);
@@ -155,11 +164,11 @@ public class UnionListWriter extends AbstractFieldWriter {
@Override
public void end() {
- if (inMap) {
+// if (inMap) {
inMap = false;
final int nextOffset = offsets.getAccessor().get(idx() + 1);
offsets.getMutator().setSafe(idx() + 1, nextOffset + 1);
- }
+// }
}
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
@@ -170,7 +179,7 @@ public class UnionListWriter extends AbstractFieldWriter {
@Override
public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- assert !inMap;
+// assert !inMap;
final int nextOffset = offsets.getAccessor().get(idx() + 1);
vector.getMutator().setNotNull(idx());
writer.setPosition(nextOffset);
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/UnionReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index 44c3e55..7351ae3 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -17,6 +17,8 @@
*/
+import org.apache.arrow.vector.types.Types.MinorType;
+
<@pp.dropOutputFile />
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionReader.java" />
@@ -37,18 +39,18 @@ public class UnionReader extends AbstractFieldReader {
this.data = data;
}
- private static MajorType[] TYPES = new MajorType[43];
+ public MinorType getMinorType() {
+ return TYPES[data.getTypeValue(idx())];
+ }
+
+ private static MinorType[] TYPES = new MinorType[43];
static {
for (MinorType minorType : MinorType.values()) {
- TYPES[minorType.ordinal()] = new MajorType(minorType, DataMode.OPTIONAL);
+ TYPES[minorType.ordinal()] = minorType;
}
}
- public MajorType getType() {
- return TYPES[data.getTypeValue(idx())];
- }
-
public boolean isSet(){
return !data.getAccessor().isNull(idx());
}
@@ -69,7 +71,7 @@ public class UnionReader extends AbstractFieldReader {
return reader;
}
switch (MinorType.values()[typeValue]) {
- case LATE:
+ case NULL:
return NullReader.INSTANCE;
case MAP:
return (FieldReader) getMap();
@@ -119,9 +121,9 @@ public class UnionReader extends AbstractFieldReader {
writer.data.copyFrom(idx(), writer.idx(), data);
}
- <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
- "Character", "DateTime", "Period", "Double", "Float",
- "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
+ <#list ["Object", "Integer", "Long", "Boolean",
+ "Character", "DateTime", "Double", "Float",
+ "Text", "Byte", "Short", "byte[]"] as friendlyType>
<#assign safeType=friendlyType />
<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
@@ -141,11 +143,11 @@ public class UnionReader extends AbstractFieldReader {
<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
<#if !minor.class?starts_with("Decimal")>
- private Nullable${name}ReaderImpl ${uncappedName}Reader;
+ private ${name}ReaderImpl ${uncappedName}Reader;
- private Nullable${name}ReaderImpl get${name}() {
+ private ${name}ReaderImpl get${name}() {
if (${uncappedName}Reader == null) {
- ${uncappedName}Reader = new Nullable${name}ReaderImpl(data.get${name}Vector());
+ ${uncappedName}Reader = new ${name}ReaderImpl(data.get${name}Vector());
${uncappedName}Reader.setPosition(idx());
readers[MinorType.${name?upper_case}.ordinal()] = ${uncappedName}Reader;
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 0f089b7..e2f19f4 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -16,6 +16,16 @@
* limitations under the License.
*/
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Field;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.flatbuf.Union;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+
+import java.util.ArrayList;
+import java.util.List;
+
<@pp.dropOutputFile />
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/UnionVector.java" />
@@ -29,7 +39,6 @@ import java.util.ArrayList;
import java.util.Iterator;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.BasicTypeHelper;
/*
* This class is generated using freemarker and the ${.template_name} template.
@@ -47,34 +56,30 @@ import org.apache.arrow.vector.util.BasicTypeHelper;
*/
public class UnionVector implements ValueVector {
- private MaterializedField field;
+ private String name;
private BufferAllocator allocator;
private Accessor accessor = new Accessor();
private Mutator mutator = new Mutator();
int valueCount;
MapVector internalMap;
- private UInt1Vector typeVector;
+ UInt1Vector typeVector;
private MapVector mapVector;
private ListVector listVector;
private FieldReader reader;
- private NullableBitVector bit;
private int singleType = 0;
private ValueVector singleVector;
- private MajorType majorType;
private final CallBack callBack;
- public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- this.field = field.clone();
+ public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
+ this.name = name;
this.allocator = allocator;
this.internalMap = new MapVector("internal", allocator, callBack);
- this.typeVector = internalMap.addOrGet("types", new MajorType(MinorType.UINT1, DataMode.REQUIRED), UInt1Vector.class);
- this.field.addChild(internalMap.getField().clone());
- this.majorType = field.getType();
+ this.typeVector = new UInt1Vector("types", allocator);
this.callBack = callBack;
}
@@ -82,34 +87,20 @@ public class UnionVector implements ValueVector {
return allocator;
}
- public List<MinorType> getSubTypes() {
- return majorType.getSubTypes();
- }
-
- public void addSubType(MinorType type) {
- if (majorType.getSubTypes().contains(type)) {
- return;
- }
- List<MinorType> subTypes = this.majorType.getSubTypes();
- List<MinorType> newSubTypes = new ArrayList<>(subTypes);
- newSubTypes.add(type);
- majorType = new MajorType(this.majorType.getMinorType(), this.majorType.getMode(), this.majorType.getPrecision(),
- this.majorType.getScale(), this.majorType.getTimezone(), newSubTypes);
- field = MaterializedField.create(field.getName(), majorType);
- if (callBack != null) {
- callBack.doWork();
- }
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.UNION;
}
- private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
-
public MapVector getMap() {
if (mapVector == null) {
int vectorCount = internalMap.size();
- mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class);
- addSubType(MinorType.MAP);
+ mapVector = internalMap.addOrGet("map", MinorType.MAP, MapVector.class);
if (internalMap.size() > vectorCount) {
mapVector.allocateNew();
+ if (callBack != null) {
+ callBack.doWork();
+ }
}
}
return mapVector;
@@ -121,15 +112,16 @@ public class UnionVector implements ValueVector {
<#if !minor.class?starts_with("Decimal")>
private Nullable${name}Vector ${uncappedName}Vector;
- private static final MajorType ${name?upper_case}_TYPE = new MajorType(MinorType.${name?upper_case}, DataMode.OPTIONAL);
public Nullable${name}Vector get${name}Vector() {
if (${uncappedName}Vector == null) {
int vectorCount = internalMap.size();
- ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class);
- addSubType(MinorType.${name?upper_case});
+ ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", MinorType.${name?upper_case}, Nullable${name}Vector.class);
if (internalMap.size() > vectorCount) {
${uncappedName}Vector.allocateNew();
+ if (callBack != null) {
+ callBack.doWork();
+ }
}
}
return ${uncappedName}Vector;
@@ -139,15 +131,15 @@ public class UnionVector implements ValueVector {
</#list></#list>
- private static final MajorType LIST_TYPE = new MajorType(MinorType.LIST, DataMode.OPTIONAL);
-
public ListVector getList() {
if (listVector == null) {
int vectorCount = internalMap.size();
- listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class);
- addSubType(MinorType.LIST);
+ listVector = internalMap.addOrGet("list", MinorType.LIST, ListVector.class);
if (internalMap.size() > vectorCount) {
listVector.allocateNew();
+ if (callBack != null) {
+ callBack.doWork();
+ }
}
}
return listVector;
@@ -164,6 +156,7 @@ public class UnionVector implements ValueVector {
@Override
public void allocateNew() throws OutOfMemoryException {
internalMap.allocateNew();
+ typeVector.allocateNew();
if (typeVector != null) {
typeVector.zeroVector();
}
@@ -172,6 +165,7 @@ public class UnionVector implements ValueVector {
@Override
public boolean allocateNewSafe() {
boolean safe = internalMap.allocateNewSafe();
+ safe = safe && typeVector.allocateNewSafe();
if (safe) {
if (typeVector != null) {
typeVector.zeroVector();
@@ -196,22 +190,27 @@ public class UnionVector implements ValueVector {
@Override
public void clear() {
+ typeVector.clear();
internalMap.clear();
}
@Override
- public MaterializedField getField() {
- return field;
+ public Field getField() {
+ List<org.apache.arrow.vector.types.pojo.Field> childFields = new ArrayList<>();
+ for (ValueVector v : internalMap.getChildren()) {
+ childFields.add(v.getField());
+ }
+ return new Field(name, true, new ArrowType.Union(), childFields);
}
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return new TransferImpl(field, allocator);
+ return new TransferImpl(name, allocator);
}
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new TransferImpl(field.withPath(ref), allocator);
+ return new TransferImpl(ref, allocator);
}
@Override
@@ -219,10 +218,9 @@ public class UnionVector implements ValueVector {
return new TransferImpl((UnionVector) target);
}
- public void transferTo(UnionVector target) {
+ public void transferTo(org.apache.arrow.vector.complex.UnionVector target) {
internalMap.makeTransferPair(target.internalMap).transfer();
target.valueCount = valueCount;
- target.majorType = majorType;
}
public void copyFrom(int inIndex, int outIndex, UnionVector from) {
@@ -236,13 +234,14 @@ public class UnionVector implements ValueVector {
}
public ValueVector addVector(ValueVector v) {
- String name = v.getField().getType().getMinorType().name().toLowerCase();
- MajorType type = v.getField().getType();
+ String name = v.getMinorType().name().toLowerCase();
Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
- final ValueVector newVector = internalMap.addOrGet(name, type, (Class<ValueVector>) BasicTypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()));
+ final ValueVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass());
v.makeTransferPair(newVector).transfer();
internalMap.putChild(name, newVector);
- addSubType(v.getField().getType().getMinorType());
+ if (callBack != null) {
+ callBack.doWork();
+ }
return newVector;
}
@@ -250,8 +249,8 @@ public class UnionVector implements ValueVector {
UnionVector to;
- public TransferImpl(MaterializedField field, BufferAllocator allocator) {
- to = new UnionVector(field, allocator, null);
+ public TransferImpl(String name, BufferAllocator allocator) {
+ to = new UnionVector(name, allocator, null);
}
public TransferImpl(UnionVector to) {
@@ -357,7 +356,7 @@ public class UnionVector implements ValueVector {
public Object getObject(int index) {
int type = typeVector.getAccessor().get(index);
switch (MinorType.values()[type]) {
- case LATE:
+ case NULL:
return null;
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
@@ -421,7 +420,7 @@ public class UnionVector implements ValueVector {
writer = new UnionWriter(UnionVector.this);
}
writer.setPosition(index);
- MinorType type = reader.getType().getMinorType();
+ MinorType type = reader.getMinorType();
switch (type) {
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
@@ -460,7 +459,7 @@ public class UnionVector implements ValueVector {
</#list></#list>
public void setType(int index, MinorType type) {
- typeVector.getMutator().setSafe(index, type.ordinal());
+ typeVector.getMutator().setSafe(index, (byte) type.ordinal());
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/UnionWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index c9c29e0..1137e2c 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -37,17 +37,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
private UnionListWriter listWriter;
private List<BaseWriter> writers = Lists.newArrayList();
- public UnionWriter(BufferAllocator allocator) {
- super(null);
- }
-
public UnionWriter(UnionVector vector) {
- super(null);
- data = vector;
- }
-
- public UnionWriter(UnionVector vector, FieldWriter parent) {
- super(null);
data = vector;
}
@@ -84,7 +74,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
private MapWriter getMapWriter() {
if (mapWriter == null) {
- mapWriter = new SingleMapWriter(data.getMap(), null, true);
+ mapWriter = new SingleMapWriter(data.getMap());
mapWriter.setPosition(idx());
writers.add(mapWriter);
}
@@ -120,7 +110,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
private ${name}Writer get${name}Writer() {
if (${uncappedName}Writer == null) {
- ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null);
+ ${uncappedName}Writer = new ${name}WriterImpl(data.get${name}Vector());
${uncappedName}Writer.setPosition(idx());
writers.add(${uncappedName}Writer);
}
@@ -217,7 +207,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return data.getField();
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java
index 2b14194..d744c52 100644
--- a/java/vector/src/main/codegen/templates/ValueHolders.java
+++ b/java/vector/src/main/codegen/templates/ValueHolders.java
@@ -31,10 +31,6 @@ package org.apache.arrow.vector.holders;
public final class ${className} implements ValueHolder{
- public static final MajorType TYPE = new MajorType(MinorType.${minor.class?upper_case}, DataMode.${mode.name?upper_case});
-
- public MajorType getType() {return TYPE;}
-
<#if mode.name == "Repeated">
/** The first index (inclusive) into the Vector. **/
@@ -49,48 +45,13 @@ public final class ${className} implements ValueHolder{
<#else>
public static final int WIDTH = ${type.width};
- <#if mode.name == "Optional">public int isSet;</#if>
+ <#if mode.name == "Optional">public int isSet;
+ <#else>public final int isSet = 1;</#if>
<#assign fields = minor.fields!type.fields />
<#list fields as field>
public ${field.type} ${field.name};
</#list>
- <#if minor.class.startsWith("Decimal")>
- public static final int maxPrecision = ${minor.maxPrecisionDigits};
- <#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")>
- public static final int nDecimalDigits = ${minor.nDecimalDigits};
-
- public static int getInteger(int index, int start, ArrowBuf buffer) {
- int value = buffer.getInt(start + (index * 4));
-
- if (index == 0) {
- /* the first byte contains sign bit, return value without it */
- <#if minor.class.endsWith("Sparse")>
- value = (value & 0x7FFFFFFF);
- <#elseif minor.class.endsWith("Dense")>
- value = (value & 0x0000007F);
- </#if>
- }
- return value;
- }
-
- public static void setInteger(int index, int value, int start, ArrowBuf buffer) {
- buffer.setInt(start + (index * 4), value);
- }
-
- public static void setSign(boolean sign, int start, ArrowBuf buffer) {
- // Set MSB to 1 if sign is negative
- if (sign == true) {
- int value = getInteger(0, start, buffer);
- setInteger(0, (value | 0x80000000), start, buffer);
- }
- }
-
- public static boolean getSign(int start, ArrowBuf buffer) {
- return ((buffer.getInt(start) & 0x80000000) != 0);
- }
- </#if></#if>
-
@Deprecated
public int hashCode(){
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index 84fb3eb..bcd639a 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -56,9 +56,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
private static final int MIN_BYTE_COUNT = 4096;
public final static String OFFSETS_VECTOR_NAME = "$offsets$";
- private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
- final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
- private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
+ final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(OFFSETS_VECTOR_NAME, allocator);
private final Accessor accessor;
private final Mutator mutator;
@@ -68,16 +66,42 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
private int allocationSizeInBytes = INITIAL_BYTE_COUNT;
private int allocationMonitor = 0;
- public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
+ <#if minor.class == "Decimal">
+
+ private final int precision;
+ private final int scale;
+
+ public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) {
+ super(name, allocator);
+ this.oAccessor = offsetVector.getAccessor();
+ this.accessor = new Accessor();
+ this.mutator = new Mutator();
+ this.precision = precision;
+ this.scale = scale;
+ }
+ <#else>
+
+ public ${minor.class}Vector(String name, BufferAllocator allocator) {
+ super(name, allocator);
this.oAccessor = offsetVector.getAccessor();
this.accessor = new Accessor();
this.mutator = new Mutator();
}
+ </#if>
+
+ @Override
+ public Field getField() {
+ throw new UnsupportedOperationException("internal vector");
+ }
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.${minor.class?upper_case};
+ }
@Override
public FieldReader getReader(){
- return reader;
+ throw new UnsupportedOperationException("internal vector");
}
@Override
@@ -125,27 +149,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
return offsetVector.getAccessor().get(valueCount);
}
-// @Override
-// public SerializedField getMetadata() {
-// return getMetadataBuilder() //
-// .addChild(offsetVector.getMetadata())
-// .setValueCount(getAccessor().getValueCount()) //
-// .setBufferLength(getBufferSize()) //
-// .build();
-// }
-//
-// @Override
-// public void load(SerializedField metadata, ArrowBuf buffer) {
-// the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
-// final SerializedField offsetField = metadata.getChild(0);
-// offsetVector.load(offsetField, buffer);
-//
-// final int capacity = buffer.capacity();
-// final int offsetsLength = offsetField.getBufferLength();
-// data = buffer.slice(offsetsLength, capacity - offsetsLength);
-// data.retain();
-// }
-
@Override
public void clear() {
super.clear();
@@ -175,12 +178,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
@Override
public TransferPair getTransferPair(BufferAllocator allocator){
- return new TransferImpl(getField(), allocator);
+ return new TransferImpl(name, allocator);
}
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator){
- return new TransferImpl(getField().withPath(ref), allocator);
+ return new TransferImpl(ref, allocator);
}
@Override
@@ -241,8 +244,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
private class TransferImpl implements TransferPair{
${minor.class}Vector to;
- public TransferImpl(MaterializedField field, BufferAllocator allocator){
- to = new ${minor.class}Vector(field, allocator);
+ public TransferImpl(String name, BufferAllocator allocator){
+ <#if minor.class == "Decimal">
+ to = new ${minor.class}Vector(name, allocator, precision, scale);
+ <#else>
+ to = new ${minor.class}Vector(name, allocator);
+ </#if>
}
public TransferImpl(${minor.class}Vector to){
@@ -426,10 +433,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
return text;
}
<#break>
- <#case "Var16Char">
+ <#case "Decimal">
@Override
public ${friendlyType} getObject(int index) {
- return new String(get(index), Charsets.UTF_16);
+ return new BigDecimal(new BigInteger(get(index)), scale);
}
<#break>
<#default>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
index b129ea9..05b7cf1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -20,7 +20,6 @@ package org.apache.arrow.vector;
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.types.MaterializedField;
public abstract class BaseDataValueVector extends BaseValueVector {
@@ -29,8 +28,8 @@ public abstract class BaseDataValueVector extends BaseValueVector {
protected ArrowBuf data;
- public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
+ public BaseDataValueVector(String name, BufferAllocator allocator) {
+ super(name, allocator);
data = allocator.getEmpty();
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 932e6f1..884cdf0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -19,8 +19,8 @@ package org.apache.arrow.vector;
import java.util.Iterator;
+import com.google.flatbuffers.FlatBufferBuilder;
import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.types.MaterializedField;
import org.apache.arrow.vector.util.TransferPair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,16 +38,16 @@ public abstract class BaseValueVector implements ValueVector {
public static final int INITIAL_VALUE_ALLOCATION = 4096;
protected final BufferAllocator allocator;
- protected final MaterializedField field;
+ protected final String name;
- protected BaseValueVector(MaterializedField field, BufferAllocator allocator) {
- this.field = Preconditions.checkNotNull(field, "field cannot be null");
+ protected BaseValueVector(String name, BufferAllocator allocator) {
this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
+ this.name = name;
}
@Override
public String toString() {
- return super.toString() + "[field = " + field + ", ...]";
+ return super.toString() + "[name = " + name + ", ...]";
}
@Override
@@ -61,29 +61,10 @@ public abstract class BaseValueVector implements ValueVector {
}
@Override
- public MaterializedField getField() {
- return field;
- }
-
- public MaterializedField getField(String ref){
- return getField().withPath(ref);
- }
-
- @Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return getTransferPair(getField().getPath(), allocator);
+ return getTransferPair(name, allocator);
}
-// public static SerializedField getMetadata(BaseValueVector vector) {
-// return getMetadataBuilder(vector).build();
-// }
-//
-// protected static SerializedField.Builder getMetadataBuilder(BaseValueVector vector) {
-// return SerializedFieldHelper.getAsBuilder(vector.getField())
-// .setValueCount(vector.getAccessor().getValueCount())
-// .setBufferLength(vector.getBufferSize());
-// }
-
public abstract static class BaseAccessor implements ValueVector.Accessor {
protected BaseAccessor() { }
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index c5bcb2d..fee6e9c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -21,11 +21,11 @@ import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.complex.impl.BitReaderImpl;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.holders.BitHolder;
import org.apache.arrow.vector.holders.NullableBitHolder;
-import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.util.TransferPair;
@@ -37,7 +37,6 @@ import org.apache.arrow.vector.util.TransferPair;
public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
- private final FieldReader reader = new BitReaderImpl(BitVector.this);
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
@@ -45,13 +44,23 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
private int allocationMonitor = 0;
- public BitVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
+ public BitVector(String name, BufferAllocator allocator) {
+ super(name, allocator);
+ }
+
+ @Override
+ public Field getField() {
+ throw new UnsupportedOperationException("internal vector");
+ }
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.BIT;
}
@Override
public FieldReader getReader() {
- return reader;
+ throw new UnsupportedOperationException("internal vector");
}
@Override
@@ -180,20 +189,6 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
return true;
}
-// @Override
-// public void load(SerializedField metadata, DrillBuf buffer) {
-// Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
-// final int valueCount = metadata.getValueCount();
-// final int expectedLength = getSizeFromCount(valueCount);
-// final int actualLength = metadata.getBufferLength();
-// assert expectedLength == actualLength: "expected and actual buffer sizes do not match";
-//
-// clear();
-// data = buffer.slice(0, actualLength);
-// data.retain();
-// this.valueCount = valueCount;
-// }
-
@Override
public Mutator getMutator() {
return new Mutator();
@@ -206,12 +201,12 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return new TransferImpl(getField(), allocator);
+ return new TransferImpl(name, allocator);
}
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new TransferImpl(getField().withPath(ref), allocator);
+ return new TransferImpl(ref, allocator);
}
@Override
@@ -270,8 +265,8 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
private class TransferImpl implements TransferPair {
BitVector to;
- public TransferImpl(MaterializedField field, BufferAllocator allocator) {
- this.to = new BitVector(field, allocator);
+ public TransferImpl(String name, BufferAllocator allocator) {
+ this.to = new BitVector(name, allocator);
}
public TransferImpl(BitVector to) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
deleted file mode 100644
index b806b18..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ObjectHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.util.TransferPair;
-
-public class ObjectVector extends BaseValueVector {
- private final Accessor accessor = new Accessor();
- private final Mutator mutator = new Mutator();
- private int maxCount = 0;
- private int count = 0;
- private int allocationSize = 4096;
-
- private List<Object[]> objectArrayList = new ArrayList<>();
-
- public ObjectVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- public void addNewArray() {
- objectArrayList.add(new Object[allocationSize]);
- maxCount += allocationSize;
- }
-
- @Override
- public FieldReader getReader() {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- public final class Mutator implements ValueVector.Mutator {
-
- public void set(int index, Object obj) {
- int listOffset = index / allocationSize;
- if (listOffset >= objectArrayList.size()) {
- addNewArray();
- }
- objectArrayList.get(listOffset)[index % allocationSize] = obj;
- }
-
- public boolean setSafe(int index, long value) {
- set(index, value);
- return true;
- }
-
- protected void set(int index, ObjectHolder holder) {
- set(index, holder.obj);
- }
-
- public boolean setSafe(int index, ObjectHolder holder){
- set(index, holder);
- return true;
- }
-
- @Override
- public void setValueCount(int valueCount) {
- count = valueCount;
- }
-
- @Override
- public void reset() {
- count = 0;
- maxCount = 0;
- objectArrayList = new ArrayList<>();
- addNewArray();
- }
-
- @Override
- public void generateTestData(int values) {
- }
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- // NoOp
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryException {
- addNewArray();
- }
-
- public void allocateNew(int valueCount) throws OutOfMemoryException {
- while (maxCount < valueCount) {
- addNewArray();
- }
- }
-
- @Override
- public boolean allocateNewSafe() {
- allocateNew();
- return true;
- }
-
- @Override
- public int getBufferSize() {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public void close() {
- clear();
- }
-
- @Override
- public void clear() {
- objectArrayList.clear();
- maxCount = 0;
- count = 0;
- }
-
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- @Override
- public TransferPair getTransferPair(BufferAllocator allocator) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- @Override
- public int getValueCapacity() {
- return maxCount;
- }
-
- @Override
- public Accessor getAccessor() {
- return accessor;
- }
-
- @Override
- public ArrowBuf[] getBuffers(boolean clear) {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
-// @Override
-// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-// throw new UnsupportedOperationException("ObjectVector does not support this");
-// }
-//
-// @Override
-// public UserBitShared.SerializedField getMetadata() {
-// throw new UnsupportedOperationException("ObjectVector does not support this");
-// }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- @Override
- public Iterator<ValueVector> iterator() {
- throw new UnsupportedOperationException("ObjectVector does not support this");
- }
-
- public final class Accessor extends BaseAccessor {
- @Override
- public Object getObject(int index) {
- int listOffset = index / allocationSize;
- if (listOffset >= objectArrayList.size()) {
- addNewArray();
- }
- return objectArrayList.get(listOffset)[index % allocationSize];
- }
-
- @Override
- public int getValueCount() {
- return count;
- }
-
- public Object get(int index) {
- return getObject(index);
- }
-
- public void get(int index, ObjectHolder holder){
- holder.obj = getObject(index);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
deleted file mode 100644
index 61ce285..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.math.BigDecimal;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.holders.BigIntHolder;
-import org.apache.arrow.vector.holders.BitHolder;
-import org.apache.arrow.vector.holders.DateHolder;
-import org.apache.arrow.vector.holders.Decimal18Holder;
-import org.apache.arrow.vector.holders.Decimal28SparseHolder;
-import org.apache.arrow.vector.holders.Decimal38SparseHolder;
-import org.apache.arrow.vector.holders.Decimal9Holder;
-import org.apache.arrow.vector.holders.Float4Holder;
-import org.apache.arrow.vector.holders.Float8Holder;
-import org.apache.arrow.vector.holders.IntHolder;
-import org.apache.arrow.vector.holders.IntervalDayHolder;
-import org.apache.arrow.vector.holders.IntervalYearHolder;
-import org.apache.arrow.vector.holders.NullableBitHolder;
-import org.apache.arrow.vector.holders.TimeHolder;
-import org.apache.arrow.vector.holders.TimeStampHolder;
-import org.apache.arrow.vector.holders.VarCharHolder;
-import org.apache.arrow.vector.util.DecimalUtility;
-
-import com.google.common.base.Charsets;
-
-
-public class ValueHolderHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
-
- public static IntHolder getIntHolder(int value) {
- IntHolder holder = new IntHolder();
- holder.value = value;
-
- return holder;
- }
-
- public static BigIntHolder getBigIntHolder(long value) {
- BigIntHolder holder = new BigIntHolder();
- holder.value = value;
-
- return holder;
- }
-
- public static Float4Holder getFloat4Holder(float value) {
- Float4Holder holder = new Float4Holder();
- holder.value = value;
-
- return holder;
- }
-
- public static Float8Holder getFloat8Holder(double value) {
- Float8Holder holder = new Float8Holder();
- holder.value = value;
-
- return holder;
- }
-
- public static DateHolder getDateHolder(long value) {
- DateHolder holder = new DateHolder();
- holder.value = value;
- return holder;
- }
-
- public static TimeHolder getTimeHolder(int value) {
- TimeHolder holder = new TimeHolder();
- holder.value = value;
- return holder;
- }
-
- public static TimeStampHolder getTimeStampHolder(long value) {
- TimeStampHolder holder = new TimeStampHolder();
- holder.value = value;
- return holder;
- }
-
- public static BitHolder getBitHolder(int value) {
- BitHolder holder = new BitHolder();
- holder.value = value;
-
- return holder;
- }
-
- public static NullableBitHolder getNullableBitHolder(boolean isNull, int value) {
- NullableBitHolder holder = new NullableBitHolder();
- holder.isSet = isNull? 0 : 1;
- if (! isNull) {
- holder.value = value;
- }
-
- return holder;
- }
-
- public static VarCharHolder getVarCharHolder(ArrowBuf buf, String s){
- VarCharHolder vch = new VarCharHolder();
-
- byte[] b = s.getBytes(Charsets.UTF_8);
- vch.start = 0;
- vch.end = b.length;
- vch.buffer = buf.reallocIfNeeded(b.length);
- vch.buffer.setBytes(0, b);
- return vch;
- }
-
- public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
- VarCharHolder vch = new VarCharHolder();
-
- byte[] b = s.getBytes(Charsets.UTF_8);
- vch.start = 0;
- vch.end = b.length;
- vch.buffer = a.buffer(b.length); //
- vch.buffer.setBytes(0, b);
- return vch;
- }
-
-
- public static IntervalYearHolder getIntervalYearHolder(int intervalYear) {
- IntervalYearHolder holder = new IntervalYearHolder();
-
- holder.value = intervalYear;
- return holder;
- }
-
- public static IntervalDayHolder getIntervalDayHolder(int days, int millis) {
- IntervalDayHolder dch = new IntervalDayHolder();
-
- dch.days = days;
- dch.milliseconds = millis;
- return dch;
- }
-
- public static Decimal9Holder getDecimal9Holder(int decimal, int scale, int precision) {
- Decimal9Holder dch = new Decimal9Holder();
-
- dch.scale = scale;
- dch.precision = precision;
- dch.value = decimal;
-
- return dch;
- }
-
- public static Decimal18Holder getDecimal18Holder(long decimal, int scale, int precision) {
- Decimal18Holder dch = new Decimal18Holder();
-
- dch.scale = scale;
- dch.precision = precision;
- dch.value = decimal;
-
- return dch;
- }
-
- public static Decimal28SparseHolder getDecimal28Holder(ArrowBuf buf, String decimal) {
-
- Decimal28SparseHolder dch = new Decimal28SparseHolder();
-
- BigDecimal bigDecimal = new BigDecimal(decimal);
-
- dch.scale = bigDecimal.scale();
- dch.precision = bigDecimal.precision();
- Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
- dch.start = 0;
- dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE);
- DecimalUtility
- .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
-
- return dch;
- }
-
- public static Decimal38SparseHolder getDecimal38Holder(ArrowBuf buf, String decimal) {
-
- Decimal38SparseHolder dch = new Decimal38SparseHolder();
-
- BigDecimal bigDecimal = new BigDecimal(decimal);
-
- dch.scale = bigDecimal.scale();
- dch.precision = bigDecimal.precision();
- Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
- dch.start = 0;
- dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE);
- DecimalUtility
- .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
-
- return dch;
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index a170c59..35321c9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -24,8 +24,9 @@ import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.util.TransferPair;
+import org.apache.arrow.vector.types.pojo.Field;
/**
* An abstraction that is used to store a sequence of values in an individual column.
@@ -33,8 +34,7 @@ import org.apache.arrow.vector.util.TransferPair;
* A {@link ValueVector value vector} stores underlying data in-memory in a columnar fashion that is compact and
* efficient. The column whose data is stored, is referred by {@link #getField()}.
*
- * A vector when instantiated, relies on a {@link org.apache.drill.exec.record.DeadBuf dead buffer}. It is important
- * that vector is allocated before attempting to read or write.
+ * It is important that vector is allocated before attempting to read or write.
*
* There are a few "rules" around vectors:
*
@@ -94,7 +94,9 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
/**
* Get information about how this field is materialized.
*/
- MaterializedField getField();
+ Field getField();
+
+ MinorType getMinorType();
/**
* Returns a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
deleted file mode 100644
index fdad99a..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector;
-
-import java.util.Collection;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.MajorType;
-
-public class VectorDescriptor {
- private static final String DEFAULT_NAME = "NONE";
-
- private final MaterializedField field;
-
- public VectorDescriptor(final MajorType type) {
- this(DEFAULT_NAME, type);
- }
-
- public VectorDescriptor(final String name, final MajorType type) {
- this(MaterializedField.create(name, type));
- }
-
- public VectorDescriptor(final MaterializedField field) {
- this.field = Preconditions.checkNotNull(field, "field cannot be null");
- }
-
- public MaterializedField getField() {
- return field;
- }
-
- public MajorType getType() {
- return field.getType();
- }
-
- public String getName() {
- return field.getLastName();
- }
-
- public Collection<MaterializedField> getChildren() {
- return field.getChildren();
- }
-
- public boolean hasName() {
- return getName() != DEFAULT_NAME;
- }
-
- public VectorDescriptor withName(final String name) {
- return new VectorDescriptor(field.withPath(name));
- }
-
- public VectorDescriptor withType(final MajorType type) {
- return new VectorDescriptor(field.withType(type));
- }
-
- public static VectorDescriptor create(final String name, final MajorType type) {
- return new VectorDescriptor(name, type);
- }
-
- public static VectorDescriptor create(final MajorType type) {
- return new VectorDescriptor(type);
- }
-
- public static VectorDescriptor create(final MaterializedField field) {
- return new VectorDescriptor(field);
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index c94e8d1..705a24b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -17,18 +17,20 @@
*/
package org.apache.arrow.vector;
+import com.google.flatbuffers.FlatBufferBuilder;
import io.netty.buffer.ArrowBuf;
import java.util.Collections;
import java.util.Iterator;
+import org.apache.arrow.flatbuf.Type;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.TransferPair;
import com.google.common.collect.Iterators;
@@ -36,7 +38,7 @@ import com.google.common.collect.Iterators;
public class ZeroVector implements ValueVector {
public final static ZeroVector INSTANCE = new ZeroVector();
- private final MaterializedField field = MaterializedField.create("[DEFAULT]", Types.required(MinorType.LATE));
+ private final String name = "[DEFAULT]";
private final TransferPair defaultPair = new TransferPair() {
@Override
@@ -91,24 +93,21 @@ public class ZeroVector implements ValueVector {
public void clear() { }
@Override
- public MaterializedField getField() {
- return field;
+ public Field getField() {
+ return new Field(name, true, new Null(), null);
}
@Override
+ public MinorType getMinorType() {
+ return MinorType.NULL;
+ }
+
+
+ @Override
public TransferPair getTransferPair(BufferAllocator allocator) {
return defaultPair;
}
-// @Override
-// public UserBitShared.SerializedField getMetadata() {
-// return getField()
-// .getAsBuilder()
-// .setBufferLength(getBufferSize())
-// .setValueCount(getAccessor().getValueCount())
-// .build();
-// }
-
@Override
public Iterator<ValueVector> iterator() {
return Collections.emptyIterator();
@@ -176,7 +175,4 @@ public class ZeroVector implements ValueVector {
public FieldReader getReader() {
return NullReader.INSTANCE;
}
-
-// @Override
-// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { }
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 9fae238..ed77975 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -21,12 +21,10 @@ import java.util.Collection;
import javax.annotation.Nullable;
+import org.apache.arrow.flatbuf.Field;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.util.CallBack;
@@ -43,12 +41,12 @@ import com.google.common.collect.Sets;
public abstract class AbstractContainerVector implements ValueVector {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
- protected MaterializedField field;
+ protected final String name;
protected final BufferAllocator allocator;
protected final CallBack callBack;
- protected AbstractContainerVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- this.field = Preconditions.checkNotNull(field);
+ protected AbstractContainerVector(String name, BufferAllocator allocator, CallBack callBack) {
+ this.name = name;
this.allocator = allocator;
this.callBack = callBack;
}
@@ -65,14 +63,6 @@ public abstract class AbstractContainerVector implements ValueVector {
}
/**
- * Returns the field definition of this instance.
- */
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- /**
* Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
*/
public ValueVector getChild(String name) {
@@ -80,19 +70,6 @@ public abstract class AbstractContainerVector implements ValueVector {
}
/**
- * Returns a sequence of field names in the order that they show up in the schema.
- */
- protected Collection<String> getChildFieldNames() {
- return Sets.newLinkedHashSet(Iterables.transform(field.getChildren(), new Function<MaterializedField, String>() {
- @Nullable
- @Override
- public String apply(MaterializedField field) {
- return Preconditions.checkNotNull(field).getLastName();
- }
- }));
- }
-
- /**
* Clears out all underlying child vectors.
*/
@Override
@@ -109,22 +86,6 @@ public abstract class AbstractContainerVector implements ValueVector {
throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Arrow doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
}
- MajorType getLastPathType() {
- if((this.getField().getType().getMinorType() == MinorType.LIST &&
- this.getField().getType().getMode() == DataMode.REPEATED)) { // Use Repeated scalar type instead of Required List.
- VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
- ValueVector v = vord.vector;
- if (! (v instanceof AbstractContainerVector)) {
- return v.getField().getType();
- }
- } else if (this.getField().getType().getMinorType() == MinorType.MAP &&
- this.getField().getType().getMode() == DataMode.REPEATED) { // Use Required Map
- return new MajorType(MinorType.MAP, DataMode.REQUIRED);
- }
-
- return this.getField().getType();
- }
-
protected boolean supportsDirectRead() {
return false;
}
@@ -133,7 +94,7 @@ public abstract class AbstractContainerVector implements ValueVector {
public abstract int size();
// add a new vector with the input MajorType or return the existing vector if we already added one with the same type
- public abstract <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz);
+ public abstract <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale);
// return the child vector with the input name
public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
[3/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead
of MaterializedField
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index de6ae82..5964f80 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -17,17 +17,17 @@
*/
package org.apache.arrow.vector.complex;
+import com.google.common.collect.ImmutableList;
import io.netty.buffer.ArrowBuf;
-import java.util.Collection;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.arrow.flatbuf.Field;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.MapWithOrdinal;
@@ -43,17 +43,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
// Maintains a map with key as field name and value is the vector itself
private final MapWithOrdinal<String, ValueVector> vectors = new MapWithOrdinal<>();
- protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- super(field.clone(), allocator, callBack);
- MaterializedField clonedField = field.clone();
- // create the hierarchy of the child vectors based on the materialized field
- for (MaterializedField child : clonedField.getChildren()) {
- if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
- final String fieldName = child.getLastName();
- final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
- putVector(fieldName, v);
- }
- }
+ protected AbstractMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+ super(name, allocator, callBack);
}
@Override
@@ -109,8 +100,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
* </li>
* </ul>
*
- * @param name name of the field
- * @param type type of the field
+ * @param name the name of the field
+ * @param minorType the minorType for the vector
* @param clazz class of expected vector type
* @param <T> class type of expected vector type
* @throws java.lang.IllegalStateException raised if there is a hard schema change
@@ -118,7 +109,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
* @return resultant {@link org.apache.arrow.vector.ValueVector}
*/
@Override
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+ public <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
final ValueVector existing = getChild(name);
boolean create = false;
if (existing == null) {
@@ -130,7 +121,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
create = true;
}
if (create) {
- final T vector = (T) BasicTypeHelper.getNewVector(name, allocator, type, callBack);
+ final T vector = (T) minorType.getNewVector(name, allocator, callBack, precisionScale);
putChild(name, vector);
if (callBack!=null) {
callBack.doWork();
@@ -177,7 +168,6 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
*/
protected void putChild(String name, ValueVector vector) {
putVector(name, vector);
- field.addChild(vector.getField());
}
/**
@@ -199,8 +189,21 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
/**
* Returns a sequence of underlying child vectors.
*/
- protected Collection<ValueVector> getChildren() {
- return vectors.values();
+ protected List<ValueVector> getChildren() {
+ int size = vectors.size();
+ List<ValueVector> children = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ children.add(vectors.getByOrdinal(i));
+ }
+ return children;
+ }
+
+ protected List<String> getChildFieldNames() {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ for (ValueVector child : getChildren()) {
+ builder.add(child.getField().getName());
+ }
+ return builder.build();
}
/**
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 6518897..4226274 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -22,22 +22,18 @@ import io.netty.buffer.ArrowBuf;
import java.util.Collections;
import java.util.Iterator;
+import org.apache.arrow.flatbuf.Type;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.AddOrGetResult;
import org.apache.arrow.vector.BaseValueVector;
import org.apache.arrow.vector.UInt4Vector;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
-import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import com.google.common.base.Preconditions;
import com.google.common.collect.ObjectArrays;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
@@ -45,19 +41,16 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
public final static String OFFSETS_VECTOR_NAME = "$offsets$";
public final static String DATA_VECTOR_NAME = "$data$";
- public final static MaterializedField OFFSETS_FIELD =
- MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
-
protected final UInt4Vector offsets;
protected ValueVector vector;
- protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
- this(field, allocator, DEFAULT_DATA_VECTOR);
+ protected BaseRepeatedValueVector(String name, BufferAllocator allocator) {
+ this(name, allocator, DEFAULT_DATA_VECTOR);
}
- protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
- super(field, allocator);
- this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
+ protected BaseRepeatedValueVector(String name, BufferAllocator allocator, ValueVector vector) {
+ super(name, allocator);
+ this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
}
@@ -109,13 +102,6 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
return Math.min(vector.getValueCapacity(), offsetValueCapacity);
}
-// @Override
-// protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
-// return super.getMetadataBuilder()
-// .addChild(offsets.getMetadata())
-// .addChild(vector.getMetadata());
-// }
-
@Override
public int getBufferSize() {
if (getAccessor().getValueCount() == 0) {
@@ -157,47 +143,24 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
return buffers;
}
-// @Override
-// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-// final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
-// offsets.load(offsetMetadata, buffer);
-//
-// final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
-// if (getDataVector() == DEFAULT_DATA_VECTOR) {
-// addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
-// }
-//
-// final int offsetLength = offsetMetadata.getBufferLength();
-// final int vectorLength = vectorMetadata.getBufferLength();
-// vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
-// }
-
/**
* Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
- *
- * @see {@link ContainerVectorLike#size}
*/
- @Override
public int size() {
return vector == DEFAULT_DATA_VECTOR ? 0:1;
}
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
boolean created = false;
- if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != MinorType.LATE) {
- final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
- vector = BasicTypeHelper.getNewVector(field, allocator);
+ if (vector instanceof ZeroVector) {
+ vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, null);
// returned vector must have the same field
- assert field.equals(vector.getField());
- getField().addChild(field);
created = true;
}
- final MajorType actual = vector.getField().getType();
- if (!actual.equals(descriptor.getType())) {
+ if (vector.getField().getType().getTypeType() != minorType.getType().getTypeType()) {
final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
- descriptor.getType(), actual);
+ Type.name(minorType.getType().getTypeType()), Type.name(vector.getField().getType().getTypeType()));
throw new SchemaChangeRuntimeException(msg);
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
deleted file mode 100644
index 655b55a..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-
-/**
- * A mix-in used for introducing container vector-like behaviour.
- */
-public interface ContainerVectorLike {
-
- /**
- * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
- *
- * @param descriptor vector descriptor
- * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
- * @throws org.apache.arrow.vector.util.SchemaChangeRuntimeException
- * if schema change is not permissible between the given and existing data vector types.
- */
- <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
-
- /**
- * Returns the number of child vectors in this container vector-like instance.
- */
- int size();
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 3e60c76..c6c6b09 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -18,6 +18,8 @@
******************************************************************************/
package org.apache.arrow.vector.complex;
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
import io.netty.buffer.ArrowBuf;
import java.util.List;
@@ -28,17 +30,14 @@ import org.apache.arrow.vector.AddOrGetResult;
import org.apache.arrow.vector.UInt1Vector;
import org.apache.arrow.vector.UInt4Vector;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
import org.apache.arrow.vector.ZeroVector;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.complex.impl.UnionListReader;
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;
@@ -55,11 +54,10 @@ public class ListVector extends BaseRepeatedValueVector {
private UnionListReader reader;
private CallBack callBack;
- public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- super(field, allocator);
- this.bits = new UInt1Vector(MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED)), allocator);
+ public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
+ super(name, allocator);
+ this.bits = new UInt1Vector("$bits$", allocator);
offsets = getOffsetVector();
- this.field.addChild(getDataVector().getField());
this.writer = new UnionListWriter(this);
this.reader = new UnionListReader(this);
this.callBack = callBack;
@@ -75,15 +73,6 @@ public class ListVector extends BaseRepeatedValueVector {
bits.allocateNewSafe();
}
- public void transferTo(ListVector target) {
- offsets.makeTransferPair(target.offsets).transfer();
- bits.makeTransferPair(target.bits).transfer();
- if (target.getDataVector() instanceof ZeroVector) {
- target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
- }
- getDataVector().makeTransferPair(target.getDataVector()).transfer();
- }
-
public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
copyFrom(inIndex, outIndex, from);
}
@@ -103,7 +92,7 @@ public class ListVector extends BaseRepeatedValueVector {
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new TransferImpl(field.withPath(ref), allocator);
+ return new TransferImpl(ref, allocator);
}
@Override
@@ -114,20 +103,28 @@ public class ListVector extends BaseRepeatedValueVector {
private class TransferImpl implements TransferPair {
ListVector to;
+ TransferPair pairs[] = new TransferPair[3];
- public TransferImpl(MaterializedField field, BufferAllocator allocator) {
- to = new ListVector(field, allocator, null);
- to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ public TransferImpl(String name, BufferAllocator allocator) {
+ this(new ListVector(name, allocator, null));
}
public TransferImpl(ListVector to) {
this.to = to;
- to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ to.addOrGetVector(vector.getMinorType());
+ pairs[0] = offsets.makeTransferPair(to.offsets);
+ pairs[1] = bits.makeTransferPair(to.bits);
+ if (to.getDataVector() instanceof ZeroVector) {
+ to.addOrGetVector(vector.getMinorType());
+ }
+ pairs[2] = getDataVector().makeTransferPair(to.getDataVector());
}
@Override
public void transfer() {
- transferTo(to);
+ for (TransferPair pair : pairs) {
+ pair.transfer();
+ }
}
@Override
@@ -190,17 +187,8 @@ public class ListVector extends BaseRepeatedValueVector {
return success;
}
-// @Override
-// protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
-// return getField().getAsBuilder()
-// .setValueCount(getAccessor().getValueCount())
-// .setBufferLength(getBufferSize())
-// .addChild(offsets.getMetadata())
-// .addChild(bits.getMetadata())
-// .addChild(vector.getMetadata());
-// }
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
+ AddOrGetResult<T> result = super.addOrGetVector(minorType);
reader = new UnionListReader(this);
return result;
}
@@ -214,6 +202,17 @@ public class ListVector extends BaseRepeatedValueVector {
}
@Override
+ public Field getField() {
+ return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
+ ImmutableList.of(getDataVector().getField()));
+ }
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.LIST;
+ }
+
+ @Override
public void clear() {
offsets.clear();
vector.clear();
@@ -235,28 +234,8 @@ public class ListVector extends BaseRepeatedValueVector {
return buffers;
}
-// @Override
-// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-// final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
-// offsets.load(offsetMetadata, buffer);
-//
-// final int offsetLength = offsetMetadata.getBufferLength();
-// final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
-// final int bitLength = bitMetadata.getBufferLength();
-// bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
-//
-// final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
-// if (getDataVector() == DEFAULT_DATA_VECTOR) {
-// addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
-// }
-//
-// final int vectorLength = vectorMetadata.getBufferLength();
-// vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
-// }
-
public UnionVector promoteToUnion() {
- MaterializedField newField = MaterializedField.create(getField().getPath(), new MajorType(MinorType.UNION, DataMode.OPTIONAL));
- UnionVector vector = new UnionVector(newField, allocator, null);
+ UnionVector vector = new UnionVector(name, allocator, null);
replaceDataVector(vector);
reader = new UnionListReader(this);
return vector;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index cc0953a..0cb613e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -19,8 +19,10 @@ package org.apache.arrow.vector.complex;
import io.netty.buffer.ArrowBuf;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
@@ -28,14 +30,13 @@ import javax.annotation.Nullable;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.BaseValueVector;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector.MapSingleCopier;
import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringHashMap;
import org.apache.arrow.vector.util.TransferPair;
@@ -47,19 +48,13 @@ import com.google.common.primitives.Ints;
public class MapVector extends AbstractMapVector {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
- public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
-
private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
int valueCount;
- public MapVector(String path, BufferAllocator allocator, CallBack callBack){
- this(MaterializedField.create(path, TYPE), allocator, callBack);
- }
-
- public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- super(field, allocator, callBack);
+ public MapVector(String name, BufferAllocator allocator, CallBack callBack){
+ super(name, allocator, callBack);
}
@Override
@@ -69,7 +64,6 @@ public class MapVector extends AbstractMapVector {
}
transient private MapTransferPair ephPair;
- transient private MapSingleCopier ephPair2;
public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
if(ephPair == null || ephPair.from != from) {
@@ -78,13 +72,6 @@ public class MapVector extends AbstractMapVector {
ephPair.copyValueSafe(fromIndex, thisIndex);
}
- public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
- if(ephPair2 == null || ephPair2.from != from) {
- ephPair2 = from.makeSingularCopier(this);
- }
- ephPair2.copySafe(fromSubIndex, thisIndex);
- }
-
@Override
protected boolean supportsDirectRead() {
return true;
@@ -139,7 +126,7 @@ public class MapVector extends AbstractMapVector {
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return new MapTransferPair(this, getField().getPath(), allocator);
+ return new MapTransferPair(this, name, allocator);
}
@Override
@@ -157,8 +144,8 @@ public class MapVector extends AbstractMapVector {
private final MapVector from;
private final MapVector to;
- public MapTransferPair(MapVector from, String path, BufferAllocator allocator) {
- this(from, new MapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
+ public MapTransferPair(MapVector from, String name, BufferAllocator allocator) {
+ this(from, new MapVector(name, allocator, from.callBack), false);
}
public MapTransferPair(MapVector from, MapVector to) {
@@ -170,7 +157,6 @@ public class MapVector extends AbstractMapVector {
this.to = to;
this.pairs = new TransferPair[from.size()];
this.to.ephPair = null;
- this.to.ephPair2 = null;
int i = 0;
ValueVector vector;
@@ -189,7 +175,7 @@ public class MapVector extends AbstractMapVector {
// (This is similar to what happens in ScanBatch where the children cannot be added till they are
// read). To take care of this, we ensure that the hashCode of the MaterializedField does not
// include the hashCode of the children but is based only on MaterializedField$key.
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+ final ValueVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass());
if (allocate && to.size() != preSize) {
newVector.allocateNew();
}
@@ -251,46 +237,6 @@ public class MapVector extends AbstractMapVector {
return accessor;
}
-// @Override
-// public void load(SerializedField metadata, DrillBuf buf) {
-// final List<SerializedField> fields = metadata.getChildList();
-// valueCount = metadata.getValueCount();
-//
-// int bufOffset = 0;
-// for (final SerializedField child : fields) {
-// final MaterializedField fieldDef = SerializedFieldHelper.create(child);
-//
-// ValueVector vector = getChild(fieldDef.getLastName());
-// if (vector == null) {
-// if we arrive here, we didn't have a matching vector.
-// vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
-// putChild(fieldDef.getLastName(), vector);
-// }
-// if (child.getValueCount() == 0) {
-// vector.clear();
-// } else {
-// vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
-// }
-// bufOffset += child.getBufferLength();
-// }
-//
-// assert bufOffset == buf.capacity();
-// }
-//
-// @Override
-// public SerializedField getMetadata() {
-// SerializedField.Builder b = getField() //
-// .getAsBuilder() //
-// .setBufferLength(getBufferSize()) //
-// .setValueCount(valueCount);
-//
-//
-// for(ValueVector v : getChildren()) {
-// b.addChild(v.getMetadata());
-// }
-// return b.build();
-// }
-
@Override
public Mutator getMutator() {
return mutator;
@@ -303,13 +249,6 @@ public class MapVector extends AbstractMapVector {
Map<String, Object> vv = new JsonStringHashMap<>();
for (String child:getChildFieldNames()) {
ValueVector v = getChild(child);
- // TODO(DRILL-4001): Resolve this hack:
- // The index/value count check in the following if statement is a hack
- // to work around the current fact that RecordBatchLoader.load and
- // MapVector.load leave child vectors with a length of zero (as opposed
- // to matching the lengths of siblings and the parent map vector)
- // because they don't remove (or set the lengths of) vectors from
- // previous batches that aren't in the current batch.
if (v != null && index < v.getAccessor().getValueCount()) {
Object value = v.getAccessor().getObject(index);
if (value != null) {
@@ -361,6 +300,20 @@ public class MapVector extends AbstractMapVector {
}
@Override
+ public Field getField() {
+ List<Field> children = new ArrayList<>();
+ for (ValueVector child : getChildren()) {
+ children.add(child.getField());
+ }
+ return new Field(name, false, Tuple.INSTANCE, children);
+ }
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.MAP;
+ }
+
+ @Override
public void close() {
final Collection<ValueVector> vectors = getChildren();
for (final ValueVector v : vectors) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
deleted file mode 100644
index f337f9c..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-import org.apache.arrow.vector.complex.impl.NullReader;
-import org.apache.arrow.vector.complex.impl.RepeatedListReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.holders.RepeatedListHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class RepeatedListVector extends AbstractContainerVector
- implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
-
- public final static MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
- private final RepeatedListReaderImpl reader = new RepeatedListReaderImpl(null, this);
- final DelegateRepeatedVector delegate;
-
- protected static class DelegateRepeatedVector extends BaseRepeatedValueVector {
-
- private final RepeatedListAccessor accessor = new RepeatedListAccessor();
- private final RepeatedListMutator mutator = new RepeatedListMutator();
- private final EmptyValuePopulator emptyPopulator;
- private transient DelegateTransferPair ephPair;
-
- public class RepeatedListAccessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
-
- @Override
- public Object getObject(int index) {
- final List<Object> list = new JsonStringArrayList<>();
- final int start = offsets.getAccessor().get(index);
- final int until = offsets.getAccessor().get(index+1);
- for (int i = start; i < until; i++) {
- list.add(vector.getAccessor().getObject(i));
- }
- return list;
- }
-
- public void get(int index, RepeatedListHolder holder) {
- assert index <= getValueCapacity();
- holder.start = getOffsetVector().getAccessor().get(index);
- holder.end = getOffsetVector().getAccessor().get(index+1);
- }
-
- public void get(int index, ComplexHolder holder) {
- final FieldReader reader = getReader();
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- public void get(int index, int arrayIndex, ComplexHolder holder) {
- final RepeatedListHolder listHolder = new RepeatedListHolder();
- get(index, listHolder);
- int offset = listHolder.start + arrayIndex;
- if (offset >= listHolder.end) {
- holder.reader = NullReader.INSTANCE;
- } else {
- FieldReader r = getDataVector().getReader();
- r.setPosition(offset);
- holder.reader = r;
- }
- }
- }
-
- public class RepeatedListMutator extends BaseRepeatedValueVector.BaseRepeatedMutator {
-
- public int add(int index) {
- final int curEnd = getOffsetVector().getAccessor().get(index+1);
- getOffsetVector().getMutator().setSafe(index + 1, curEnd + 1);
- return curEnd;
- }
-
- @Override
- public void startNewValue(int index) {
- emptyPopulator.populate(index+1);
- super.startNewValue(index);
- }
-
- @Override
- public void setValueCount(int valueCount) {
- emptyPopulator.populate(valueCount);
- super.setValueCount(valueCount);
- }
- }
-
-
- public class DelegateTransferPair implements TransferPair {
- private final DelegateRepeatedVector target;
- private final TransferPair[] children;
-
- public DelegateTransferPair(DelegateRepeatedVector target) {
- this.target = Preconditions.checkNotNull(target);
- if (target.getDataVector() == DEFAULT_DATA_VECTOR) {
- target.addOrGetVector(VectorDescriptor.create(getDataVector().getField()));
- target.getDataVector().allocateNew();
- }
- this.children = new TransferPair[] {
- getOffsetVector().makeTransferPair(target.getOffsetVector()),
- getDataVector().makeTransferPair(target.getDataVector())
- };
- }
-
- @Override
- public void transfer() {
- for (TransferPair child:children) {
- child.transfer();
- }
- }
-
- @Override
- public ValueVector getTo() {
- return target;
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- target.allocateNew();
- for (int i = 0; i < length; i++) {
- copyValueSafe(startIndex + i, i);
- }
- }
-
- @Override
- public void copyValueSafe(int srcIndex, int destIndex) {
- final RepeatedListHolder holder = new RepeatedListHolder();
- getAccessor().get(srcIndex, holder);
- target.emptyPopulator.populate(destIndex+1);
- final TransferPair vectorTransfer = children[1];
- int newIndex = target.getOffsetVector().getAccessor().get(destIndex);
- //todo: make this a bulk copy.
- for (int i = holder.start; i < holder.end; i++, newIndex++) {
- vectorTransfer.copyValueSafe(i, newIndex);
- }
- target.getOffsetVector().getMutator().setSafe(destIndex + 1, newIndex);
- }
- }
-
- public DelegateRepeatedVector(String path, BufferAllocator allocator) {
- this(MaterializedField.create(path, TYPE), allocator);
- }
-
- public DelegateRepeatedVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- emptyPopulator = new EmptyValuePopulator(getOffsetVector());
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryException {
- if (!allocateNewSafe()) {
- throw new OutOfMemoryException();
- }
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return makeTransferPair(new DelegateRepeatedVector(ref, allocator));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector target) {
- return new DelegateTransferPair(DelegateRepeatedVector.class.cast(target));
- }
-
- @Override
- public RepeatedListAccessor getAccessor() {
- return accessor;
- }
-
- @Override
- public RepeatedListMutator getMutator() {
- return mutator;
- }
-
- @Override
- public FieldReader getReader() {
- throw new UnsupportedOperationException();
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) {
- if(ephPair == null || ephPair.target != from) {
- ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this));
- }
- ephPair.copyValueSafe(fromIndex, thisIndex);
- }
-
- }
-
- protected class RepeatedListTransferPair implements TransferPair {
- private final TransferPair delegate;
-
- public RepeatedListTransferPair(TransferPair delegate) {
- this.delegate = delegate;
- }
-
- public void transfer() {
- delegate.transfer();
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- delegate.splitAndTransfer(startIndex, length);
- }
-
- @Override
- public ValueVector getTo() {
- final DelegateRepeatedVector delegateVector = DelegateRepeatedVector.class.cast(delegate.getTo());
- return new RepeatedListVector(getField(), allocator, callBack, delegateVector);
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- delegate.copyValueSafe(from, to);
- }
- }
-
- public RepeatedListVector(String path, BufferAllocator allocator, CallBack callBack) {
- this(MaterializedField.create(path, TYPE), allocator, callBack);
- }
-
- public RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- this(field, allocator, callBack, new DelegateRepeatedVector(field, allocator));
- }
-
- protected RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack, DelegateRepeatedVector delegate) {
- super(field, allocator, callBack);
- this.delegate = Preconditions.checkNotNull(delegate);
-
- final List<MaterializedField> children = Lists.newArrayList(field.getChildren());
- final int childSize = children.size();
- assert childSize < 3;
- final boolean hasChild = childSize > 0;
- if (hasChild) {
- // the last field is data field
- final MaterializedField child = children.get(childSize-1);
- addOrGetVector(VectorDescriptor.create(child));
- }
- }
-
-
- @Override
- public RepeatedListReaderImpl getReader() {
- return reader;
- }
-
- @Override
- public DelegateRepeatedVector.RepeatedListAccessor getAccessor() {
- return delegate.getAccessor();
- }
-
- @Override
- public DelegateRepeatedVector.RepeatedListMutator getMutator() {
- return delegate.getMutator();
- }
-
- @Override
- public UInt4Vector getOffsetVector() {
- return delegate.getOffsetVector();
- }
-
- @Override
- public ValueVector getDataVector() {
- return delegate.getDataVector();
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryException {
- delegate.allocateNew();
- }
-
- @Override
- public boolean allocateNewSafe() {
- return delegate.allocateNewSafe();
- }
-
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- final AddOrGetResult<T> result = delegate.addOrGetVector(descriptor);
- if (result.isCreated() && callBack != null) {
- callBack.doWork();
- }
- return result;
- }
-
- @Override
- public int size() {
- return delegate.size();
- }
-
- @Override
- public int getBufferSize() {
- return delegate.getBufferSize();
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- return delegate.getBufferSizeFor(valueCount);
- }
-
- @Override
- public void close() {
- delegate.close();
- }
-
- @Override
- public void clear() {
- delegate.clear();
- }
-
- @Override
- public TransferPair getTransferPair(BufferAllocator allocator) {
- return new RepeatedListTransferPair(delegate.getTransferPair(allocator));
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new RepeatedListTransferPair(delegate.getTransferPair(ref, allocator));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- final RepeatedListVector target = RepeatedListVector.class.cast(to);
- return new RepeatedListTransferPair(delegate.makeTransferPair(target.delegate));
- }
-
- @Override
- public int getValueCapacity() {
- return delegate.getValueCapacity();
- }
-
- @Override
- public ArrowBuf[] getBuffers(boolean clear) {
- return delegate.getBuffers(clear);
- }
-
-
-// @Override
-// public void load(SerializedField metadata, DrillBuf buf) {
-// delegate.load(metadata, buf);
-// }
-
-// @Override
-// public SerializedField getMetadata() {
-// return delegate.getMetadata();
-// }
-
- @Override
- public Iterator<ValueVector> iterator() {
- return delegate.iterator();
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- delegate.setInitialCapacity(numRecords);
- }
-
- /**
- * @deprecated
- * prefer using {@link #addOrGetVector(org.apache.arrow.vector.VectorDescriptor)} instead.
- */
- @Override
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
- final AddOrGetResult<T> result = addOrGetVector(VectorDescriptor.create(type));
- return result.getVector();
- }
-
- @Override
- public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
- if (name != null) {
- return null;
- }
- return typeify(delegate.getDataVector(), clazz);
- }
-
- @Override
- public void allocateNew(int valueCount, int innerValueCount) {
- clear();
- getOffsetVector().allocateNew(valueCount + 1);
- getMutator().reset();
- }
-
- @Override
- public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
- if (name != null) {
- return null;
- }
- return new VectorWithOrdinal(delegate.getDataVector(), 0);
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) {
- delegate.copyFromSafe(fromIndex, thisIndex, from.delegate);
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
deleted file mode 100644
index 686414e..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
+++ /dev/null
@@ -1,584 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.AllocationHelper;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-import org.apache.arrow.vector.complex.impl.NullReader;
-import org.apache.arrow.vector.complex.impl.RepeatedMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-import org.apache.commons.lang3.ArrayUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-public class RepeatedMapVector extends AbstractMapVector
- implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
-
- public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.REPEATED);
-
- final UInt4Vector offsets; // offsets to start of each record (considering record indices are 0-indexed)
- private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
- private final RepeatedMapAccessor accessor = new RepeatedMapAccessor();
- private final Mutator mutator = new Mutator();
- private final EmptyValuePopulator emptyPopulator;
-
- public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- super(field, allocator, callBack);
- this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator);
- this.emptyPopulator = new EmptyValuePopulator(offsets);
- }
-
- @Override
- public UInt4Vector getOffsetVector() {
- return offsets;
- }
-
- @Override
- public ValueVector getDataVector() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- offsets.setInitialCapacity(numRecords + 1);
- for(final ValueVector v : (Iterable<ValueVector>) this) {
- v.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
- }
- }
-
- @Override
- public RepeatedMapReaderImpl getReader() {
- return reader;
- }
-
- @Override
- public void allocateNew(int groupCount, int innerValueCount) {
- clear();
- try {
- offsets.allocateNew(groupCount + 1);
- for (ValueVector v : getChildren()) {
- AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
- }
- } catch (OutOfMemoryException e){
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- public Iterator<String> fieldNameIterator() {
- return getChildFieldNames().iterator();
- }
-
- @Override
- public List<ValueVector> getPrimitiveVectors() {
- final List<ValueVector> primitiveVectors = super.getPrimitiveVectors();
- primitiveVectors.add(offsets);
- return primitiveVectors;
- }
-
- @Override
- public int getBufferSize() {
- if (getAccessor().getValueCount() == 0) {
- return 0;
- }
- long bufferSize = offsets.getBufferSize();
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSize();
- }
- return (int) bufferSize;
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- long bufferSize = 0;
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSizeFor(valueCount);
- }
-
- return (int) bufferSize;
- }
-
- @Override
- public void close() {
- offsets.close();
- super.close();
- }
-
- @Override
- public TransferPair getTransferPair(BufferAllocator allocator) {
- return new RepeatedMapTransferPair(this, getField().getPath(), allocator);
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new RepeatedMapTransferPair(this, (RepeatedMapVector)to);
- }
-
- MapSingleCopier makeSingularCopier(MapVector to) {
- return new MapSingleCopier(this, to);
- }
-
- protected static class MapSingleCopier {
- private final TransferPair[] pairs;
- public final RepeatedMapVector from;
-
- public MapSingleCopier(RepeatedMapVector from, MapVector to) {
- this.from = from;
- this.pairs = new TransferPair[from.size()];
-
- int i = 0;
- ValueVector vector;
- for (final String child:from.getChildFieldNames()) {
- int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (to.size() != preSize) {
- newVector.allocateNew();
- }
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
- public void copySafe(int fromSubIndex, int toIndex) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(fromSubIndex, toIndex);
- }
- }
- }
-
- public TransferPair getTransferPairToSingleMap(String reference, BufferAllocator allocator) {
- return new SingleMapTransferPair(this, reference, allocator);
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new RepeatedMapTransferPair(this, ref, allocator);
- }
-
- @Override
- public boolean allocateNewSafe() {
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- if (!offsets.allocateNewSafe()) {
- return false;
- }
- success = super.allocateNewSafe();
- } finally {
- if (!success) {
- clear();
- }
- }
- offsets.zeroVector();
- return success;
- }
-
- protected static class SingleMapTransferPair implements TransferPair {
- private final TransferPair[] pairs;
- private final RepeatedMapVector from;
- private final MapVector to;
- private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.REQUIRED);
-
- public SingleMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
- this(from, new MapVector(MaterializedField.create(path, MAP_TYPE), allocator, from.callBack), false);
- }
-
- public SingleMapTransferPair(RepeatedMapVector from, MapVector to) {
- this(from, to, true);
- }
-
- public SingleMapTransferPair(RepeatedMapVector from, MapVector to, boolean allocate) {
- this.from = from;
- this.to = to;
- this.pairs = new TransferPair[from.size()];
- int i = 0;
- ValueVector vector;
- for (final String child : from.getChildFieldNames()) {
- int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (allocate && to.size() != preSize) {
- newVector.allocateNew();
- }
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
-
- @Override
- public void transfer() {
- for (TransferPair p : pairs) {
- p.transfer();
- }
- to.getMutator().setValueCount(from.getAccessor().getValueCount());
- from.clear();
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(from, to);
- }
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- for (TransferPair p : pairs) {
- p.splitAndTransfer(startIndex, length);
- }
- to.getMutator().setValueCount(length);
- }
- }
-
- private static class RepeatedMapTransferPair implements TransferPair{
-
- private final TransferPair[] pairs;
- private final RepeatedMapVector to;
- private final RepeatedMapVector from;
-
- public RepeatedMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
- this(from, new RepeatedMapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
- }
-
- public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to) {
- this(from, to, true);
- }
-
- public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to, boolean allocate) {
- this.from = from;
- this.to = to;
- this.pairs = new TransferPair[from.size()];
- this.to.ephPair = null;
-
- int i = 0;
- ValueVector vector;
- for (final String child : from.getChildFieldNames()) {
- final int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
-
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (to.size() != preSize) {
- newVector.allocateNew();
- }
-
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
- @Override
- public void transfer() {
- from.offsets.transferTo(to.offsets);
- for (TransferPair p : pairs) {
- p.transfer();
- }
- from.clear();
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int srcIndex, int destIndex) {
- RepeatedMapHolder holder = new RepeatedMapHolder();
- from.getAccessor().get(srcIndex, holder);
- to.emptyPopulator.populate(destIndex + 1);
- int newIndex = to.offsets.getAccessor().get(destIndex);
- //todo: make these bulk copies
- for (int i = holder.start; i < holder.end; i++, newIndex++) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(i, newIndex);
- }
- }
- to.offsets.getMutator().setSafe(destIndex + 1, newIndex);
- }
-
- @Override
- public void splitAndTransfer(final int groupStart, final int groups) {
- final UInt4Vector.Accessor a = from.offsets.getAccessor();
- final UInt4Vector.Mutator m = to.offsets.getMutator();
-
- final int startPos = a.get(groupStart);
- final int endPos = a.get(groupStart + groups);
- final int valuesToCopy = endPos - startPos;
-
- to.offsets.clear();
- to.offsets.allocateNew(groups + 1);
-
- int normalizedPos;
- for (int i = 0; i < groups + 1; i++) {
- normalizedPos = a.get(groupStart + i) - startPos;
- m.set(i, normalizedPos);
- }
-
- m.setValueCount(groups + 1);
- to.emptyPopulator.populate(groups);
-
- for (final TransferPair p : pairs) {
- p.splitAndTransfer(startPos, valuesToCopy);
- }
- }
- }
-
-
- transient private RepeatedMapTransferPair ephPair;
-
- public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) {
- if (ephPair == null || ephPair.from != from) {
- ephPair = (RepeatedMapTransferPair) from.makeTransferPair(this);
- }
- ephPair.copyValueSafe(fromIndex, thisIndex);
- }
-
- @Override
- public int getValueCapacity() {
- return Math.max(offsets.getValueCapacity() - 1, 0);
- }
-
- @Override
- public RepeatedMapAccessor getAccessor() {
- return accessor;
- }
-
- @Override
- public ArrowBuf[] getBuffers(boolean clear) {
- final int expectedBufferSize = getBufferSize();
- final int actualBufferSize = super.getBufferSize();
-
- Preconditions.checkArgument(expectedBufferSize == actualBufferSize + offsets.getBufferSize());
- return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
- }
-
-
-// @Override
-// public void load(SerializedField metadata, DrillBuf buffer) {
-// final List<SerializedField> children = metadata.getChildList();
-//
-// final SerializedField offsetField = children.get(0);
-// offsets.load(offsetField, buffer);
-// int bufOffset = offsetField.getBufferLength();
-//
-// for (int i = 1; i < children.size(); i++) {
-// final SerializedField child = children.get(i);
-// final MaterializedField fieldDef = SerializedFieldHelper.create(child);
-// ValueVector vector = getChild(fieldDef.getLastName());
-// if (vector == null) {
- // if we arrive here, we didn't have a matching vector.
-// vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
-// putChild(fieldDef.getLastName(), vector);
-// }
-// final int vectorLength = child.getBufferLength();
-// vector.load(child, buffer.slice(bufOffset, vectorLength));
-// bufOffset += vectorLength;
-// }
-//
-// assert bufOffset == buffer.capacity();
-// }
-//
-//
-// @Override
-// public SerializedField getMetadata() {
-// SerializedField.Builder builder = getField() //
-// .getAsBuilder() //
-// .setBufferLength(getBufferSize()) //
- // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
-// .setValueCount(accessor.getValueCount());
-// builder.addChild(offsets.getMetadata());
-// for (final ValueVector child : getChildren()) {
-// builder.addChild(child.getMetadata());
-// }
-// return builder.build();
-// }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- public class RepeatedMapAccessor implements RepeatedAccessor {
- @Override
- public Object getObject(int index) {
- final List<Object> list = new JsonStringArrayList<>();
- final int end = offsets.getAccessor().get(index+1);
- String fieldName;
- for (int i = offsets.getAccessor().get(index); i < end; i++) {
- final Map<String, Object> vv = Maps.newLinkedHashMap();
- for (final MaterializedField field : getField().getChildren()) {
- if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
- fieldName = field.getLastName();
- final Object value = getChild(fieldName).getAccessor().getObject(i);
- if (value != null) {
- vv.put(fieldName, value);
- }
- }
- }
- list.add(vv);
- }
- return list;
- }
-
- @Override
- public int getValueCount() {
- return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
- }
-
- @Override
- public int getInnerValueCount() {
- final int valueCount = getValueCount();
- if (valueCount == 0) {
- return 0;
- }
- return offsets.getAccessor().get(valueCount);
- }
-
- @Override
- public int getInnerValueCountAt(int index) {
- return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
- }
-
- @Override
- public boolean isEmpty(int index) {
- return false;
- }
-
- @Override
- public boolean isNull(int index) {
- return false;
- }
-
- public void get(int index, RepeatedMapHolder holder) {
- assert index < getValueCapacity() :
- String.format("Attempted to access index %d when value capacity is %d",
- index, getValueCapacity());
- final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
- holder.start = offsetsAccessor.get(index);
- holder.end = offsetsAccessor.get(index + 1);
- }
-
- public void get(int index, ComplexHolder holder) {
- final FieldReader reader = getReader();
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- public void get(int index, int arrayIndex, ComplexHolder holder) {
- final RepeatedMapHolder h = new RepeatedMapHolder();
- get(index, h);
- final int offset = h.start + arrayIndex;
-
- if (offset >= h.end) {
- holder.reader = NullReader.INSTANCE;
- } else {
- reader.setSinglePosition(index, arrayIndex);
- holder.reader = reader;
- }
- }
- }
-
- public class Mutator implements RepeatedMutator {
- @Override
- public void startNewValue(int index) {
- emptyPopulator.populate(index + 1);
- offsets.getMutator().setSafe(index + 1, offsets.getAccessor().get(index));
- }
-
- @Override
- public void setValueCount(int topLevelValueCount) {
- emptyPopulator.populate(topLevelValueCount);
- offsets.getMutator().setValueCount(topLevelValueCount == 0 ? 0 : topLevelValueCount + 1);
- int childValueCount = offsets.getAccessor().get(topLevelValueCount);
- for (final ValueVector v : getChildren()) {
- v.getMutator().setValueCount(childValueCount);
- }
- }
-
- @Override
- public void reset() {}
-
- @Override
- public void generateTestData(int values) {}
-
- public int add(int index) {
- final int prevEnd = offsets.getAccessor().get(index + 1);
- offsets.getMutator().setSafe(index + 1, prevEnd + 1);
- return prevEnd;
- }
- }
-
- @Override
- public void clear() {
- getMutator().reset();
-
- offsets.clear();
- for(final ValueVector vector : getChildren()) {
- vector.clear();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index 99c0a0a..54db393 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -28,7 +28,7 @@ import org.apache.arrow.vector.ValueVector;
* uses the offset vector to determine the sequence of cells pertaining to an individual value.
*
*/
-public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
+public interface RepeatedValueVector extends ValueVector {
final static int DEFAULT_REPEAT_PER_RECORD = 5;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
index 264e241..259a954 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
@@ -19,20 +19,20 @@ package org.apache.arrow.vector.complex.impl;
import java.util.Iterator;
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.flatbuf.Union;
+import org.apache.arrow.flatbuf.UnionMode;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.holders.UnionHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
abstract class AbstractBaseReader implements FieldReader{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseReader.class);
- private static final MajorType LATE_BIND_TYPE = new MajorType(MinorType.LATE, DataMode.OPTIONAL);
private int index;
@@ -58,15 +58,6 @@ abstract class AbstractBaseReader implements FieldReader{
throw new IllegalStateException("The current reader doesn't support reading as a map.");
}
- public MajorType getType(){
- throw new IllegalStateException("The current reader doesn't support getting type information.");
- }
-
- @Override
- public MaterializedField getField() {
- return MaterializedField.create("unknown", LATE_BIND_TYPE);
- }
-
@Override
public boolean next() {
throw new IllegalStateException("The current reader doesn't support getting next information.");
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
index 4e1e103..e6cf098 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
@@ -23,25 +23,11 @@ import org.apache.arrow.vector.complex.writer.FieldWriter;
abstract class AbstractBaseWriter implements FieldWriter {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseWriter.class);
- final FieldWriter parent;
private int index;
- public AbstractBaseWriter(FieldWriter parent) {
- this.parent = parent;
- }
-
@Override
public String toString() {
- return super.toString() + "[index = " + index + ", parent = " + parent + "]";
- }
-
- @Override
- public FieldWriter getParent() {
- return parent;
- }
-
- public boolean isRoot() {
- return parent == null;
+ return super.toString() + "[index = " + index + "]";
}
int idx() {
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 4e2051f..4d2adfb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -17,20 +17,20 @@
*/
package org.apache.arrow.vector.complex.impl;
+import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.StateTool;
import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.MinorType;
import com.google.common.base.Preconditions;
+import org.apache.arrow.vector.types.pojo.Field;
public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
private SingleMapWriter mapRoot;
- private SingleListWriter listRoot;
+ private UnionListWriter listRoot;
private final MapVector container;
Mode mode = Mode.INIT;
@@ -40,7 +40,6 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
private enum Mode { INIT, MAP, LIST };
public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){
- super(null);
this.name = name;
this.container = container;
this.unionEnabled = unionEnabled;
@@ -51,7 +50,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return container.getField();
}
@@ -123,7 +122,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
case INIT:
MapVector map = (MapVector) container;
- mapRoot = new SingleMapWriter(map, this, unionEnabled);
+ mapRoot = new SingleMapWriter(map);
mapRoot.setPosition(idx());
mode = Mode.MAP;
break;
@@ -143,8 +142,8 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
switch(mode){
case INIT:
- MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
- mapRoot = new SingleMapWriter(map, this, unionEnabled);
+ MapVector map = container.addOrGet(name, MinorType.MAP, MapVector.class);
+ mapRoot = new SingleMapWriter(map);
mapRoot.setPosition(idx());
mode = Mode.MAP;
break;
@@ -174,7 +173,12 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
switch(mode){
case INIT:
- listRoot = new SingleListWriter(name, container, this);
+ int vectorCount = container.size();
+ ListVector listVector = container.addOrGet(name, MinorType.LIST, ListVector.class);
+ if (container.size() > vectorCount) {
+ listVector.allocateNew();
+ }
+ listRoot = new UnionListWriter(listVector);
listRoot.setPosition(idx());
mode = Mode.LIST;
break;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 462ec9d..586b128 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -17,20 +17,14 @@
*/
package org.apache.arrow.vector.complex.impl;
-import java.lang.reflect.Constructor;
-
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
import org.apache.arrow.vector.ZeroVector;
import org.apache.arrow.vector.complex.AbstractMapVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.TransferPair;
/**
@@ -56,14 +50,12 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
private FieldWriter writer;
public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
- super(null);
this.parentContainer = parentContainer;
this.listVector = null;
init(v);
}
public PromotableWriter(ValueVector v, ListVector listVector) {
- super(null);
this.listVector = listVector;
this.parentContainer = null;
init(v);
@@ -84,30 +76,8 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
private void setWriter(ValueVector v) {
state = State.SINGLE;
vector = v;
- type = v.getField().getType().getMinorType();
- Class<?> writerClass = BasicTypeHelper
- .getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
- if (writerClass.equals(SingleListWriter.class)) {
- writerClass = UnionListWriter.class;
- }
- Class<?> vectorClass = BasicTypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField()
- .getDataMode());
- try {
- Constructor<?> constructor = null;
- for (Constructor<?> c : writerClass.getConstructors()) {
- if (c.getParameterTypes().length == 3) {
- constructor = c;
- }
- }
- if (constructor == null) {
- constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class);
- writer = (FieldWriter) constructor.newInstance(vector, null);
- } else {
- writer = (FieldWriter) constructor.newInstance(vector, null, true);
- }
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException(e);
- }
+ type = v.getMinorType();
+ writer = type.getNewFieldWriter(vector);
}
@Override
@@ -129,7 +99,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
if (type == null) {
return null;
}
- ValueVector v = listVector.addOrGetVector(new VectorDescriptor(new MajorType(type, DataMode.OPTIONAL))).getVector();
+ ValueVector v = listVector.addOrGetVector(type).getVector();
v.allocateNew();
setWriter(v);
writer.setPosition(position);
@@ -150,11 +120,11 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
}
private FieldWriter promoteToUnion() {
- String name = vector.getField().getLastName();
- TransferPair tp = vector.getTransferPair(vector.getField().getType().getMinorType().name().toLowerCase(), vector.getAllocator());
+ String name = vector.getField().getName();
+ TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator());
tp.transfer();
if (parentContainer != null) {
- unionVector = parentContainer.addOrGet(name, new MajorType(MinorType.UNION, DataMode.OPTIONAL), UnionVector.class);
+ unionVector = parentContainer.addOrGet(name, MinorType.UNION, UnionVector.class);
unionVector.allocateNew();
} else if (listVector != null) {
unionVector = listVector.promoteToUnion();
@@ -163,7 +133,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
writer = new UnionWriter(unionVector);
writer.setPosition(idx());
for (int i = 0; i < idx(); i++) {
- unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
+ unionVector.getMutator().setType(i, vector.getMinorType());
}
vector = null;
state = State.UNION;
@@ -181,7 +151,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return getWriter().getField();
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
deleted file mode 100644
index dd1a152..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex.impl;
-
-
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedListVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.holders.RepeatedListHolder;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-
-public class RepeatedListReaderImpl extends AbstractFieldReader{
- private static final int NO_VALUES = Integer.MAX_VALUE - 1;
- private static final MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
- private final String name;
- private final RepeatedListVector container;
- private FieldReader reader;
-
- public RepeatedListReaderImpl(String name, RepeatedListVector container) {
- super();
- this.name = name;
- this.container = container;
- }
-
- @Override
- public MajorType getType() {
- return TYPE;
- }
-
- @Override
- public void copyAsValue(ListWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedListWriter impl = (RepeatedListWriter) writer;
- impl.container.copyFromSafe(idx(), impl.idx(), container);
- }
-
- @Override
- public void copyAsField(String name, MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedListWriter impl = (RepeatedListWriter) writer.list(name);
- impl.container.copyFromSafe(idx(), impl.idx(), container);
- }
-
- private int currentOffset;
- private int maxOffset;
-
- @Override
- public void reset() {
- super.reset();
- currentOffset = 0;
- maxOffset = 0;
- if (reader != null) {
- reader.reset();
- }
- reader = null;
- }
-
- @Override
- public int size() {
- return maxOffset - currentOffset;
- }
-
- @Override
- public void setPosition(int index) {
- if (index < 0 || index == NO_VALUES) {
- currentOffset = NO_VALUES;
- return;
- }
-
- super.setPosition(index);
- RepeatedListHolder h = new RepeatedListHolder();
- container.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- currentOffset = h.start-1;
- maxOffset = h.end;
- if(reader != null) {
- reader.setPosition(currentOffset);
- }
- }
- }
-
- @Override
- public boolean next() {
- if (currentOffset +1 < maxOffset) {
- currentOffset++;
- if (reader != null) {
- reader.setPosition(currentOffset);
- }
- return true;
- } else {
- currentOffset = NO_VALUES;
- return false;
- }
- }
-
- @Override
- public Object readObject() {
- return container.getAccessor().getObject(idx());
- }
-
- @Override
- public FieldReader reader() {
- if (reader == null) {
- ValueVector child = container.getChild(name);
- if (child == null) {
- reader = NullReader.INSTANCE;
- } else {
- reader = child.getReader();
- }
- reader.setPosition(currentOffset);
- }
- return reader;
- }
-
- public boolean isSet() {
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
deleted file mode 100644
index 09a831d..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex.impl;
-
-import java.util.Map;
-
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
-import org.apache.arrow.vector.types.Types.MajorType;
-
-import com.google.common.collect.Maps;
-
-@SuppressWarnings("unused")
-public class RepeatedMapReaderImpl extends AbstractFieldReader{
- private static final int NO_VALUES = Integer.MAX_VALUE - 1;
-
- private final RepeatedMapVector vector;
- private final Map<String, FieldReader> fields = Maps.newHashMap();
-
- public RepeatedMapReaderImpl(RepeatedMapVector vector) {
- this.vector = vector;
- }
-
- private void setChildrenPosition(int index) {
- for (FieldReader r : fields.values()) {
- r.setPosition(index);
- }
- }
-
- @Override
- public FieldReader reader(String name) {
- FieldReader reader = fields.get(name);
- if (reader == null) {
- ValueVector child = vector.getChild(name);
- if (child == null) {
- reader = NullReader.INSTANCE;
- } else {
- reader = child.getReader();
- }
- fields.put(name, reader);
- reader.setPosition(currentOffset);
- }
- return reader;
- }
-
- @Override
- public FieldReader reader() {
- if (currentOffset == NO_VALUES) {
- return NullReader.INSTANCE;
- }
-
- setChildrenPosition(currentOffset);
- return new SingleLikeRepeatedMapReaderImpl(vector, this);
- }
-
- private int currentOffset;
- private int maxOffset;
-
- @Override
- public void reset() {
- super.reset();
- currentOffset = 0;
- maxOffset = 0;
- for (FieldReader reader:fields.values()) {
- reader.reset();
- }
- fields.clear();
- }
-
- @Override
- public int size() {
- if (isNull()) {
- return 0;
- }
- return maxOffset - (currentOffset < 0 ? 0 : currentOffset);
- }
-
- @Override
- public void setPosition(int index) {
- if (index < 0 || index == NO_VALUES) {
- currentOffset = NO_VALUES;
- return;
- }
-
- super.setPosition(index);
- RepeatedMapHolder h = new RepeatedMapHolder();
- vector.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- currentOffset = h.start-1;
- maxOffset = h.end;
- setChildrenPosition(currentOffset);
- }
- }
-
- public void setSinglePosition(int index, int childIndex) {
- super.setPosition(index);
- RepeatedMapHolder h = new RepeatedMapHolder();
- vector.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- int singleOffset = h.start + childIndex;
- assert singleOffset < h.end;
- currentOffset = singleOffset;
- maxOffset = singleOffset + 1;
- setChildrenPosition(singleOffset);
- }
- }
-
- @Override
- public boolean next() {
- if (currentOffset +1 < maxOffset) {
- setChildrenPosition(++currentOffset);
- return true;
- } else {
- currentOffset = NO_VALUES;
- return false;
- }
- }
-
- public boolean isNull() {
- return currentOffset == NO_VALUES;
- }
-
- @Override
- public Object readObject() {
- return vector.getAccessor().getObject(idx());
- }
-
- @Override
- public MajorType getType() {
- return vector.getField().getType();
- }
-
- @Override
- public java.util.Iterator<String> iterator() {
- return vector.fieldNameIterator();
- }
-
- @Override
- public boolean isSet() {
- return true;
- }
-
- @Override
- public void copyAsValue(MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedMapWriter impl = (RepeatedMapWriter) writer;
- impl.container.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public void copyAsValueSingle(MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- SingleMapWriter impl = (SingleMapWriter) writer;
- impl.container.copyFromSafe(currentOffset, impl.idx(), vector);
- }
-
- @Override
- public void copyAsField(String name, MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedMapWriter impl = (RepeatedMapWriter) writer.map(name);
- impl.container.copyFromSafe(idx(), impl.idx(), vector);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
deleted file mode 100644
index 086d26e..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex.impl;
-
-import java.util.Iterator;
-
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-
-public class SingleLikeRepeatedMapReaderImpl extends AbstractFieldReader{
-
- private RepeatedMapReaderImpl delegate;
-
- public SingleLikeRepeatedMapReaderImpl(RepeatedMapVector vector, FieldReader delegate) {
- this.delegate = (RepeatedMapReaderImpl) delegate;
- }
-
- @Override
- public int size() {
- throw new UnsupportedOperationException("You can't call size on a single map reader.");
- }
-
- @Override
- public boolean next() {
- throw new UnsupportedOperationException("You can't call next on a single map reader.");
- }
-
- @Override
- public MajorType getType() {
- return Types.required(MinorType.MAP);
- }
-
-
- @Override
- public void copyAsValue(MapWriter writer) {
- delegate.copyAsValueSingle(writer);
- }
-
- public void copyAsValueSingle(MapWriter writer){
- delegate.copyAsValueSingle(writer);
- }
-
- @Override
- public FieldReader reader(String name) {
- return delegate.reader(name);
- }
-
- @Override
- public void setPosition(int index) {
- delegate.setPosition(index);
- }
-
- @Override
- public Object readObject() {
- return delegate.readObject();
- }
-
- @Override
- public Iterator<String> iterator() {
- return delegate.iterator();
- }
-
- @Override
- public boolean isSet() {
- return ! delegate.isNull();
- }
-
-
-}
[5/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead
of MaterializedField
Posted by sm...@apache.org.
ARROW-259: Use Flatbuffer Field type instead of MaterializedField
Remove MaterializedField, MajorType, RepeatedTypes
Add code to convert from FlatBuf representation to Pojo
also adds tests to test the conversion
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e7e399db
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e7e399db
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e7e399db
Branch: refs/heads/master
Commit: e7e399db5fc6913e67426514279f81766a0778d2
Parents: 246a126
Author: Steven Phillips <st...@dremio.com>
Authored: Tue May 24 13:38:09 2016 -0700
Committer: Steven Phillips <st...@dremio.com>
Committed: Thu Aug 18 16:29:27 2016 -0700
----------------------------------------------------------------------
format/Message.fbs | 22 +-
header | 16 +
java/format/pom.xml | 163 ++++
java/memory/pom.xml | 2 +-
java/pom.xml | 3 +-
java/vector/pom.xml | 7 +-
java/vector/src/main/codegen/config.fmpp | 1 +
.../vector/src/main/codegen/data/ArrowTypes.tdd | 80 ++
.../src/main/codegen/data/ValueVectorTypes.tdd | 59 +-
.../src/main/codegen/includes/vv_imports.ftl | 4 +
.../codegen/templates/AbstractFieldReader.java | 8 +-
.../codegen/templates/AbstractFieldWriter.java | 10 +-
.../AbstractPromotableFieldWriter.java | 4 -
.../src/main/codegen/templates/ArrowType.java | 129 +++
.../src/main/codegen/templates/BaseReader.java | 5 +-
.../src/main/codegen/templates/BaseWriter.java | 3 +-
.../main/codegen/templates/BasicTypeHelper.java | 539 -------------
.../main/codegen/templates/ComplexCopier.java | 18 +-
.../main/codegen/templates/ComplexReaders.java | 72 +-
.../main/codegen/templates/ComplexWriters.java | 30 +-
.../codegen/templates/FixedValueVectors.java | 94 ++-
.../codegen/templates/HolderReaderImpl.java | 98 +--
.../src/main/codegen/templates/ListWriters.java | 234 ------
.../src/main/codegen/templates/MapWriters.java | 42 +-
.../src/main/codegen/templates/NullReader.java | 23 +-
.../codegen/templates/NullableValueVectors.java | 104 ++-
.../codegen/templates/RepeatedValueVectors.java | 421 ----------
.../main/codegen/templates/UnionListWriter.java | 23 +-
.../src/main/codegen/templates/UnionReader.java | 28 +-
.../src/main/codegen/templates/UnionVector.java | 105 ++-
.../src/main/codegen/templates/UnionWriter.java | 16 +-
.../main/codegen/templates/ValueHolders.java | 43 +-
.../templates/VariableLengthVectors.java | 73 +-
.../arrow/vector/BaseDataValueVector.java | 5 +-
.../apache/arrow/vector/BaseValueVector.java | 31 +-
.../java/org/apache/arrow/vector/BitVector.java | 43 +-
.../org/apache/arrow/vector/ObjectVector.java | 220 -----
.../apache/arrow/vector/ValueHolderHelper.java | 203 -----
.../org/apache/arrow/vector/ValueVector.java | 10 +-
.../apache/arrow/vector/VectorDescriptor.java | 83 --
.../org/apache/arrow/vector/ZeroVector.java | 30 +-
.../vector/complex/AbstractContainerVector.java | 49 +-
.../arrow/vector/complex/AbstractMapVector.java | 47 +-
.../vector/complex/BaseRepeatedValueVector.java | 63 +-
.../vector/complex/ContainerVectorLike.java | 43 -
.../apache/arrow/vector/complex/ListVector.java | 89 +-
.../apache/arrow/vector/complex/MapVector.java | 97 +--
.../vector/complex/RepeatedListVector.java | 427 ----------
.../arrow/vector/complex/RepeatedMapVector.java | 584 --------------
.../vector/complex/RepeatedValueVector.java | 2 +-
.../vector/complex/impl/AbstractBaseReader.java | 19 +-
.../vector/complex/impl/AbstractBaseWriter.java | 16 +-
.../vector/complex/impl/ComplexWriterImpl.java | 22 +-
.../vector/complex/impl/PromotableWriter.java | 48 +-
.../complex/impl/RepeatedListReaderImpl.java | 145 ----
.../complex/impl/RepeatedMapReaderImpl.java | 192 -----
.../impl/SingleLikeRepeatedMapReaderImpl.java | 89 --
.../complex/impl/SingleListReaderImpl.java | 14 +-
.../complex/impl/SingleMapReaderImpl.java | 10 +-
.../vector/complex/impl/UnionListReader.java | 19 +-
.../arrow/vector/holders/ObjectHolder.java | 38 -
.../arrow/vector/holders/UnionHolder.java | 7 +-
.../arrow/vector/types/MaterializedField.java | 217 -----
.../org/apache/arrow/vector/types/Types.java | 596 +++++++++++---
.../apache/arrow/vector/types/pojo/Field.java | 105 +++
.../apache/arrow/vector/types/pojo/Schema.java | 74 ++
.../arrow/vector/util/ByteFunctionHelpers.java | 50 --
.../arrow/vector/util/CoreDecimalUtility.java | 91 ---
.../arrow/vector/util/DecimalUtility.java | 802 +++++++++----------
.../arrow/vector/util/MapWithOrdinal.java | 12 +
.../apache/arrow/vector/TestDecimalVector.java | 63 ++
.../TestOversizedAllocationForValueVector.java | 11 +-
.../apache/arrow/vector/TestUnionVector.java | 5 +-
.../apache/arrow/vector/TestValueVector.java | 137 +---
.../complex/impl/TestPromotableWriter.java | 7 +-
.../complex/writer/TestComplexWriter.java | 270 +++++++
.../apache/arrow/vector/pojo/TestConvert.java | 80 ++
77 files changed, 2464 insertions(+), 5180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/format/Message.fbs
----------------------------------------------------------------------
diff --git a/format/Message.fbs b/format/Message.fbs
index 3f688c1..2928207 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -1,10 +1,13 @@
-namespace apache.arrow.flatbuf;
+namespace org.apache.arrow.flatbuf;
/// ----------------------------------------------------------------------
/// Logical types and their metadata (if any)
///
/// These are stored in the flatbuffer in the Type union below
+table Null {
+}
+
/// A Tuple in the flatbuffer metadata is the same as an Arrow Struct
/// (according to the physical memory layout). We used Tuple here as Struct is
/// a reserved word in Flatbuffers
@@ -45,10 +48,22 @@ table Decimal {
scale: int;
}
+table Date {
+}
+
+table Time {
+}
+
table Timestamp {
timezone: string;
}
+table IntervalDay {
+}
+
+table IntervalYear {
+}
+
table JSONScalar {
dense:bool=true;
}
@@ -58,13 +73,18 @@ table JSONScalar {
/// add new logical types to Type without breaking backwards compatibility
union Type {
+ Null,
Int,
FloatingPoint,
Binary,
Utf8,
Bool,
Decimal,
+ Date,
+ Time,
Timestamp,
+ IntervalDay,
+ IntervalYear,
List,
Tuple,
Union,
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/header
----------------------------------------------------------------------
diff --git a/header b/header
new file mode 100644
index 0000000..70665d1
--- /dev/null
+++ b/header
@@ -0,0 +1,16 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/format/pom.xml
----------------------------------------------------------------------
diff --git a/java/format/pom.xml b/java/format/pom.xml
new file mode 100644
index 0000000..ea27a30
--- /dev/null
+++ b/java/format/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<modelVersion>4.0.0</modelVersion>
+
+<parent>
+ <artifactId>arrow-java-root</artifactId>
+ <groupId>org.apache.arrow</groupId>
+ <version>0.1-decimal</version>
+</parent>
+
+<artifactId>arrow-format</artifactId>
+<packaging>jar</packaging>
+<name>Arrow Format</name>
+
+ <properties>
+ <fbs.version>1.2.0-3f79e055</fbs.version>
+ <maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
+ <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
+ <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.vlkan</groupId>
+ <artifactId>flatbuffers</artifactId>
+ <version>${fbs.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <extensions>
+ <!-- provides os.detected.classifier (i.e. linux-x86_64, osx-x86_64) property -->
+ <extension>
+ <groupId>kr.motd.maven</groupId>
+ <artifactId>os-maven-plugin</artifactId>
+ <version>${os-maven-plugin.version}</version>
+ </extension>
+ </extensions>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>${maven-dependency-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy-flatc</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>com.vlkan</groupId>
+ <artifactId>flatc-${os.detected.classifier}</artifactId>
+ <version>${fbs.version}</version>
+ <type>exe</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.4.0</version>
+ <executions>
+ <execution>
+ <id>script-chmod</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <executable>chmod</executable>
+ <arguments>
+ <argument>+x</argument>
+ <argument>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</executable>
+ <arguments>
+ <argument>-j</argument>
+ <argument>-o</argument>
+ <argument>target/generated-sources/</argument>
+ <argument>../../format/Message.fbs</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.mycila</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <version>2.3</version>
+ <configuration>
+ <header>${basedir}/../../header</header>
+ <includes>
+ <include>**/*.java</include>
+ </includes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>format</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>add-sources-as-resources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+
+</build>
+</project>
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/memory/pom.xml
----------------------------------------------------------------------
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 44332f5..12ff4c8 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -15,7 +15,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.1-decimal</version>
</parent>
<artifactId>arrow-memory</artifactId>
<name>arrow-memory</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 71f59ca..92ab109 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.1-decimal</version>
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
@@ -465,6 +465,7 @@
</dependencies>
<modules>
+ <module>format</module>
<module>memory</module>
<module>vector</module>
</modules>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/pom.xml
----------------------------------------------------------------------
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index df53892..fac788c 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -15,7 +15,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.1-decimal</version>
</parent>
<artifactId>vector</artifactId>
<name>vectors</name>
@@ -24,6 +24,11 @@
<dependency>
<groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-format</artifactId>
+ <version>0.1-decimal</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/config.fmpp b/java/vector/src/main/codegen/config.fmpp
index 663677c..6d92ba8 100644
--- a/java/vector/src/main/codegen/config.fmpp
+++ b/java/vector/src/main/codegen/config.fmpp
@@ -17,6 +17,7 @@
data: {
# TODO: Rename to ~valueVectorModesAndTypes for clarity.
vv: tdd(../data/ValueVectorTypes.tdd),
+ arrowTypes: tdd(../data/ArrowTypes.tdd)
}
freemarkerLinks: {
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/data/ArrowTypes.tdd
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
new file mode 100644
index 0000000..4ab7f85
--- /dev/null
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ types: [
+ {
+ name: "Null",
+ fields: []
+ },
+ {
+ name: "Tuple",
+ fields: []
+ },
+ {
+ name: "List",
+ fields: []
+ },
+ {
+ name: "Union",
+ fields: []
+ },
+ {
+ name: "Int",
+ fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
+ },
+ {
+ name: "FloatingPoint",
+ fields: [{name: precision, type: int}]
+ },
+ {
+ name: "Utf8",
+ fields: []
+ },
+ {
+ name: "Binary",
+ fields: []
+ },
+ {
+ name: "Bool",
+ fields: []
+ },
+ {
+ name: "Decimal",
+ fields: [{name: "precision", type: int}, {name: "scale", type: int}]
+ },
+ {
+ name: "Date",
+ fields: []
+ },
+ {
+ name: "Time",
+ fields: []
+ },
+ {
+ name: "Timestamp",
+ fields: [{name: "timezone", type: "String"}]
+ },
+ {
+ name: "IntervalDay",
+ fields: []
+ },
+ {
+ name: "IntervalYear",
+ fields: []
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index e747c30..421dd7e 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -17,8 +17,7 @@
{
modes: [
{name: "Optional", prefix: "Nullable"},
- {name: "Required", prefix: ""},
- {name: "Repeated", prefix: "Repeated"}
+ {name: "Required", prefix: ""}
],
types: [
{
@@ -61,9 +60,8 @@
{ class: "Int", valueHolder: "IntHolder"},
{ class: "UInt4", valueHolder: "UInt4Holder" },
{ class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]},
- { class: "Time", javaType: "int", friendlyType: "DateTime" },
{ class: "IntervalYear", javaType: "int", friendlyType: "Period" }
- { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal", fields: [{name:"value", type:"int"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
+ { class: "Time", javaType: "int", friendlyType: "DateTime" }
]
},
{
@@ -78,15 +76,11 @@
{ class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
{ class: "Date", javaType: "long", friendlyType: "DateTime" },
{ class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
- { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal", fields: [{name:"value", type:"long"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
- <#--
- { class: "Money", maxPrecisionDigits: 2, scale: 1, },
- -->
]
},
{
major: "Fixed",
- width: 12,
+ width: 8,
javaType: "ArrowBuf",
boxedType: "ArrowBuf",
minor: [
@@ -96,51 +90,11 @@
{
major: "Fixed",
width: 16,
- javaType: "ArrowBuf"
- boxedType: "ArrowBuf",
- minor: [
- { class: "Interval", daysOffset: 4, millisecondsOffset: 8, friendlyType: "Period", fields: [ {name: "months", type: "int"}, {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
- ]
- },
- {
- major: "Fixed",
- width: 12,
- javaType: "ArrowBuf",
- boxedType: "ArrowBuf",
- minor: [
- <#--
- { class: "TimeTZ" },
- { class: "Interval" }
- -->
- { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 16,
- javaType: "ArrowBuf",
- boxedType: "ArrowBuf",
-
- minor: [
- { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 24,
- javaType: "ArrowBuf",
- boxedType: "ArrowBuf",
- minor: [
- { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 20,
javaType: "ArrowBuf",
boxedType: "ArrowBuf",
+
minor: [
- { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+ { class: "Decimal", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
]
},
{
@@ -151,8 +105,7 @@
fields: [{name: "start", type: "int"}, {name: "end", type: "int"}, {name: "buffer", type: "ArrowBuf"}],
minor: [
{ class: "VarBinary" , friendlyType: "byte[]" },
- { class: "VarChar" , friendlyType: "Text" },
- { class: "Var16Char" , friendlyType: "String" }
+ { class: "VarChar" , friendlyType: "Text" }
]
},
{
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
index 2d808b1..9b4b79b 100644
--- a/java/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -17,6 +17,8 @@ import com.google.common.collect.ObjectArrays;
import com.google.common.base.Charsets;
import com.google.common.collect.ObjectArrays;
+import com.google.flatbuffers.FlatBufferBuilder;
+
import com.google.common.base.Preconditions;
import io.netty.buffer.*;
@@ -25,6 +27,8 @@ import org.apache.commons.lang3.ArrayUtils;
import org.apache.arrow.memory.*;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.*;
+import org.apache.arrow.vector.types.pojo.*;
+import org.apache.arrow.vector.types.pojo.ArrowType.*;
import org.apache.arrow.vector.types.*;
import org.apache.arrow.vector.*;
import org.apache.arrow.vector.holders.*;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/AbstractFieldReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
index b83dba2..e0d0fc9 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -41,7 +41,13 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe
return true;
}
- <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
+ @Override
+ public Field getField() {
+ fail("getField");
+ return null;
+ }
+
+ <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
"Character", "DateTime", "Period", "Double", "Float",
"Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
<#assign safeType=friendlyType />
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 6ee9dad..de076fc 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -31,10 +31,6 @@ package org.apache.arrow.vector.complex.impl;
*/
@SuppressWarnings("unused")
abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter {
- AbstractFieldWriter(FieldWriter parent) {
- super(parent);
- }
-
@Override
public void start() {
throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
@@ -62,9 +58,15 @@ abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWr
fail("${name}");
}
+ <#if minor.class == "Decimal">
+ public void writeDecimal(int start, ArrowBuf buffer) {
+ fail("${name}");
+ }
+ <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
fail("${name}");
}
+ </#if>
</#list></#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 549dbf1..7e60320 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -37,10 +37,6 @@ package org.apache.arrow.vector.complex.impl;
*/
@SuppressWarnings("unused")
abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
- AbstractPromotableFieldWriter(FieldWriter parent) {
- super(parent);
- }
-
/**
* Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type
* @param type
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ArrowType.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
new file mode 100644
index 0000000..6dfaf21
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.arrow.flatbuf.Field;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+
+import java.util.Objects;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/types/pojo/ArrowType.java" />
+
+
+<#include "/@includes/license.ftl" />
+package org.apache.arrow.vector.types.pojo;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Type;
+
+import java.util.Objects;
+
+public abstract class ArrowType {
+
+ public abstract byte getTypeType();
+ public abstract int getType(FlatBufferBuilder builder);
+
+
+ <#list arrowTypes.types as type>
+ <#assign name = type.name>
+ <#assign fields = type.fields>
+ public static class ${name} extends ArrowType {
+ public static final byte TYPE_TYPE = Type.${name};
+ <#if type.fields?size == 0>
+ public static final ${name} INSTANCE = new ${name}();
+ </#if>
+
+ <#list fields as field>
+ <#assign fieldName = field.name>
+ <#assign fieldType = field.type>
+ ${fieldType} ${fieldName};
+ </#list>
+
+ <#if type.fields?size != 0>
+ public ${type.name}(<#list type.fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ <#list type.fields as field>
+ this.${field.name} = ${field.name};
+ </#list>
+ }
+ </#if>
+
+ @Override
+ public byte getTypeType() {
+ return TYPE_TYPE;
+ }
+
+ @Override
+ public int getType(FlatBufferBuilder builder) {
+ org.apache.arrow.flatbuf.${type.name}.start${type.name}(builder);
+ <#list type.fields as field>
+ org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, <#if field.type == "String">builder.createString(${field.name})<#else>${field.name}</#if>);
+ </#list>
+ return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
+ }
+
+ <#list fields as field>
+ public ${field.type} get${field.name?cap_first}() {
+ return ${field.name};
+ }
+ </#list>
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ${type.name})) {
+ return false;
+ }
+ <#if type.fields?size == 0>
+ return true;
+ <#else>
+ ${type.name} that = (${type.name}) obj;
+ return
+ <#list type.fields as field>Objects.equals(this.${field.name}, that.${field.name}) <#if field_has_next>&&<#else>;</#if>
+ </#list>
+ </#if>
+ }
+ }
+ </#list>
+
+ public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.apache.arrow.flatbuf.Field field) {
+ switch(field.typeType()) {
+ <#list arrowTypes.types as type>
+ <#assign name = type.name>
+ <#assign nameLower = type.name?lower_case>
+ <#assign fields = type.fields>
+ case Type.${type.name}:
+ org.apache.arrow.flatbuf.${type.name} ${nameLower}Type = (org.apache.arrow.flatbuf.${type.name}) field.type(new org.apache.arrow.flatbuf.${type.name}());
+ return new ${type.name}(<#list type.fields as field>${nameLower}Type.${field.name}()<#if field_has_next>, </#if></#list>);
+ </#list>
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + field.typeType());
+ }
+ }
+
+ public static Int getInt(org.apache.arrow.flatbuf.Field field) {
+ org.apache.arrow.flatbuf.Int intType = (org.apache.arrow.flatbuf.Int) field.type(new org.apache.arrow.flatbuf.Int());
+ return new Int(intType.bitWidth(), intType.isSigned());
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/BaseReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java
index 8f12b1d..72fea58 100644
--- a/java/vector/src/main/codegen/templates/BaseReader.java
+++ b/java/vector/src/main/codegen/templates/BaseReader.java
@@ -30,8 +30,8 @@ package org.apache.arrow.vector.complex.reader;
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
- MajorType getType();
- MaterializedField getField();
+ Field getField();
+ MinorType getMinorType();
void reset();
void read(UnionHolder holder);
void read(int index, UnionHolder holder);
@@ -60,7 +60,6 @@ public interface BaseReader extends Positionable{
public interface ScalarReader extends
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, </#list></#list>
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> Repeated${name}Reader, </#list></#list>
BaseReader {}
interface ComplexReader{
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/BaseWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java
index 299b238..08bd39e 100644
--- a/java/vector/src/main/codegen/templates/BaseWriter.java
+++ b/java/vector/src/main/codegen/templates/BaseWriter.java
@@ -31,12 +31,11 @@ package org.apache.arrow.vector.complex.writer;
*/
@SuppressWarnings("unused")
public interface BaseWriter extends AutoCloseable, Positionable {
- FieldWriter getParent();
int getValueCapacity();
public interface MapWriter extends BaseWriter {
- MaterializedField getField();
+ Field getField();
/**
* Whether this writer is a map writer and is empty (has no children).
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/BasicTypeHelper.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/BasicTypeHelper.java b/java/vector/src/main/codegen/templates/BasicTypeHelper.java
deleted file mode 100644
index 0bae715..0000000
--- a/java/vector/src/main/codegen/templates/BasicTypeHelper.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/arrow/vector/util/BasicTypeHelper.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector.util;
-
-<#include "/@includes/vv_imports.ftl" />
-import org.apache.arrow.vector.complex.UnionVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.util.CallBack;
-
-public class BasicTypeHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicTypeHelper.class);
-
- private static final int WIDTH_ESTIMATE = 50;
-
- // Default length when casting to varchar : 65536 = 2^16
- // This only defines an absolute maximum for values, setting
- // a high value like this will not inflate the size for small values
- public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
-
- protected static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
- return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
- }
-
- protected static String buildErrorMessage(final String operation, final MajorType type) {
- return buildErrorMessage(operation, type.getMinorType(), type.getMode());
- }
-
- public static int getSize(MajorType major) {
- switch (major.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
- minor.class?substring(0, 3) == "PRO" ||
- minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
- </#list>
-</#list>
-// case FIXEDCHAR: return major.getWidth();
-// case FIXED16CHAR: return major.getWidth();
-// case FIXEDBINARY: return major.getWidth();
- }
- throw new UnsupportedOperationException(buildErrorMessage("get size", major));
- }
-
- public static ValueVector getNewVector(String name, BufferAllocator allocator, MajorType type, CallBack callback){
- MaterializedField field = MaterializedField.create(name, type);
- return getNewVector(field, allocator, callback);
- }
-
-
- public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return UnionVector.class;
- case MAP:
- switch (mode) {
- case OPTIONAL:
- case REQUIRED:
- return MapVector.class;
- case REPEATED:
- return RepeatedMapVector.class;
- }
-
- case LIST:
- switch (mode) {
- case REPEATED:
- return RepeatedListVector.class;
- case REQUIRED:
- case OPTIONAL:
- return ListVector.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}Vector.class;
- case OPTIONAL:
- return Nullable${minor.class}Vector.class;
- case REPEATED:
- return Repeated${minor.class}Vector.class;
- }
- </#list>
-</#list>
- case GENERIC_OBJECT :
- return ObjectVector.class ;
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
- }
- public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
- switch (type) {
- case MAP:
- switch (mode) {
- case REQUIRED:
- if (!isSingularRepeated)
- return SingleMapReaderImpl.class;
- else
- return SingleLikeRepeatedMapReaderImpl.class;
- case REPEATED:
- return RepeatedMapReaderImpl.class;
- }
- case LIST:
- switch (mode) {
- case REQUIRED:
- return SingleListReaderImpl.class;
- case REPEATED:
- return RepeatedListReaderImpl.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}ReaderImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}ReaderImpl.class;
- case REPEATED:
- return Repeated${minor.class}ReaderImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
- }
-
- public static Class<?> getWriterInterface( MinorType type, DataMode mode){
- switch (type) {
- case UNION: return UnionWriter.class;
- case MAP: return MapWriter.class;
- case LIST: return ListWriter.class;
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}: return ${minor.class}Writer.class;
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
- }
-
- public static Class<?> getWriterImpl( MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return UnionWriter.class;
- case MAP:
- switch (mode) {
- case REQUIRED:
- case OPTIONAL:
- return SingleMapWriter.class;
- case REPEATED:
- return RepeatedMapWriter.class;
- }
- case LIST:
- switch (mode) {
- case REQUIRED:
- case OPTIONAL:
- return UnionListWriter.class;
- case REPEATED:
- return RepeatedListWriter.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}WriterImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}WriterImpl.class;
- case REPEATED:
- return Repeated${minor.class}WriterImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
- }
-
- public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
- switch (type) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}HolderReaderImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}HolderReaderImpl.class;
- case REPEATED:
- return Repeated${minor.class}HolderReaderImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
- }
-
- public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
- return getNewVector(field, allocator, null);
- }
- public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- field = field.clone();
- MajorType type = field.getType();
-
- switch (type.getMinorType()) {
-
- case UNION:
- return new UnionVector(field, allocator, callBack);
-
- case MAP:
- switch (type.getMode()) {
- case REQUIRED:
- case OPTIONAL:
- return new MapVector(field, allocator, callBack);
- case REPEATED:
- return new RepeatedMapVector(field, allocator, callBack);
- }
- case LIST:
- switch (type.getMode()) {
- case REPEATED:
- return new RepeatedListVector(field, allocator, callBack);
- case OPTIONAL:
- case REQUIRED:
- return new ListVector(field, allocator, callBack);
- }
-<#list vv. types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Vector(field, allocator);
- case OPTIONAL:
- return new Nullable${minor.class}Vector(field, allocator);
- case REPEATED:
- return new Repeated${minor.class}Vector(field, allocator);
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- return new ObjectVector(field, allocator) ;
- default:
- break;
- }
- // All ValueVector types have been handled.
- throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
- }
-
- public static ValueHolder getValue(ValueVector vector, int index) {
- MajorType type = vector.getField().getType();
- ValueHolder holder;
- switch(type.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
- minor.class?starts_with("Decimal28") || minor.class?starts_with("Decimal38")>
- switch (type.getMode()) {
- case REQUIRED:
- holder = new ${minor.class}Holder();
- ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
- return holder;
- case OPTIONAL:
- holder = new Nullable${minor.class}Holder();
- ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
- if (((Nullable${minor.class}Holder)holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
- }
- return holder;
- }
- <#else>
- switch (type.getMode()) {
- case REQUIRED:
- holder = new ${minor.class}Holder();
- ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
- return holder;
- case OPTIONAL:
- holder = new Nullable${minor.class}Holder();
- ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
- if (((Nullable${minor.class}Holder)holder).isSet == 1) {
- ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
- }
- return holder;
- }
- </#if>
- </#list>
-</#list>
- case GENERIC_OBJECT:
- holder = new ObjectHolder();
- ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index) ;
- break;
- }
-
- throw new UnsupportedOperationException(buildErrorMessage("get value", type));
- }
-
- public static void setValue(ValueVector vector, int index, ValueHolder holder) {
- MajorType type = vector.getField().getType();
-
- switch(type.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
- return;
- case OPTIONAL:
- if (((Nullable${minor.class}Holder) holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
- }
- return;
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
- return;
- default:
- throw new UnsupportedOperationException(buildErrorMessage("set value", type));
- }
- }
-
- public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
- MajorType type = vector.getField().getType();
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
- return;
- case OPTIONAL:
- if (((Nullable${minor.class}Holder) holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
- } else {
- ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
- }
- return;
- }
- </#list>
- </#list>
- case GENERIC_OBJECT:
- ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
- default:
- throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
- }
- }
-
- public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
- MajorType type1 = v1.getField().getType();
- MajorType type2 = v2.getField().getType();
-
- if (type1.getMinorType() != type2.getMinorType()) {
- return false;
- }
-
- switch(type1.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) ==
- ((${minor.class}Vector) v2).getAccessor().get(v2index) )
- return true;
- break;
- </#list>
-</#list>
- default:
- break;
- }
- return false;
- }
-
- /**
- * Create a ValueHolder of MajorType.
- * @param type
- * @return
- */
- public static ValueHolder createValueHolder(MajorType type) {
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Holder();
- case OPTIONAL:
- return new Nullable${minor.class}Holder();
- case REPEATED:
- return new Repeated${minor.class}Holder();
- }
- </#list>
- </#list>
- case GENERIC_OBJECT:
- return new ObjectHolder();
- default:
- throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
- }
- }
-
- public static boolean isNull(ValueHolder holder) {
- MajorType type = getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return true;
- case OPTIONAL:
- return ((Nullable${minor.class}Holder) holder).isSet == 0;
- case REPEATED:
- return true;
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
- }
- }
-
- public static ValueHolder deNullify(ValueHolder holder) {
- MajorType type = getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return holder;
- case OPTIONAL:
- if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
- ${minor.class}Holder newHolder = new ${minor.class}Holder();
-
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
- </#list>
-
- return newHolder;
- } else {
- throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
- }
- case REPEATED:
- return holder;
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
- }
- }
-
- public static ValueHolder nullify(ValueHolder holder) {
- MajorType type = getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
- newHolder.isSet = 1;
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
- </#list>
- return newHolder;
- case OPTIONAL:
- return holder;
- case REPEATED:
- throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
- }
- }
-
- public static MajorType getValueHolderType(ValueHolder holder) {
-
- if (0 == 1) {
- return null;
- }
- <#list vv.types as type>
- <#list type.minor as minor>
- else if (holder instanceof ${minor.class}Holder) {
- return ((${minor.class}Holder) holder).TYPE;
- } else if (holder instanceof Nullable${minor.class}Holder) {
- return ((Nullable${minor.class}Holder) holder).TYPE;
- }
- </#list>
- </#list>
-
- throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ComplexCopier.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index 3614231..a5756a4 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -42,13 +42,7 @@ public class ComplexCopier {
}
private static void writeValue(FieldReader reader, FieldWriter writer) {
- final DataMode m = reader.getType().getMode();
- final MinorType mt = reader.getType().getMinorType();
-
- switch(m){
- case OPTIONAL:
- case REQUIRED:
-
+ final MinorType mt = reader.getMinorType();
switch (mt) {
@@ -89,12 +83,10 @@ public class ComplexCopier {
</#if>
</#list></#list>
}
- break;
- }
}
private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
- switch (reader.getType().getMinorType()) {
+ switch (reader.getMinorType()) {
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
@@ -108,12 +100,12 @@ public class ComplexCopier {
case LIST:
return (FieldWriter) writer.list(name);
default:
- throw new UnsupportedOperationException(reader.getType().toString());
+ throw new UnsupportedOperationException(reader.getMinorType().toString());
}
}
private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) {
- switch (reader.getType().getMinorType()) {
+ switch (reader.getMinorType()) {
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
@@ -127,7 +119,7 @@ public class ComplexCopier {
case LIST:
return (FieldWriter) writer.list();
default:
- throw new UnsupportedOperationException(reader.getType().toString());
+ throw new UnsupportedOperationException(reader.getMinorType().toString());
}
}
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ComplexReaders.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 34c6571..74a19a6 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -27,10 +27,10 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
<@pp.dropOutputFile />
<#list vv.types as type>
<#list type.minor as minor>
-<#list ["", "Repeated"] as mode>
+<#list [""] as mode>
<#assign lowerName = minor.class?uncap_first />
<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-<#assign name = mode + minor.class?cap_first />
+<#assign name = minor.class?cap_first />
<#assign javaType = (minor.javaType!type.javaType) />
<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
<#assign safeType=friendlyType />
@@ -38,9 +38,9 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
<#assign hasFriendly = minor.friendlyType!"no" == "no" />
-<#list ["", "Nullable"] as nullMode>
-<#if (mode == "Repeated" && nullMode == "") || mode == "" >
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${nullMode}${name}ReaderImpl.java" />
+<#list ["Nullable"] as nullMode>
+<#if mode == "" >
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${name}ReaderImpl.java" />
<#include "/@includes/license.ftl" />
package org.apache.arrow.vector.complex.impl;
@@ -48,20 +48,20 @@ package org.apache.arrow.vector.complex.impl;
<#include "/@includes/vv_imports.ftl" />
@SuppressWarnings("unused")
-public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
+public class ${name}ReaderImpl extends AbstractFieldReader {
private final ${nullMode}${name}Vector vector;
- public ${nullMode}${name}ReaderImpl(${nullMode}${name}Vector vector){
+ public ${name}ReaderImpl(${nullMode}${name}Vector vector){
super();
this.vector = vector;
}
- public MajorType getType(){
- return vector.getField().getType();
+ public MinorType getMinorType(){
+ return vector.getMinorType();
}
- public MaterializedField getField(){
+ public Field getField(){
return vector.getField();
}
@@ -73,50 +73,13 @@ public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
</#if>
}
-
-
-
- <#if mode == "Repeated">
-
- public void copyAsValue(${minor.class?cap_first}Writer writer){
- Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer;
- impl.vector.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public void copyAsField(String name, MapWriter writer){
- Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer.list(name).${lowerName}();
- impl.vector.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public int size(){
- return vector.getAccessor().getInnerValueCountAt(idx());
- }
-
- public void read(int arrayIndex, ${minor.class?cap_first}Holder h){
- vector.getAccessor().get(idx(), arrayIndex, h);
- }
- public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h){
- vector.getAccessor().get(idx(), arrayIndex, h);
- }
-
- public ${friendlyType} read${safeType}(int arrayIndex){
- return vector.getAccessor().getSingleObject(idx(), arrayIndex);
- }
-
-
- public List<Object> readObject(){
- return (List<Object>) (Object) vector.getAccessor().getObject(idx());
- }
-
- <#else>
-
public void copyAsValue(${minor.class?cap_first}Writer writer){
- ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer;
+ ${minor.class?cap_first}WriterImpl impl = (${minor.class?cap_first}WriterImpl) writer;
impl.vector.copyFromSafe(idx(), impl.idx(), vector);
}
public void copyAsField(String name, MapWriter writer){
- ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
+ ${minor.class?cap_first}WriterImpl impl = (${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
impl.vector.copyFromSafe(idx(), impl.idx(), vector);
}
@@ -141,9 +104,6 @@ public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
public Object readObject(){
return vector.getAccessor().getObject(idx());
}
-
-
- </#if>
}
</#if>
</#list>
@@ -156,18 +116,10 @@ package org.apache.arrow.vector.complex.reader;
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{
- <#if mode == "Repeated">
- public int size();
- public void read(int arrayIndex, ${minor.class?cap_first}Holder h);
- public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h);
- public Object readObject(int arrayIndex);
- public ${friendlyType} read${safeType}(int arrayIndex);
- <#else>
public void read(${minor.class?cap_first}Holder h);
public void read(Nullable${minor.class?cap_first}Holder h);
public Object readObject();
public ${friendlyType} read${safeType}();
- </#if>
public boolean isSet();
public void copyAsValue(${minor.class}Writer writer);
public void copyAsField(String name, ${minor.class}Writer writer);
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ComplexWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 8f9a6e7..3457545 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -19,8 +19,8 @@
<@pp.dropOutputFile />
<#list vv.types as type>
<#list type.minor as minor>
-<#list ["", "Nullable", "Repeated"] as mode>
-<#assign name = mode + minor.class?cap_first />
+<#list ["Nullable"] as mode>
+<#assign name = minor.class?cap_first />
<#assign eName = name />
<#assign javaType = (minor.javaType!type.javaType) />
<#assign fields = minor.fields!type.fields />
@@ -38,17 +38,16 @@ package org.apache.arrow.vector.complex.impl;
@SuppressWarnings("unused")
public class ${eName}WriterImpl extends AbstractFieldWriter {
- private final ${name}Vector.Mutator mutator;
- final ${name}Vector vector;
+ private final Nullable${name}Vector.Mutator mutator;
+ final Nullable${name}Vector vector;
- public ${eName}WriterImpl(${name}Vector vector, AbstractFieldWriter parent) {
- super(parent);
+ public ${eName}WriterImpl(Nullable${name}Vector vector) {
this.mutator = vector.getMutator();
this.vector = vector;
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return vector.getField();
}
@@ -89,12 +88,10 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
vector.getMutator().setValueCount(idx()+1);
}
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
vector.getMutator().setValueCount(idx()+1);
}
- </#if>
public void setPosition(int idx) {
super.setPosition(idx);
@@ -114,11 +111,17 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
vector.getMutator().setValueCount(idx()+1);
}
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+ <#if minor.class == "Decimal">
+ public void writeDecimal(int start, ArrowBuf buffer) {
+ mutator.setSafe(idx(), 1, start, buffer);
+ vector.getMutator().setValueCount(idx()+1);
+ }
+ <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- mutator.setSafe(idx(), <#if mode == "Nullable">1, </#if><#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
vector.getMutator().setValueCount(idx()+1);
}
+ </#if>
<#if mode == "Nullable">
@@ -128,7 +131,6 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
}
</#if>
</#if>
- </#if>
}
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/writer/${eName}Writer.java" />
@@ -141,7 +143,9 @@ package org.apache.arrow.vector.complex.writer;
public interface ${eName}Writer extends BaseWriter {
public void write(${minor.class}Holder h);
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+ <#if minor.class == "Decimal">
+ public void writeDecimal(int start, ArrowBuf buffer);
+ <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
</#if>
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 18fcac9..fe2b5c5 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -43,20 +43,42 @@ package org.apache.arrow.vector;
public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
- private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
private int allocationMonitor = 0;
- public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
+ <#if minor.class == "Decimal">
+
+ private int precision;
+ private int scale;
+
+ public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) {
+ super(name, allocator);
+ this.precision = precision;
+ this.scale = scale;
+ }
+ <#else>
+ public ${minor.class}Vector(String name, BufferAllocator allocator) {
+ super(name, allocator);
+ }
+ </#if>
+
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.${minor.class?upper_case};
+ }
+
+ @Override
+ public Field getField() {
+ throw new UnsupportedOperationException("internal vector");
}
@Override
public FieldReader getReader(){
- return reader;
+ throw new UnsupportedOperationException("non-nullable vectors cannot be used in readers");
}
@Override
@@ -162,7 +184,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
}
- logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize);
+ logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", name, allocationSizeInBytes, newAllocationSize);
final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
newBuf.setBytes(0, data, 0, data.capacity());
final int halfNewCapacity = newBuf.capacity() / 2;
@@ -181,30 +203,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
data.setZero(0, data.capacity());
}
-// @Override
-// public void load(SerializedField metadata, ArrowBuf buffer) {
-// Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
-// final int actualLength = metadata.getBufferLength();
-// final int valueCount = metadata.getValueCount();
-// final int expectedLength = valueCount * ${type.width};
-// assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength);
-//
-// clear();
-// if (data != null) {
-// data.release(1);
-// }
-// data = buffer.slice(0, actualLength);
-// data.retain(1);
-// data.writerIndex(actualLength);
-// }
-
public TransferPair getTransferPair(BufferAllocator allocator){
- return new TransferImpl(getField(), allocator);
+ return new TransferImpl(name, allocator);
}
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator){
- return new TransferImpl(getField().withPath(ref), allocator);
+ return new TransferImpl(ref, allocator);
}
@Override
@@ -230,8 +235,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
private class TransferImpl implements TransferPair{
private ${minor.class}Vector to;
- public TransferImpl(MaterializedField field, BufferAllocator allocator){
- to = new ${minor.class}Vector(field, allocator);
+ public TransferImpl(String name, BufferAllocator allocator){
+ <#if minor.class == "Decimal">
+ to = new ${minor.class}Vector(name, allocator, precision, scale);
+ <#else>
+ to = new ${minor.class}Vector(name, allocator);
+ </#if>
}
public TransferImpl(${minor.class}Vector to) {
@@ -260,7 +269,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
- <#if (type.width > 8)>
+ <#if (type.width > 8 || minor.class == "IntervalDay")>
from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
<#else> <#-- type.width <= 8 -->
data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width},
@@ -298,7 +307,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
return false;
}
- <#if (type.width > 8)>
+ <#if (type.width > 8 || minor.class == "IntervalDay")>
public ${minor.javaType!type.javaType} get(int index) {
return data.slice(index * ${type.width}, ${type.width});
@@ -416,31 +425,30 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
append(millis));
}
- <#elseif (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+ <#elseif minor.class == "Decimal">
public void get(int index, ${minor.class}Holder holder) {
holder.start = index * ${type.width};
holder.buffer = data;
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
+ holder.scale = scale;
+ holder.precision = precision;
}
public void get(int index, Nullable${minor.class}Holder holder) {
holder.isSet = 1;
holder.start = index * ${type.width};
holder.buffer = data;
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
+ holder.scale = scale;
+ holder.precision = precision;
}
- @Override
- public ${friendlyType} getObject(int index) {
- <#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")>
- // Get the BigDecimal object
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
- <#else>
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
- </#if>
+ @Override
+ public ${friendlyType} getObject(int index) {
+ byte[] bytes = new byte[${type.width}];
+ int start = ${type.width} * index;
+ data.getBytes(start, bytes, 0, ${type.width});
+ ${friendlyType} value = new BigDecimal(new BigInteger(bytes), scale);
+ return value;
}
<#else>
@@ -581,7 +589,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
* @param index position of the bit to set
* @param value value to set
*/
- <#if (type.width > 8)>
+ <#if (type.width > 8) || minor.class == "IntervalDay">
public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
data.setBytes(index * ${type.width}, value, 0, ${type.width});
}
@@ -653,7 +661,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
setSafe(index, holder.days, holder.milliseconds);
}
- <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+ <#elseif minor.class == "Decimal">
public void set(int index, ${minor.class}Holder holder){
set(index, holder.start, holder.buffer);
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index 3005fca..1ed9287 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -19,9 +19,8 @@
<@pp.dropOutputFile />
<#list vv.types as type>
<#list type.minor as minor>
-<#list ["", "Nullable", "Repeated"] as holderMode>
+<#list ["", "Nullable"] as holderMode>
<#assign nullMode = holderMode />
-<#if holderMode == "Repeated"><#assign nullMode = "Nullable" /></#if>
<#assign lowerName = minor.class?uncap_first />
<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
@@ -50,42 +49,18 @@ import org.joda.time.Period;
public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
private ${nullMode}${name}Holder holder;
-<#if holderMode == "Repeated" >
- private int index = -1;
- private ${holderMode}${name}Holder repeatedHolder;
-</#if>
-
public ${holderMode}${name}HolderReaderImpl(${holderMode}${name}Holder holder) {
-<#if holderMode == "Repeated" >
- this.holder = new ${nullMode}${name}Holder();
- this.repeatedHolder = holder;
-<#else>
this.holder = holder;
-</#if>
}
@Override
public int size() {
-<#if holderMode == "Repeated">
- return repeatedHolder.end - repeatedHolder.start;
-<#else>
throw new UnsupportedOperationException("You can't call size on a Holder value reader.");
-</#if>
}
@Override
public boolean next() {
-<#if holderMode == "Repeated">
- if(index + 1 < repeatedHolder.end) {
- index++;
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
- return true;
- } else {
- return false;
- }
-<#else>
throw new UnsupportedOperationException("You can't call next on a single value reader.");
-</#if>
}
@@ -95,19 +70,13 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
}
@Override
- public MajorType getType() {
-<#if holderMode == "Repeated">
- return this.repeatedHolder.TYPE;
-<#else>
- return this.holder.TYPE;
-</#if>
+ public MinorType getMinorType() {
+ return MinorType.${name?upper_case};
}
@Override
public boolean isSet() {
- <#if holderMode == "Repeated">
- return this.repeatedHolder.end!=this.repeatedHolder.start;
- <#elseif nullMode == "Nullable">
+ <#if holderMode == "Nullable">
return this.holder.isSet == 1;
<#else>
return true;
@@ -115,7 +84,6 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
}
-<#if holderMode != "Repeated">
@Override
public void read(${name}Holder h) {
<#list fields as field>
@@ -130,19 +98,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
</#list>
h.isSet = isSet() ? 1 : 0;
}
-</#if>
-<#if holderMode == "Repeated">
- @Override
- public ${friendlyType} read${safeType}(int index){
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
- ${friendlyType} value = read${safeType}();
- if (this.index > -1) {
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + this.index, holder);
- }
- return value;
- }
-</#if>
@Override
public ${friendlyType} read${safeType}(){
@@ -176,29 +132,10 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
Period p = new Period();
return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-<#elseif minor.class == "Decimal9" ||
- minor.class == "Decimal18" >
- BigInteger value = BigInteger.valueOf(holder.value);
- return new BigDecimal(value, holder.scale);
-
-<#elseif minor.class == "Decimal28Dense" ||
- minor.class == "Decimal38Dense">
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale,
- holder.maxPrecision,
- holder.WIDTH);
-
-<#elseif minor.class == "Decimal28Sparse" ||
- minor.class == "Decimal38Sparse">
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale);
-
<#elseif minor.class == "Bit" >
return new Boolean(holder.value != 0);
+<#elseif minor.class == "Decimal" >
+ return (BigDecimal) readSingleObject();
<#else>
${friendlyType} value = new ${friendlyType}(this.holder.value);
return value;
@@ -208,15 +145,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
@Override
public Object readObject() {
-<#if holderMode == "Repeated" >
- List<Object> valList = Lists.newArrayList();
- for (int i = repeatedHolder.start; i < repeatedHolder.end; i++) {
- valList.add(repeatedHolder.vector.getAccessor().getObject(i));
- }
- return valList;
-<#else>
return readSingleObject();
-</#if>
}
private Object readSingleObject() {
@@ -239,6 +168,9 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
Text text = new Text();
text.set(value);
return text;
+<#elseif minor.class == "Decimal" >
+ return new BigDecimal(new BigInteger(value), holder.scale);
+
</#if>
<#elseif minor.class == "Interval">
@@ -249,11 +181,6 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
Period p = new Period();
return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-<#elseif minor.class == "Decimal9" ||
- minor.class == "Decimal18" >
- BigInteger value = BigInteger.valueOf(holder.value);
- return new BigDecimal(value, holder.scale);
-
<#elseif minor.class == "Decimal28Dense" ||
minor.class == "Decimal38Dense">
return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
@@ -272,13 +199,18 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#elseif minor.class == "Bit" >
return new Boolean(holder.value != 0);
+<#elseif minor.class == "Decimal">
+ byte[] bytes = new byte[${type.width}];
+ holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
+ ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
+ return value;
<#else>
${friendlyType} value = new ${friendlyType}(this.holder.value);
return value;
</#if>
}
-<#if holderMode != "Repeated" && nullMode != "Nullable">
+<#if nullMode != "Nullable">
public void copyAsValue(${minor.class?cap_first}Writer writer){
writer.write(holder);
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ListWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ListWriters.java b/java/vector/src/main/codegen/templates/ListWriters.java
deleted file mode 100644
index 94b812b..0000000
--- a/java/vector/src/main/codegen/templates/ListWriters.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-
-<#list ["Single", "Repeated"] as mode>
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}ListWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector.complex.impl;
-<#if mode == "Single">
- <#assign containerClass = "AbstractContainerVector" />
- <#assign index = "idx()">
-<#else>
- <#assign containerClass = "RepeatedListVector" />
- <#assign index = "currentChildIndex">
-</#if>
-
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using FreeMarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-public class ${mode}ListWriter extends AbstractFieldWriter {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${mode}ListWriter.class);
-
- static enum Mode { INIT, IN_MAP, IN_LIST <#list vv.types as type><#list type.minor as minor>, IN_${minor.class?upper_case}</#list></#list> }
-
- private final String name;
- protected final ${containerClass} container;
- private Mode mode = Mode.INIT;
- private FieldWriter writer;
- protected RepeatedValueVector innerVector;
-
- <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
- public ${mode}ListWriter(String name, ${containerClass} container, FieldWriter parent){
- super(parent);
- this.name = name;
- this.container = container;
- }
-
- public ${mode}ListWriter(${containerClass} container, FieldWriter parent){
- super(parent);
- this.name = null;
- this.container = container;
- }
-
- @Override
- public void allocate() {
- if(writer != null) {
- writer.allocate();
- }
-
- <#if mode == "Repeated">
- container.allocateNew();
- </#if>
- }
-
- @Override
- public void clear() {
- if (writer != null) {
- writer.clear();
- }
- }
-
- @Override
- public void close() {
- clear();
- container.close();
- if (innerVector != null) {
- innerVector.close();
- }
- }
-
- @Override
- public int getValueCapacity() {
- return innerVector == null ? 0 : innerVector.getValueCapacity();
- }
-
- public void setValueCount(int count){
- if(innerVector != null) innerVector.getMutator().setValueCount(count);
- }
-
- @Override
- public MapWriter map() {
- switch(mode) {
- case INIT:
- int vectorCount = container.size();
- final RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class);
- innerVector = vector;
- writer = new RepeatedMapWriter(vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_MAP;
- return writer;
- case IN_MAP:
- return writer;
- }
-
- throw new RuntimeException(getUnsupportedErrorMsg("MAP", mode.name()));
-
- }
-
- @Override
- public ListWriter list() {
- switch(mode) {
- case INIT:
- final int vectorCount = container.size();
- final RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class);
- innerVector = vector;
- writer = new RepeatedListWriter(null, vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_LIST;
- return writer;
- case IN_LIST:
- return writer;
- }
-
- throw new RuntimeException(getUnsupportedErrorMsg("LIST", mode.name()));
-
- }
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-
- private static final MajorType ${upperName}_TYPE = Types.repeated(MinorType.${upperName});
-
- @Override
- public ${capName}Writer ${lowerName}() {
- switch(mode) {
- case INIT:
- final int vectorCount = container.size();
- final Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class);
- innerVector = vector;
- writer = new Repeated${capName}WriterImpl(vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_${upperName};
- return writer;
- case IN_${upperName}:
- return writer;
- }
-
- throw new RuntimeException(getUnsupportedErrorMsg("${upperName}", mode.name()));
-
- }
- </#list></#list>
-
- public MaterializedField getField() {
- return container.getField();
- }
-
- <#if mode == "Repeated">
-
- public void startList() {
- final RepeatedListVector list = (RepeatedListVector) container;
- final RepeatedListVector.RepeatedMutator mutator = list.getMutator();
-
- // make sure that the current vector can support the end position of this list.
- if(container.getValueCapacity() <= idx()) {
- mutator.setValueCount(idx()+1);
- }
-
- // update the repeated vector to state that there is current+1 objects.
- final RepeatedListHolder h = new RepeatedListHolder();
- list.getAccessor().get(idx(), h);
- if (h.start >= h.end) {
- mutator.startNewValue(idx());
- }
- currentChildIndex = container.getMutator().add(idx());
- if(writer != null) {
- writer.setPosition(currentChildIndex);
- }
- }
-
- public void endList() {
- // noop, we initialize state at start rather than end.
- }
- <#else>
-
- public void setPosition(int index) {
- super.setPosition(index);
- if(writer != null) {
- writer.setPosition(index);
- }
- }
-
- public void startList() {
- // noop
- }
-
- public void endList() {
- // noop
- }
- </#if>
-
- private String getUnsupportedErrorMsg(String expected, String found) {
- final String f = found.substring(3);
- return String.format("In a list of type %s, encountered a value of type %s. "+
- "Arrow does not support lists of different types.",
- f, expected
- );
- }
-}
-</#list>
[2/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead
of MaterializedField
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
index f16f628..b8f5865 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleListReaderImpl.java
@@ -24,14 +24,11 @@ import org.apache.arrow.vector.complex.AbstractContainerVector;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
@SuppressWarnings("unused")
public class SingleListReaderImpl extends AbstractFieldReader{
- private static final MajorType TYPE = Types.optional(MinorType.LIST);
private final String name;
private final AbstractContainerVector container;
private FieldReader reader;
@@ -43,12 +40,6 @@ public class SingleListReaderImpl extends AbstractFieldReader{
}
@Override
- public MajorType getType() {
- return TYPE;
- }
-
-
- @Override
public void setPosition(int index) {
super.setPosition(index);
if (reader != null) {
@@ -71,6 +62,11 @@ public class SingleListReaderImpl extends AbstractFieldReader{
}
@Override
+ public MinorType getMinorType() {
+ return MinorType.LIST;
+ }
+
+ @Override
public boolean isSet() {
return false;
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index 84b9980..1c43240 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -27,9 +27,9 @@ import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types.MajorType;
import com.google.common.collect.Maps;
+import org.apache.arrow.vector.types.Types.MinorType;
@SuppressWarnings("unused")
public class SingleMapReaderImpl extends AbstractFieldReader{
@@ -77,13 +77,13 @@ public class SingleMapReaderImpl extends AbstractFieldReader{
}
@Override
- public boolean isSet() {
- return true;
+ public MinorType getMinorType() {
+ return MinorType.MAP;
}
@Override
- public MajorType getType(){
- return vector.getField().getType();
+ public boolean isSet() {
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
index 9b54d02..39cf004 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java
@@ -25,8 +25,6 @@ import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.holders.UnionHolder;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
public class UnionListReader extends AbstractFieldReader {
@@ -46,12 +44,6 @@ public class UnionListReader extends AbstractFieldReader {
return true;
}
- MajorType type = new MajorType(MinorType.LIST, DataMode.OPTIONAL);
-
- public MajorType getType() {
- return type;
- }
-
private int currentOffset;
private int maxOffset;
@@ -73,6 +65,11 @@ public class UnionListReader extends AbstractFieldReader {
}
@Override
+ public MinorType getMinorType() {
+ return MinorType.LIST;
+ }
+
+ @Override
public void read(int index, UnionHolder holder) {
setPosition(idx());
for (int i = -1; i < index; i++) {
@@ -83,6 +80,12 @@ public class UnionListReader extends AbstractFieldReader {
}
@Override
+ public int size() {
+ int size = maxOffset - currentOffset - 1;
+ return size < 0 ? 0 : size;
+ }
+
+ @Override
public boolean next() {
if (currentOffset + 1 < maxOffset) {
data.getReader().setPosition(++currentOffset);
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
deleted file mode 100644
index 5a5fe03..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/ObjectHolder.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.holders;
-
-import org.apache.arrow.vector.types.Types;
-
-/*
- * Holder class for the vector ObjectVector. This holder internally stores a
- * reference to an object. The ObjectVector maintains an array of these objects.
- * This holder can be used only as workspace variables in aggregate functions.
- * Using this holder should be avoided and we should stick to native holder types.
- */
-@Deprecated
-public class ObjectHolder implements ValueHolder {
- public static final Types.MajorType TYPE = Types.required(Types.MinorType.GENERIC_OBJECT);
-
- public Types.MajorType getType() {
- return TYPE;
- }
-
- public Object obj;
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java b/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
index b868a62..b1b695e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/holders/UnionHolder.java
@@ -18,17 +18,14 @@
package org.apache.arrow.vector.holders;
import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
public class UnionHolder implements ValueHolder {
- public static final MajorType TYPE = new MajorType(MinorType.UNION, DataMode.OPTIONAL);
public FieldReader reader;
public int isSet;
- public MajorType getType() {
- return reader.getType();
+ public MinorType getMinorType() {
+ return reader.getMinorType();
}
public boolean isSet() {
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java b/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
deleted file mode 100644
index c73098b..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/MaterializedField.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.types;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Objects;
-
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
-
-
-public class MaterializedField {
- private final String name;
- private final MajorType type;
- // use an ordered set as existing code relies on order (e,g. parquet writer)
- private final LinkedHashSet<MaterializedField> children;
-
- MaterializedField(String name, MajorType type, LinkedHashSet<MaterializedField> children) {
- this.name = name;
- this.type = type;
- this.children = children;
- }
-
- public Collection<MaterializedField> getChildren() {
- return new ArrayList<>(children);
- }
-
- public MaterializedField newWithChild(MaterializedField child) {
- MaterializedField newField = clone();
- newField.addChild(child);
- return newField;
- }
-
- public void addChild(MaterializedField field){
- children.add(field);
- }
-
- public MaterializedField clone() {
- return withPathAndType(name, getType());
- }
-
- public MaterializedField withType(MajorType type) {
- return withPathAndType(name, type);
- }
-
- public MaterializedField withPath(String name) {
- return withPathAndType(name, getType());
- }
-
- public MaterializedField withPathAndType(String name, final MajorType type) {
- final LinkedHashSet<MaterializedField> newChildren = new LinkedHashSet<>(children.size());
- for (final MaterializedField child:children) {
- newChildren.add(child.clone());
- }
- return new MaterializedField(name, type, newChildren);
- }
-
-// public String getLastName(){
-// PathSegment seg = key.path.getRootSegment();
-// while (seg.getChild() != null) {
-// seg = seg.getChild();
-// }
-// return seg.getNameSegment().getPath();
-// }
-
-
- // TODO: rewrite without as direct match rather than conversion then match.
-// public boolean matches(SerializedField booleanfield){
-// MaterializedField f = create(field);
-// return f.equals(this);
-// }
-
- public static MaterializedField create(String name, MajorType type){
- return new MaterializedField(name, type, new LinkedHashSet<MaterializedField>());
- }
-
-// public String getName(){
-// StringBuilder sb = new StringBuilder();
-// boolean first = true;
-// for(NamePart np : def.getNameList()){
-// if(np.getType() == Type.ARRAY){
-// sb.append("[]");
-// }else{
-// if(first){
-// first = false;
-// }else{
-// sb.append(".");
-// }
-// sb.append('`');
-// sb.append(np.getName());
-// sb.append('`');
-//
-// }
-// }
-// return sb.toString();
-// }
-
- public String getPath() {
- return getName();
- }
-
- public String getLastName() {
- return getName();
- }
-
- public String getName() {
- return name;
- }
-
-// public int getWidth() {
-// return type.getWidth();
-// }
-
- public MajorType getType() {
- return type;
- }
-
- public int getScale() {
- return type.getScale();
- }
- public int getPrecision() {
- return type.getPrecision();
- }
- public boolean isNullable() {
- return type.getMode() == DataMode.OPTIONAL;
- }
-
- public DataMode getDataMode() {
- return type.getMode();
- }
-
- public MaterializedField getOtherNullableVersion(){
- MajorType mt = type;
- DataMode newDataMode = null;
- switch (mt.getMode()){
- case OPTIONAL:
- newDataMode = DataMode.REQUIRED;
- break;
- case REQUIRED:
- newDataMode = DataMode.OPTIONAL;
- break;
- default:
- throw new UnsupportedOperationException();
- }
- return new MaterializedField(name, new MajorType(mt.getMinorType(), newDataMode, mt.getPrecision(), mt.getScale(), mt.getTimezone(), mt.getSubTypes()), children);
- }
-
- public Class<?> getValueClass() {
- return BasicTypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(this.name, this.type, this.children);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- MaterializedField other = (MaterializedField) obj;
- // DRILL-1872: Compute equals only on key. See also the comment
- // in MapVector$MapTransferPair
-
- return this.name.equalsIgnoreCase(other.name) &&
- Objects.equals(this.type, other.type);
- }
-
-
- @Override
- public String toString() {
- final int maxLen = 10;
- String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : "";
- return name + "(" + type.getMinorType().name() + ":" + type.getMode().name() + ")" + childStr;
- }
-
-
- private String toString(Collection<?> collection, int maxLen) {
- StringBuilder builder = new StringBuilder();
- builder.append("[");
- int i = 0;
- for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
- if (i > 0){
- builder.append(", ");
- }
- builder.append(iterator.next());
- }
- builder.append("]");
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 88999cb..5ea1456 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -17,150 +17,508 @@
*/
package org.apache.arrow.vector.types;
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.NullableBigIntVector;
+import org.apache.arrow.vector.NullableBitVector;
+import org.apache.arrow.vector.NullableDateVector;
+import org.apache.arrow.vector.NullableDecimalVector;
+import org.apache.arrow.vector.NullableFloat4Vector;
+import org.apache.arrow.vector.NullableFloat8Vector;
+import org.apache.arrow.vector.NullableIntVector;
+import org.apache.arrow.vector.NullableIntervalDayVector;
+import org.apache.arrow.vector.NullableIntervalYearVector;
+import org.apache.arrow.vector.NullableSmallIntVector;
+import org.apache.arrow.vector.NullableTimeStampVector;
+import org.apache.arrow.vector.NullableTimeVector;
+import org.apache.arrow.vector.NullableTinyIntVector;
+import org.apache.arrow.vector.NullableUInt1Vector;
+import org.apache.arrow.vector.NullableUInt2Vector;
+import org.apache.arrow.vector.NullableUInt4Vector;
+import org.apache.arrow.vector.NullableUInt8Vector;
+import org.apache.arrow.vector.NullableVarBinaryVector;
+import org.apache.arrow.vector.NullableVarCharVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
+import org.apache.arrow.vector.complex.impl.BitWriterImpl;
+import org.apache.arrow.vector.complex.impl.DateWriterImpl;
+import org.apache.arrow.vector.complex.impl.Float4WriterImpl;
+import org.apache.arrow.vector.complex.impl.Float8WriterImpl;
+import org.apache.arrow.vector.complex.impl.IntWriterImpl;
+import org.apache.arrow.vector.complex.impl.IntervalDayWriterImpl;
+import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl;
+import org.apache.arrow.vector.complex.impl.SingleMapWriter;
+import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeWriterImpl;
+import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl;
+import org.apache.arrow.vector.complex.impl.UInt1WriterImpl;
+import org.apache.arrow.vector.complex.impl.UInt2WriterImpl;
+import org.apache.arrow.vector.complex.impl.UInt4WriterImpl;
+import org.apache.arrow.vector.complex.impl.UInt8WriterImpl;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.impl.UnionWriter;
+import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl;
+import org.apache.arrow.vector.complex.impl.VarCharWriterImpl;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
+import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
+import org.apache.arrow.vector.types.pojo.ArrowType.Date;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.IntervalDay;
+import org.apache.arrow.vector.types.pojo.ArrowType.IntervalYear;
+import org.apache.arrow.vector.types.pojo.ArrowType.List;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.ArrowType.Time;
+import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.CallBack;
+
+import java.util.HashMap;
import java.util.Map;
-import java.util.Objects;
public class Types {
+
+ public static final Field NULL_FIELD = new Field("", true, Null.INSTANCE, null);
+ public static final Field TINYINT_FIELD = new Field("", true, new Int(8, true), null);
+ public static final Field SMALLINT_FIELD = new Field("", true, new Int(16, true), null);
+ public static final Field INT_FIELD = new Field("", true, new Int(32, true), null);
+ public static final Field BIGINT_FIELD = new Field("", true, new Int(64, true), null);
+ public static final Field UINT1_FIELD = new Field("", true, new Int(8, false), null);
+ public static final Field UINT2_FIELD = new Field("", true, new Int(16, false), null);
+ public static final Field UINT4_FIELD = new Field("", true, new Int(32, false), null);
+ public static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
+ public static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
+ public static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
+ public static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null);
+ public static final Field INTERVALDAY_FIELD = new Field("", true, IntervalDay.INSTANCE, null);
+ public static final Field INTERVALYEAR_FIELD = new Field("", true, IntervalYear.INSTANCE, null);
+ public static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(0), null);
+ public static final Field FLOAT8_FIELD = new Field("", true, new FloatingPoint(1), null);
+ public static final Field LIST_FIELD = new Field("", true, List.INSTANCE, null);
+ public static final Field VARCHAR_FIELD = new Field("", true, Utf8.INSTANCE, null);
+ public static final Field VARBINARY_FIELD = new Field("", true, Binary.INSTANCE, null);
+ public static final Field BIT_FIELD = new Field("", true, Bool.INSTANCE, null);
+
+
public enum MinorType {
- LATE, // late binding type
- MAP, // an empty map column. Useful for conceptual setup. Children listed within here
-
- TINYINT, // single byte signed integer
- SMALLINT, // two byte signed integer
- INT, // four byte signed integer
- BIGINT, // eight byte signed integer
- DECIMAL9, // a decimal supporting precision between 1 and 9
- DECIMAL18, // a decimal supporting precision between 10 and 18
- DECIMAL28SPARSE, // a decimal supporting precision between 19 and 28
- DECIMAL38SPARSE, // a decimal supporting precision between 29 and 38
- MONEY, // signed decimal with two digit precision
- DATE, // days since 4713bc
- TIME, // time in micros before or after 2000/1/1
- TIMETZ, // time in micros before or after 2000/1/1 with timezone
- TIMESTAMPTZ, // unix epoch time in millis
- TIMESTAMP, // TBD
- INTERVAL, // TBD
- FLOAT4, // 4 byte ieee 754
- FLOAT8, // 8 byte ieee 754
- BIT, // single bit value (boolean)
- FIXEDCHAR, // utf8 fixed length string, padded with spaces
- FIXED16CHAR,
- FIXEDBINARY, // fixed length binary, padded with 0 bytes
- VARCHAR, // utf8 variable length string
- VAR16CHAR, // utf16 variable length string
- VARBINARY, // variable length binary
- UINT1, // unsigned 1 byte integer
- UINT2, // unsigned 2 byte integer
- UINT4, // unsigned 4 byte integer
- UINT8, // unsigned 8 byte integer
- DECIMAL28DENSE, // dense decimal representation, supporting precision between 19 and 28
- DECIMAL38DENSE, // dense decimal representation, supporting precision between 28 and 38
- NULL, // a value of unknown type (e.g. a missing reference).
- INTERVALYEAR, // Interval type specifying YEAR to MONTH
- INTERVALDAY, // Interval type specifying DAY to SECONDS
- LIST,
- GENERIC_OBJECT,
- UNION
- }
+ NULL(Null.INSTANCE) {
+ @Override
+ public Field getField() {
+ return NULL_FIELD;
+ }
- public enum DataMode {
- REQUIRED,
- OPTIONAL,
- REPEATED
- }
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return ZeroVector.INSTANCE;
+ }
- public static class MajorType {
- private MinorType minorType;
- private DataMode mode;
- private int precision;
- private int scale;
- private int timezone;
- private int width;
- private List<MinorType> subTypes;
-
- public MajorType(MinorType minorType, DataMode mode) {
- this(minorType, mode, 0, 0, 0, 0, null);
- }
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return null;
+ }
+ },
+ MAP(Tuple.INSTANCE) {
+ @Override
+ public Field getField() {
+ throw new UnsupportedOperationException("Cannot get simple field for Map type");
+ }
- public MajorType(MinorType minorType, DataMode mode, int precision, int scale) {
- this(minorType, mode, precision, scale, 0, 0, null);
- }
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new MapVector(name, allocator, callBack);
+ }
- public MajorType(MinorType minorType, DataMode mode, int precision, int scale, int timezone, List<MinorType> subTypes) {
- this(minorType, mode, precision, scale, timezone, 0, subTypes);
- }
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new SingleMapWriter((MapVector) vector);
+ }
+ }, // an empty map column. Useful for conceptual setup. Children listed within here
- public MajorType(MinorType minorType, DataMode mode, int precision, int scale, int timezone, int width, List<MinorType> subTypes) {
- this.minorType = minorType;
- this.mode = mode;
- this.precision = precision;
- this.scale = scale;
- this.timezone = timezone;
- this.width = width;
- this.subTypes = subTypes;
- if (subTypes == null) {
- this.subTypes = new ArrayList<>();
+ TINYINT(new Int(8, true)) {
+ @Override
+ public Field getField() {
+ return TINYINT_FIELD;
}
- }
- public MinorType getMinorType() {
- return minorType;
- }
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableTinyIntVector(name, allocator);
+ }
- public DataMode getMode() {
- return mode;
- }
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new TinyIntWriterImpl((NullableTinyIntVector) vector);
+ }
+ }, // single byte signed integer
+ SMALLINT(new Int(16, true)) {
+ @Override
+ public Field getField() {
+ return SMALLINT_FIELD;
+ }
- public int getPrecision() {
- return precision;
- }
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new SmallIntVector(name, allocator);
+ }
- public int getScale() {
- return scale;
- }
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new SmallIntWriterImpl((NullableSmallIntVector) vector);
+ }
+ }, // two byte signed integer
+ INT(new Int(32, true)) {
+ @Override
+ public Field getField() {
+ return INT_FIELD;
+ }
- public int getTimezone() {
- return timezone;
- }
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableIntVector(name, allocator);
+ }
- public List<MinorType> getSubTypes() {
- return subTypes;
- }
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new IntWriterImpl((NullableIntVector) vector);
+ }
+ }, // four byte signed integer
+ BIGINT(new Int(64, true)) {
+ @Override
+ public Field getField() {
+ return BIGINT_FIELD;
+ }
- public int getWidth() {
- return width;
- }
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableBigIntVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new BigIntWriterImpl((NullableBigIntVector) vector);
+ }
+ }, // eight byte signed integer
+ DATE(Date.INSTANCE) {
+ @Override
+ public Field getField() {
+ return DATE_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableDateVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new DateWriterImpl((NullableDateVector) vector);
+ }
+ }, // days since 4713bc
+ TIME(Time.INSTANCE) {
+ @Override
+ public Field getField() {
+ return TIME_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableTimeVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new TimeWriterImpl((NullableTimeVector) vector);
+ }
+ }, // time in micros before or after 2000/1/1
+ TIMESTAMP(new Timestamp("")) {
+ @Override
+ public Field getField() {
+ return TIMESTAMP_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableTimeStampVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new TimeStampWriterImpl((NullableTimeStampVector) vector);
+ }
+ },
+ INTERVALDAY(IntervalDay.INSTANCE) {
+ @Override
+ public Field getField() {
+ return INTERVALDAY_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableIntervalDayVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new IntervalDayWriterImpl((NullableIntervalDayVector) vector);
+ }
+ },
+ INTERVALYEAR(IntervalYear.INSTANCE) {
+ @Override
+ public Field getField() {
+ return INTERVALYEAR_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableIntervalDayVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new IntervalYearWriterImpl((NullableIntervalYearVector) vector);
+ }
+ },
+ FLOAT4(new FloatingPoint(0)) {
+ @Override
+ public Field getField() {
+ return FLOAT4_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableFloat4Vector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new Float4WriterImpl((NullableFloat4Vector) vector);
+ }
+ }, // 4 byte ieee 754
+ FLOAT8(new FloatingPoint(1)) {
+ @Override
+ public Field getField() {
+ return FLOAT8_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableFloat8Vector(name, allocator);
+ }
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new Float8WriterImpl((NullableFloat8Vector) vector);
+ }
+ }, // 8 byte ieee 754
+ BIT(Bool.INSTANCE) {
+ @Override
+ public Field getField() {
+ return BIT_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableBitVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new BitWriterImpl((NullableBitVector) vector);
+ }
+ }, // single bit value (boolean)
+ VARCHAR(Utf8.INSTANCE) {
+ @Override
+ public Field getField() {
+ return VARCHAR_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableVarCharVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new VarCharWriterImpl((NullableVarCharVector) vector);
+ }
+ }, // utf8 variable length string
+ VARBINARY(Binary.INSTANCE) {
+ @Override
+ public Field getField() {
+ return VARBINARY_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableVarBinaryVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new VarBinaryWriterImpl((NullableVarBinaryVector) vector);
+ }
+ }, // variable length binary
+ DECIMAL(null) {
+ @Override
+ public ArrowType getType() {
+ throw new UnsupportedOperationException("Cannot get simple type for Decimal type");
+ }
+ @Override
+ public Field getField() {
+ throw new UnsupportedOperationException("Cannot get simple field for Decimal type");
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableDecimalVector(name, allocator, precisionScale[0], precisionScale[1]);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new VarBinaryWriterImpl((NullableVarBinaryVector) vector);
+ }
+ }, // variable length binary
+ UINT1(new Int(8, false)) {
+ @Override
+ public Field getField() {
+ return UINT1_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableUInt1Vector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new UInt1WriterImpl((NullableUInt1Vector) vector);
+ }
+ }, // unsigned 1 byte integer
+ UINT2(new Int(16, false)) {
+ @Override
+ public Field getField() {
+ return UINT2_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableUInt2Vector(name, allocator);
+ }
- @Override
- public boolean equals(Object other) {
- if (other == null) {
- return false;
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new UInt2WriterImpl((NullableUInt2Vector) vector);
}
- if (!(other instanceof MajorType)) {
- return false;
+ }, // unsigned 2 byte integer
+ UINT4(new Int(32, false)) {
+ @Override
+ public Field getField() {
+ return UINT8_FIELD;
}
- MajorType that = (MajorType) other;
- return this.minorType == that.minorType &&
- this.mode == that.mode &&
- this.precision == that.precision &&
- this.scale == that.scale &&
- this.timezone == that.timezone &&
- this.width == that.width &&
- Objects.equals(this.subTypes, that.subTypes);
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableUInt4Vector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new UInt4WriterImpl((NullableUInt4Vector) vector);
+ }
+ }, // unsigned 4 byte integer
+ UINT8(new Int(64, false)) {
+ @Override
+ public Field getField() {
+ return UINT8_FIELD;
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableUInt8Vector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new UInt8WriterImpl((NullableUInt8Vector) vector);
+ }
+ }, // unsigned 8 byte integer
+ LIST(List.INSTANCE) {
+ @Override
+ public Field getField() {
+ throw new UnsupportedOperationException("Cannot get simple field for List type");
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new ListVector(name, allocator, callBack);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new UnionListWriter((ListVector) vector);
+ }
+ },
+ UNION(Union.INSTANCE) {
+ @Override
+ public Field getField() {
+ throw new UnsupportedOperationException("Cannot get simple field for Union type");
+ }
+
+ @Override
+ public ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new UnionVector(name, allocator, callBack);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new UnionWriter((UnionVector) vector);
+ }
+ };
+
+ private final ArrowType type;
+
+ MinorType(ArrowType type) {
+ this.type = type;
}
- }
+ public ArrowType getType() {
+ return type;
+ }
+
+ public abstract Field getField();
- public static MajorType required(MinorType minorType) {
- return new MajorType(minorType, DataMode.REQUIRED);
+ public abstract ValueVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale);
+
+ public abstract FieldWriter getNewFieldWriter(ValueVector vector);
}
- public static MajorType optional(MinorType minorType) {
- return new MajorType(minorType, DataMode.OPTIONAL);
+
+ private static final Map<ArrowType,MinorType> ARROW_TYPE_MINOR_TYPE_MAP;
+
+ public static MinorType getMinorTypeForArrowType(ArrowType arrowType) {
+ if (arrowType.getTypeType() == Type.Decimal) {
+ return MinorType.DECIMAL;
+ }
+ return ARROW_TYPE_MINOR_TYPE_MAP.get(arrowType);
}
- public static MajorType repeated(MinorType minorType) {
- return new MajorType(minorType, DataMode.REPEATED);
+
+ static {
+ ARROW_TYPE_MINOR_TYPE_MAP = new HashMap<>();
+ for (MinorType minorType : MinorType.values()) {
+ if (minorType != MinorType.DECIMAL) {
+ ARROW_TYPE_MINOR_TYPE_MAP.put(minorType.getType(), minorType);
+ }
+ }
}
+
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
new file mode 100644
index 0000000..49d0503
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.types.pojo;
+
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
+
+public class Field {
+ private final String name;
+ private final boolean nullable;
+ private final ArrowType type;
+ private final List<Field> children;
+
+ public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
+ this.name = name;
+ this.nullable = nullable;
+ this.type = type;
+ if (children == null) {
+ this.children = ImmutableList.of();
+ } else {
+ this.children = children;
+ }
+ }
+
+ public static Field convertField(org.apache.arrow.flatbuf.Field field) {
+ String name = field.name();
+ boolean nullable = field.nullable();
+ ArrowType type = getTypeForField(field);
+ ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+ for (int i = 0; i < field.childrenLength(); i++) {
+ childrenBuilder.add(convertField(field.children(i)));
+ }
+ List<Field> children = childrenBuilder.build();
+ return new Field(name, nullable, type, children);
+ }
+
+ public int getField(FlatBufferBuilder builder) {
+ int nameOffset = builder.createString(name);
+ int typeOffset = type.getType(builder);
+ int[] childrenData = new int[children.size()];
+ for (int i = 0; i < children.size(); i++) {
+ childrenData[i] = children.get(i).getField(builder);
+ }
+ int childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, childrenData);
+ org.apache.arrow.flatbuf.Field.startField(builder);
+ org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
+ org.apache.arrow.flatbuf.Field.addNullable(builder, nullable);
+ org.apache.arrow.flatbuf.Field.addTypeType(builder, type.getTypeType());
+ org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
+ org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
+ return org.apache.arrow.flatbuf.Field.endField(builder);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ public ArrowType getType() {
+ return type;
+ }
+
+ public List<Field> getChildren() {
+ return children;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Field)) {
+ return false;
+ }
+ Field that = (Field) obj;
+ return Objects.equals(this.name, that.name) &&
+ Objects.equals(this.nullable, that.nullable) &&
+ Objects.equals(this.type, that.type) &&
+ (Objects.equals(this.children, that.children) ||
+ (this.children == null && that.children.size() == 0) ||
+ (this.children.size() == 0 && that.children == null));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
new file mode 100644
index 0000000..9e28941
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.types.pojo;
+
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
+import static org.apache.arrow.vector.types.pojo.Field.convertField;
+
+public class Schema {
+ private List<Field> fields;
+
+ public Schema(List<Field> fields) {
+ this.fields = ImmutableList.copyOf(fields);
+ }
+
+ public int getSchema(FlatBufferBuilder builder) {
+ int[] fieldOffsets = new int[fields.size()];
+ for (int i = 0; i < fields.size(); i++) {
+ fieldOffsets[i] = fields.get(i).getField(builder);
+ }
+ int fieldsOffset = org.apache.arrow.flatbuf.Schema.createFieldsVector(builder, fieldOffsets);
+ org.apache.arrow.flatbuf.Schema.startSchema(builder);
+ org.apache.arrow.flatbuf.Schema.addFields(builder, fieldsOffset);
+ return org.apache.arrow.flatbuf.Schema.endSchema(builder);
+ }
+
+ public List<Field> getFields() {
+ return fields;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(fields);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Schema)) {
+ return false;
+ }
+ return Objects.equals(this.fields, ((Schema) obj).fields);
+ }
+
+ public static Schema convertSchema(org.apache.arrow.flatbuf.Schema schema) {
+ ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+ for (int i = 0; i < schema.fieldsLength(); i++) {
+ childrenBuilder.add(convertField(schema.fields(i)));
+ }
+ List<Field> fields = childrenBuilder.build();
+ return new Schema(fields);
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
index b6dd13a..68b9fb2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
@@ -180,54 +180,4 @@ public class ByteFunctionHelpers {
return lLen > rLen ? 1 : -1;
}
- /*
- * Following are helper functions to interact with sparse decimal represented in a byte array.
- */
-
- // Get the integer ignore the sign
- public static int getInteger(byte[] b, int index) {
- return getInteger(b, index, true);
- }
- // Get the integer, ignore the sign
- public static int getInteger(byte[] b, int index, boolean ignoreSign) {
- int startIndex = index * DecimalUtility.INTEGER_SIZE;
-
- if (index == 0 && ignoreSign == true) {
- return (b[startIndex + 3] & 0xFF) |
- (b[startIndex + 2] & 0xFF) << 8 |
- (b[startIndex + 1] & 0xFF) << 16 |
- (b[startIndex] & 0x7F) << 24;
- }
-
- return ((b[startIndex + 3] & 0xFF) |
- (b[startIndex + 2] & 0xFF) << 8 |
- (b[startIndex + 1] & 0xFF) << 16 |
- (b[startIndex] & 0xFF) << 24);
-
- }
-
- // Set integer in the byte array
- public static void setInteger(byte[] b, int index, int value) {
- int startIndex = index * DecimalUtility.INTEGER_SIZE;
- b[startIndex] = (byte) ((value >> 24) & 0xFF);
- b[startIndex + 1] = (byte) ((value >> 16) & 0xFF);
- b[startIndex + 2] = (byte) ((value >> 8) & 0xFF);
- b[startIndex + 3] = (byte) ((value) & 0xFF);
- }
-
- // Set the sign in a sparse decimal representation
- public static void setSign(byte[] b, boolean sign) {
- int value = getInteger(b, 0);
- if (sign == true) {
- setInteger(b, 0, value | 0x80000000);
- } else {
- setInteger(b, 0, value & 0x7FFFFFFF);
- }
- }
-
- // Get the sign
- public static boolean getSign(byte[] b) {
- return ((getInteger(b, 0, false) & 0x80000000) != 0);
- }
-
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
deleted file mode 100644
index 1eb2c13..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/CoreDecimalUtility.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.util;
-
-import java.math.BigDecimal;
-
-import org.apache.arrow.vector.types.Types;
-
-public class CoreDecimalUtility {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoreDecimalUtility.class);
-
- public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- return (input.unscaledValue().longValue());
- }
-
- public static int getMaxPrecision(Types.MinorType decimalType) {
- if (decimalType == Types.MinorType.DECIMAL9) {
- return 9;
- } else if (decimalType == Types.MinorType.DECIMAL18) {
- return 18;
- } else if (decimalType == Types.MinorType.DECIMAL28SPARSE) {
- return 28;
- } else if (decimalType == Types.MinorType.DECIMAL38SPARSE) {
- return 38;
- }
- return 0;
- }
-
- /*
- * Function returns the Minor decimal type given the precision
- */
- public static Types.MinorType getDecimalDataType(int precision) {
- if (precision <= 9) {
- return Types.MinorType.DECIMAL9;
- } else if (precision <= 18) {
- return Types.MinorType.DECIMAL18;
- } else if (precision <= 28) {
- return Types.MinorType.DECIMAL28SPARSE;
- } else {
- return Types.MinorType.DECIMAL38SPARSE;
- }
- }
-
- /*
- * Given a precision it provides the max precision of that decimal data type;
- * For eg: given the precision 12, we would use DECIMAL18 to store the data
- * which has a max precision range of 18 digits
- */
- public static int getPrecisionRange(int precision) {
- return getMaxPrecision(getDecimalDataType(precision));
- }
- public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) {
- // Truncate/ or pad to set the input to the correct scale
- input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
-
- return (input.unscaledValue().intValue());
- }
-
- /*
- * Helper function to detect if the given data type is Decimal
- */
- public static boolean isDecimalType(Types.MajorType type) {
- return isDecimalType(type.getMinorType());
- }
-
- public static boolean isDecimalType(Types.MinorType minorType) {
- if (minorType == Types.MinorType.DECIMAL9 || minorType == Types.MinorType.DECIMAL18 ||
- minorType == Types.MinorType.DECIMAL28SPARSE || minorType == Types.MinorType.DECIMAL38SPARSE) {
- return true;
- }
- return false;
- }
-}