You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/07/20 03:35:00 UTC
[12/17] Working Project merge build
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/BufBitSet.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/BufBitSet.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/BufBitSet.java
deleted file mode 100644
index 574389f..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/BufBitSet.java
+++ /dev/null
@@ -1,847 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.record.vector; // from org.apache.solr.util rev 555343
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-
-/**
- * HEAVY WIP: ONLY PARTIALLY TRANSFERRED TO BUFFER METHODS. STILL NEEDS BIT SHIFT FIXES, GETLONG AND SETLONG updates to
- * fix index postion AND OTHER THINGS.
- *
- * An "open" BitSet implementation that allows direct access to the array of words storing the bits.
- * <p/>
- * Unlike java.util.bitset, the fact that bits are packed into an array of longs is part of the interface. This allows
- * efficient implementation of other algorithms by someone other than the author. It also allows one to efficiently
- * implement alternate serialization or interchange formats.
- * <p/>
- * <code>BufBitSet</code> is faster than <code>java.util.BitSet</code> in most operations and *much* faster at
- * calculating cardinality of sets and results of set operations. It can also handle sets of larger cardinality (up to
- * 64 * 2**32-1)
- * <p/>
- * The goals of <code>BufBitSet</code> are the fastest implementation possible, and maximum code reuse. Extra safety
- * and encapsulation may always be built on top, but if that's built in, the cost can never be removed (and hence people
- * re-implement their own version in order to get better performance). If you want a "safe", totally encapsulated (and
- * slower and limited) BitSet class, use <code>java.util.BitSet</code>.
- * <p/>
- */
-
-public class BufBitSet {
- private ByteBufAllocator allocator;
- private ByteBuf buf;
- // protected long[] bits;
- protected int wlen; // number of words (elements) used in the array
-
- // Used only for assert:
- private long numBits;
-
- // /** Constructs an BufBitSet large enough to hold <code>numBits</code>.
- // */
- // public BufBitSet(long numBits) {
- // this.numBits = numBits;
- // wlen = buf.capacity();
- // }
- //
- // public BufBitSet() {
- // this(64);
- // }
-
- public BufBitSet(long numBits, ByteBufAllocator allocator) {
- this.allocator = allocator;
- this.numBits = numBits;
- int words = bits2words(numBits);
- this.wlen = words;
- buf = allocator.buffer(wlen);
- }
-
- private BufBitSet(ByteBufAllocator allocator, ByteBuf buf) {
- this.allocator = allocator;
- this.numBits = buf.capacity() * 8;
- int words = buf.capacity();
- this.wlen = words;
- this.buf = buf;
- }
-
- /** Returns the current capacity in bits (1 greater than the index of the last bit) */
- public long capacity() {
- return buf.capacity() << 6;
- }
-
- /**
- * Returns the current capacity of this set. Included for compatibility. This is *not* equal to {@link #cardinality}
- */
- public long size() {
- return capacity();
- }
-
- public int length() {
- return buf.capacity() << 6;
- }
-
- /** Returns true if there are no set bits */
- public boolean isEmpty() {
- return cardinality() == 0;
- }
-
- // /** Expert: returns the long[] storing the bits */
- // public long[] getBits() { return bits; }
- //
- // /** Expert: sets a new long[] to use as the bit storage */
- // public void setBits(long[] bits) { this.bits = bits; }
-
- /** Expert: gets the number of longs in the array that are in use */
- public int getNumWords() {
- return wlen;
- }
-
- /** Expert: sets the number of longs in the array that are in use */
- public void setNumWords(int nWords) {
- this.wlen = nWords;
- }
-
- /** Returns true or false for the specified bit index. */
- public boolean get(int index) {
- int i = index >> 6; // div 64
- // signed shift will keep a negative index and force an
- // array-index-out-of-bounds-exception, removing the need for an explicit check.
- if (i >= buf.capacity()) return false;
-
- int bit = index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- return (buf.getLong(i) & bitmask) != 0;
- }
-
- /**
- * Returns true or false for the specified bit index. The index should be less than the BufBitSet size
- */
- public boolean fastGet(int index) {
- assert index >= 0 && index < numBits;
- int i = index >> 6; // div 64
- // signed shift will keep a negative index and force an
- // array-index-out-of-bounds-exception, removing the need for an explicit check.
- int bit = index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- return (buf.getLong(i) & bitmask) != 0;
- }
-
- /**
- * Returns true or false for the specified bit index
- */
- public boolean get(long index) {
- int i = (int) (index >> 6); // div 64
- if (i >= buf.capacity()) return false;
- int bit = (int) index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- return (buf.getLong(i) & bitmask) != 0;
- }
-
- /**
- * Returns true or false for the specified bit index. The index should be less than the BufBitSet size.
- */
- public boolean fastGet(long index) {
- assert index >= 0 && index < numBits;
- int i = (int) (index >> 6); // div 64
- int bit = (int) index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- return (buf.getLong(i) & bitmask) != 0;
- }
-
- /*
- * // alternate implementation of get() public boolean get1(int index) { int i = index >> 6; // div 64 int bit = index
- * & 0x3f; // mod 64 return ((buf.getLong(i)>>>bit) & 0x01) != 0; // this does a long shift and a bittest (on x86) vs
- * // a long shift, and a long AND, (the test for zero is prob a no-op) // testing on a P4 indicates this is slower
- * than (buf.getLong(i) & bitmask) != 0; }
- */
-
- /**
- * returns 1 if the bit is set, 0 if not. The index should be less than the BufBitSet size
- */
- public int getBit(int index) {
- assert index >= 0 && index < numBits;
- int i = index >> 6; // div 64
- int bit = index & 0x3f; // mod 64
- return ((int) (buf.getLong(i) >>> bit)) & 0x01;
- }
-
- /*
- * public boolean get2(int index) { int word = index >> 6; // div 64 int bit = index & 0x0000003f; // mod 64 return
- * (buf.getLong(word) << bit) < 0; // hmmm, this would work if bit order were reversed // we could right shift and
- * check for parity bit, if it was available to us. }
- */
-
- /** sets a bit, expanding the set size if necessary */
- public void set(long index) {
- int wordNum = expandingWordNum(index);
- int bit = (int) index & 0x3f;
- long bitmask = 1L << bit;
- buf.setLong(wordNum, buf.getLong(wordNum) | bitmask);
- buf.setLong(wordNum, buf.getLong(wordNum) | bitmask);
- }
-
- /**
- * Sets the bit at the specified index. The index should be less than the BufBitSet size.
- */
- public void fastSet(int index) {
- assert index >= 0 && index < numBits;
- int wordNum = index >> 6; // div 64
- int bit = index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- buf.setLong(wordNum, buf.getLong(wordNum) | bitmask);
- }
-
- /**
- * Sets the bit at the specified index. The index should be less than the BufBitSet size.
- */
- public void fastSet(long index) {
- assert index >= 0 && index < numBits;
- int wordNum = (int) (index >> 6);
- int bit = (int) index & 0x3f;
- long bitmask = 1L << bit;
- buf.setLong(wordNum, buf.getLong(wordNum) | bitmask);
- }
-
- /**
- * Sets a range of bits, expanding the set size if necessary
- *
- * @param startIndex
- * lower index
- * @param endIndex
- * one-past the last bit to set
- */
- public void set(long startIndex, long endIndex) {
- if (endIndex <= startIndex) return;
-
- int startWord = (int) (startIndex >> 6);
-
- // since endIndex is one past the end, this is index of the last
- // word to be changed.
- int endWord = expandingWordNum(endIndex - 1);
-
- long startmask = -1L << startIndex;
- long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
-
- if (startWord == endWord) {
- buf.setLong(startWord, buf.getLong(startWord) | (startmask & endmask));
- return;
- }
- buf.setLong(startWord, buf.getLong(startWord) | startmask);
-
- fill(buf, startWord + 1, endWord, -1L);
- buf.setLong(endWord, buf.getLong(endWord) | endmask);
- }
-
- private void fill(ByteBuf buf, int start, int end, long val) {
- for (int i = 0; i < buf.capacity(); i += 8) {
- buf.setLong(i, val);
- }
- }
-
- private final void setLongWord(int pos, long value) {
- buf.setLong(pos * 8, value);
- }
-
- private final long getLongWord(int pos) {
- return buf.getLong(pos * 8);
- }
-
- protected int expandingWordNum(long index) {
- int wordNum = (int) (index >> 6);
- if (wordNum >= wlen) {
- ensureCapacity(index + 1);
- wlen = wordNum + 1;
- }
- assert (numBits = Math.max(numBits, index + 1)) >= 0;
- return wordNum;
- }
-
- /**
- * clears a bit. The index should be less than the BufBitSet size.
- */
- public void fastClear(int index) {
- assert index >= 0 && index < numBits;
- int wordNum = index >> 6;
- int bit = index & 0x03f;
- long bitmask = 1L << bit;
- buf.setLong(wordNum, buf.getLong(wordNum) & ~bitmask);
- // hmmm, it takes one more instruction to clear than it does to set... any
- // way to work around this? If there were only 63 bits per word, we could
- // use a right shift of 10111111...111 in binary to position the 0 in the
- // correct place (using sign extension).
- // Could also use Long.rotateRight() or rotateLeft() *if* they were converted
- // by the JVM into a native instruction.
- // buf.getLong(word) &= Long.rotateLeft(0xfffffffe,bit);
- }
-
- /**
- * clears a bit. The index should be less than the BufBitSet size.
- */
- public void fastClear(long index) {
- assert index >= 0 && index < numBits;
- int wordNum = (int) (index >> 6); // div 64
- int bit = (int) index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- buf.setLong(wordNum, buf.getLong(wordNum) & ~bitmask);
- }
-
- /** clears a bit, allowing access beyond the current set size without changing the size. */
- public void clear(long index) {
- int wordNum = (int) (index >> 6); // div 64
- if (wordNum >= wlen) return;
- int bit = (int) index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- buf.setLong(wordNum, buf.getLong(wordNum) & ~bitmask);
- }
-
- /**
- * Clears a range of bits. Clearing past the end does not change the size of the set.
- *
- * @param startIndex
- * lower index
- * @param endIndex
- * one-past the last bit to clear
- */
- public void clear(int startIndex, int endIndex) {
- if (endIndex <= startIndex) return;
-
- int startWord = (startIndex >> 6);
- if (startWord >= wlen) return;
-
- // since endIndex is one past the end, this is index of the last
- // word to be changed.
- int endWord = ((endIndex - 1) >> 6);
-
- long startmask = -1L << startIndex;
- long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
-
- // invert masks since we are clearing
- startmask = ~startmask;
- endmask = ~endmask;
-
- if (startWord == endWord) {
- buf.setLong(startWord, buf.getLong(startWord) & (startmask | endmask));
- return;
- }
-
- buf.setLong(startWord, buf.getLong(startWord) & startmask);
-
- int middle = Math.min(wlen, endWord);
- fill(buf, startWord + 1, middle, 0L);
- if (endWord < wlen) {
- buf.setLong(endWord, buf.getLong(endWord) & endmask);
- }
- }
-
- /**
- * Clears a range of bits. Clearing past the end does not change the size of the set.
- *
- * @param startIndex
- * lower index
- * @param endIndex
- * one-past the last bit to clear
- */
- public void clear(long startIndex, long endIndex) {
- if (endIndex <= startIndex) return;
-
- int startWord = (int) (startIndex >> 6);
- if (startWord >= wlen) return;
-
- // since endIndex is one past the end, this is index of the last
- // word to be changed.
- int endWord = (int) ((endIndex - 1) >> 6);
-
- long startmask = -1L << startIndex;
- long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
-
- // invert masks since we are clearing
- startmask = ~startmask;
- endmask = ~endmask;
-
- if (startWord == endWord) {
- buf.setLong(startWord, buf.getLong(startWord) & (startmask | endmask));
- return;
- }
-
- buf.setLong(startWord, buf.getLong(startWord) & startmask);
-
- int middle = Math.min(wlen, endWord);
- fill(buf, startWord + 1, middle, 0L);
- if (endWord < wlen) {
- buf.setLong(endWord, buf.getLong(endWord) & endmask);
- }
- }
-
- /**
- * Sets a bit and returns the previous value. The index should be less than the BufBitSet size.
- */
- public boolean getAndSet(int index) {
- assert index >= 0 && index < numBits;
- int wordNum = index >> 6; // div 64
- int bit = index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- long longVal = buf.getLong(wordNum);
- boolean val = (longVal & bitmask) != 0;
- buf.setLong(wordNum, longVal | bitmask);
- return val;
- }
-
- /**
- * flips a bit. The index should be less than the BufBitSet size.
- */
- public void fastFlip(int index) {
- assert index >= 0 && index < numBits;
- int wordNum = index >> 6; // div 64
- int bit = index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- buf.setLong(wordNum, (buf.getLong(wordNum) ^ bitmask));
- }
-
- /**
- * flips a bit. The index should be less than the BufBitSet size.
- */
- public void fastFlip(long index) {
- assert index >= 0 && index < numBits;
- int wordNum = (int) (index >> 6); // div 64
- int bit = (int) index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- buf.setLong(wordNum, (buf.getLong(wordNum) ^ bitmask));
- }
-
- /** flips a bit, expanding the set size if necessary */
- public void flip(long index) {
- int wordNum = expandingWordNum(index);
- int bit = (int) index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- buf.setLong(wordNum, (buf.getLong(wordNum) ^ bitmask));
- }
-
- /**
- * flips a bit and returns the resulting bit value. The index should be less than the BufBitSet size.
- */
- public boolean flipAndGet(int index) {
- assert index >= 0 && index < numBits;
- int wordNum = index >> 6; // div 64
- int bit = index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- long longVal = buf.getLong(wordNum);
- buf.setLong(wordNum, longVal ^ bitmask);
- return (longVal & bitmask) != 0;
- }
-
- /**
- * flips a bit and returns the resulting bit value. The index should be less than the BufBitSet size.
- */
- public boolean flipAndGet(long index) {
- assert index >= 0 && index < numBits;
- int wordNum = (int) (index >> 6); // div 64
- int bit = (int) index & 0x3f; // mod 64
- long bitmask = 1L << bit;
- long longVal = buf.getLong(wordNum);
- buf.setLong(wordNum, longVal ^ bitmask);
- return (longVal & bitmask) != 0;
- }
-
- /**
- * Flips a range of bits, expanding the set size if necessary
- *
- * @param startIndex
- * lower index
- * @param endIndex
- * one-past the last bit to flip
- */
- public void flip(long startIndex, long endIndex) {
- if (endIndex <= startIndex) return;
- int startWord = (int) (startIndex >> 6);
-
- // since endIndex is one past the end, this is index of the last
- // word to be changed.
- int endWord = expandingWordNum(endIndex - 1);
-
- /***
- * Grrr, java shifting wraps around so -1L>>>64 == -1 for that reason, make sure not to use endmask if the bits to
- * flip will be zero in the last word (redefine endWord to be the last changed...) long startmask = -1L <<
- * (startIndex & 0x3f); // example: 11111...111000 long endmask = -1L >>> (64-(endIndex & 0x3f)); // example:
- * 00111...111111
- ***/
-
- long startmask = -1L << startIndex;
- long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
-
- if (startWord == endWord) {
- buf.setLong(startWord, buf.getLong(startWord) ^ (startmask & endmask));
- return;
- }
-
- buf.setLong(startWord, buf.getLong(startWord) ^ startmask);
-
- for (int i = startWord + 1; i < endWord; i++) {
- buf.setLong(i, ~buf.getLong(i));
- }
-
- buf.setLong(endWord, buf.getLong(endWord) ^ endmask);
- }
-
- /*
- * public static int pop(long v0, long v1, long v2, long v3) { // derived from pop_array by setting last four elems to
- * 0. // exchanges one pop() call for 10 elementary operations // saving about 7 instructions... is there a better
- * way? long twosA=v0 & v1; long ones=v0^v1;
- *
- * long u2=ones^v2; long twosB =(ones&v2)|(u2&v3); ones=u2^v3;
- *
- * long fours=(twosA&twosB); long twos=twosA^twosB;
- *
- * return (pop(fours)<<2) + (pop(twos)<<1) + pop(ones);
- *
- * }
- */
-
- /** @return the number of set bits */
- public long cardinality() {
- return BitUtil.pop_array(buf, 0, wlen);
- }
-
- /**
- * Returns the popcount or cardinality of the intersection of the two sets. Neither set is modified.
- */
- public static long intersectionCount(BufBitSet a, BufBitSet b) {
- return BitUtil.pop_intersect(a.buf, b.buf, 0, Math.min(a.wlen, b.wlen));
- }
-
- /**
- * Returns the popcount or cardinality of the union of the two sets. Neither set is modified.
- */
- public static long unionCount(BufBitSet a, BufBitSet b) {
- long tot = BitUtil.pop_union(a.buf, b.buf, 0, Math.min(a.wlen, b.wlen));
- if (a.wlen < b.wlen) {
- tot += BitUtil.pop_array(b.buf, a.wlen, b.wlen - a.wlen);
- } else if (a.wlen > b.wlen) {
- tot += BitUtil.pop_array(a.buf, b.wlen, a.wlen - b.wlen);
- }
- return tot;
- }
-
- /**
- * Returns the popcount or cardinality of "a and not b" or "intersection(a, not(b))". Neither set is modified.
- */
- public static long andNotCount(BufBitSet a, BufBitSet b) {
- long tot = BitUtil.pop_andnot(a.buf, b.buf, 0, Math.min(a.wlen, b.wlen));
- if (a.wlen > b.wlen) {
- tot += BitUtil.pop_array(a.buf, b.wlen, a.wlen - b.wlen);
- }
- return tot;
- }
-
- /**
- * Returns the popcount or cardinality of the exclusive-or of the two sets. Neither set is modified.
- */
- public static long xorCount(BufBitSet a, BufBitSet b) {
- long tot = BitUtil.pop_xor(a.buf, b.buf, 0, Math.min(a.wlen, b.wlen));
- if (a.wlen < b.wlen) {
- tot += BitUtil.pop_array(b.buf, a.wlen, b.wlen - a.wlen);
- } else if (a.wlen > b.wlen) {
- tot += BitUtil.pop_array(a.buf, b.wlen, a.wlen - b.wlen);
- }
- return tot;
- }
-
- /**
- * Returns the index of the first set bit starting at the index specified. -1 is returned if there are no more set
- * bits.
- */
- public int nextSetBit(int index) {
- int i = index >> 6;
- if (i >= wlen) return -1;
- int subIndex = index & 0x3f; // index within the word
- long word = buf.getLong(i) >> subIndex; // skip all the bits to the right of index
-
- if (word != 0) {
- return (i << 6) + subIndex + Long.numberOfTrailingZeros(word);
- }
-
- while (++i < wlen) {
- word = buf.getLong(i);
- if (word != 0) return (i << 6) + Long.numberOfTrailingZeros(word);
- }
-
- return -1;
- }
-
- /**
- * Returns the index of the first set bit starting at the index specified. -1 is returned if there are no more set
- * bits.
- */
- public long nextSetBit(long index) {
- int i = (int) (index >>> 6);
- if (i >= wlen) return -1;
- int subIndex = (int) index & 0x3f; // index within the word
- long word = buf.getLong(i) >>> subIndex; // skip all the bits to the right of index
-
- if (word != 0) {
- return (((long) i) << 6) + (subIndex + Long.numberOfTrailingZeros(word));
- }
-
- while (++i < wlen) {
- word = buf.getLong(i);
- if (word != 0) return (((long) i) << 6) + Long.numberOfTrailingZeros(word);
- }
-
- return -1;
- }
-
- /**
- * Returns the index of the first set bit starting downwards at the index specified. -1 is returned if there are no
- * more set bits.
- */
- public int prevSetBit(int index) {
- int i = index >> 6;
- final int subIndex;
- long word;
- if (i >= wlen) {
- i = wlen - 1;
- if (i < 0) return -1;
- subIndex = 63; // last possible bit
- word = buf.getLong(i);
- } else {
- if (i < 0) return -1;
- subIndex = index & 0x3f; // index within the word
- word = (buf.getLong(i) << (63 - subIndex)); // skip all the bits to the left of index
- }
-
- if (word != 0) {
- return (i << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
- }
-
- while (--i >= 0) {
- word = buf.getLong(i);
- if (word != 0) {
- return (i << 6) + 63 - Long.numberOfLeadingZeros(word);
- }
- }
-
- return -1;
- }
-
- /**
- * Returns the index of the first set bit starting downwards at the index specified. -1 is returned if there are no
- * more set bits.
- */
- public long prevSetBit(long index) {
- int i = (int) (index >> 6);
- final int subIndex;
- long word;
- if (i >= wlen) {
- i = wlen - 1;
- if (i < 0) return -1;
- subIndex = 63; // last possible bit
- word = buf.getLong(i);
- } else {
- if (i < 0) return -1;
- subIndex = (int) index & 0x3f; // index within the word
- word = (buf.getLong(i) << (63 - subIndex)); // skip all the bits to the left of index
- }
-
- if (word != 0) {
- return (((long) i) << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
- }
-
- while (--i >= 0) {
- word = buf.getLong(i);
- if (word != 0) {
- return (((long) i) << 6) + 63 - Long.numberOfLeadingZeros(word);
- }
- }
-
- return -1;
- }
-
- BufBitSet cloneTest() {
- BufBitSet obs = new BufBitSet(allocator, buf.copy());
- return obs;
- }
-
- /** this = this AND other */
- public void intersect(BufBitSet other) {
- int newLen = Math.min(this.wlen, other.wlen);
- ByteBuf thisArr = this.buf;
- ByteBuf otherArr = other.buf;
- // testing against zero can be more efficient
- int pos = newLen;
- while (--pos >= 0) {
- thisArr.setLong(pos, thisArr.getLong(pos) & otherArr.getLong(pos));
- }
- if (this.wlen > newLen) {
- // fill zeros from the new shorter length to the old length
- fill(buf, newLen, this.wlen, 0);
- }
- this.wlen = newLen;
- }
-
- /** this = this OR other */
- public void union(BufBitSet other) {
- int newLen = Math.max(wlen, other.wlen);
- ensureCapacityWords(newLen);
- assert (numBits = Math.max(other.numBits, numBits)) >= 0;
-
- ByteBuf thisArr = this.buf;
- ByteBuf otherArr = other.buf;
-
- int pos = Math.min(wlen, other.wlen);
- while (--pos >= 0) {
- thisArr.setLong(pos, thisArr.getLong(pos) | otherArr.getLong(pos));
- }
- if (this.wlen < newLen) {
- System.arraycopy(otherArr, this.wlen, thisArr, this.wlen, newLen - this.wlen);
- }
- this.wlen = newLen;
- }
-
- /** Remove all elements set in other. this = this AND_NOT other */
- public void remove(BufBitSet other) {
- int idx = Math.min(wlen, other.wlen);
- ByteBuf thisArr = this.buf;
- ByteBuf otherArr = other.buf;
- while (--idx >= 0) {
- thisArr.setLong(idx, thisArr.getLong(idx) & ~otherArr.getLong(idx));
- }
- }
-
- /** this = this XOR other */
- public void xor(BufBitSet other) {
- int newLen = Math.max(wlen, other.wlen);
- ensureCapacityWords(newLen);
- assert (numBits = Math.max(other.numBits, numBits)) >= 0;
-
- ByteBuf thisArr = this.buf;
- ByteBuf otherArr = other.buf;
- int pos = Math.min(wlen, other.wlen);
- while (--pos >= 0) {
- thisArr.setLong(pos, thisArr.getLong(pos) ^ otherArr.getLong(pos));
- }
- if (this.wlen < newLen) {
- otherArr.readerIndex(wlen);
- otherArr.writeBytes(thisArr);
- }
- this.wlen = newLen;
-
- }
-
- // some BitSet compatability methods
-
- // ** see {@link intersect} */
- public void and(BufBitSet other) {
- intersect(other);
- }
-
- // ** see {@link union} */
- public void or(BufBitSet other) {
- union(other);
- }
-
- // ** see {@link andNot} */
- public void andNot(BufBitSet other) {
- remove(other);
- }
-
- /** returns true if the sets have any elements in common */
- public boolean intersects(BufBitSet other) {
- int pos = Math.min(this.wlen, other.wlen);
- ByteBuf thisArr = this.buf;
- ByteBuf otherArr = other.buf;
- while (--pos >= 0) {
- if ((thisArr.getLong(pos) & otherArr.getLong(pos)) != 0) return true;
- }
- return false;
- }
-
- public void ensureCapacityWords(int numWords) {
- if (buf.capacity() < numWords) {
- ByteBuf newBuf = allocator.buffer(numWords * 8);
- buf.writeBytes(newBuf);
- buf.release();
- buf = newBuf;
- this.numBits = numWords * 64;
- }
- }
-
- /**
- * Ensure that the long[] is big enough to hold numBits, expanding it if necessary. getNumWords() is unchanged by this
- * call.
- */
- public void ensureCapacity(long numBits) {
- ensureCapacityWords(bits2words(numBits));
- }
-
- /**
- * Lowers numWords, the number of words in use, by checking for trailing zero words.
- */
- public void trimTrailingZeros() {
- int idx = wlen - 1;
- while (idx >= 0 && buf.getLong(idx) == 0)
- idx--;
- wlen = idx + 1;
- }
-
- /** returns the number of 64 bit words it would take to hold numBits */
- public static int bits2words(long numBits) {
- return (int) (((numBits - 1) >>> 6) + 1);
- }
-
- /** returns true if both sets have the same bits set */
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof BufBitSet)) return false;
- BufBitSet a;
- BufBitSet b = (BufBitSet) o;
- // make a the larger set.
- if (b.wlen > this.wlen) {
- a = b;
- b = this;
- } else {
- a = this;
- }
-
- // check for any set bits out of the range of b
- for (int i = a.wlen - 1; i >= b.wlen; i--) {
- if (a.buf.getLong(i) != 0) return false;
- }
-
- for (int i = b.wlen - 1; i >= 0; i--) {
- if (a.buf.getLong(i) != b.buf.getLong(i)) return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- // Start with a zero hash and use a mix that results in zero if the input is zero.
- // This effectively truncates trailing zeros without an explicit check.
- long h = 0;
- for (int i = buf.capacity(); --i >= 0;) {
- h ^= buf.getLong(i);
- h = (h << 1) | (h >>> 63); // rotate left
- }
- // fold leftmost bits into right and add a constant to prevent
- // empty sets from returning 0, which is too common.
- return (int) ((h >> 32) ^ h) + 0x98761234;
- }
-
- public void release() {
- this.buf.release();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullValueException.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullValueException.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullValueException.java
deleted file mode 100644
index 027b698..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullValueException.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-
-public class NullValueException extends DrillRuntimeException {
- public NullValueException(int index) {
- super("Element at index position: " + index + " is null");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
deleted file mode 100644
index 48aa0c2..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed8.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public final class NullableFixed8 extends NullableValueVector<NullableFixed8, Fixed8>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NullableFixed8.class);
-
- public NullableFixed8(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- @Override
- protected Fixed8 getNewValueVector(BufferAllocator allocator) {
- return new Fixed8(this.field, allocator);
- }
-
- public long get(int index){
- return 1l;
- }
-
- public void set(int index, long value){
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
deleted file mode 100644
index 3546bd8..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-/**
- * Convenience/Clarification Fixed2 wrapper.
- */
-public class SelectionVector {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector.class);
-
- public SelectionVector(MaterializedField field, BufferAllocator allocator) {
-
- }
-
- public int capacity() {
- return -1;
- }
-
- public void allocateNew(int count) {
-
- }
-<<<<<<< HEAD:sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
-=======
-
- public final int getInt(int index){
- index*=4;
- return data.getInt(index);
- }
->>>>>>> Build working:sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed4.java
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java
deleted file mode 100644
index b808dc6..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector2.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.DeadBuf;
-
-/**
- * A selection vector that fronts, at most, a
- */
-public class SelectionVector2{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector2.class);
-
- private final BufferAllocator allocator;
- private ByteBuf buffer = DeadBuf.DEAD_BUFFER;
-
- public SelectionVector2(BufferAllocator allocator) {
- this.allocator = allocator;
- }
-
- public int getCount(){
- return -1;
- }
-
- public int getIndex(int directIndex){
- return buffer.getChar(directIndex);
- }
-
- public void setIndex(int directIndex, char value){
- buffer.setChar(directIndex, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector4.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector4.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector4.java
deleted file mode 100644
index d857146..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector4.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.DeadBuf;
-
-public class SelectionVector4 {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector4.class);
-
- private final BufferAllocator allocator;
- private ByteBuf buffer = DeadBuf.DEAD_BUFFER;
-
- public SelectionVector4(BufferAllocator allocator) {
- this.allocator = allocator;
- }
-
- public int getCount(){
- return -1;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
index 8c31aa4..8513dfe 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
@@ -22,14 +22,8 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.vector.NullableBit;
-import org.apache.drill.exec.record.vector.NullableFixed4;
-import org.apache.drill.exec.record.vector.NullableVarLen4;
-import org.apache.drill.exec.record.vector.TypeHelper;
-import org.apache.drill.exec.record.vector.ValueVector;
import org.apache.drill.exec.schema.DiffSchema;
import org.apache.drill.exec.schema.Field;
-import org.apache.drill.exec.schema.IdGenerator;
import org.apache.drill.exec.schema.ListSchema;
import org.apache.drill.exec.schema.NamedField;
import org.apache.drill.exec.schema.ObjectSchema;
@@ -37,10 +31,15 @@ import org.apache.drill.exec.schema.OrderedField;
import org.apache.drill.exec.schema.RecordSchema;
import org.apache.drill.exec.schema.SchemaIdGenerator;
import org.apache.drill.exec.schema.json.jackson.JacksonHelper;
+import org.apache.drill.exec.vector.AllocationHelper;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.NullableFloat4Vector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableVarChar4Vector;
+import org.apache.drill.exec.vector.TypeHelper;
+import org.apache.drill.exec.vector.ValueVector;
import com.beust.jcommander.internal.Maps;
-import com.carrotsearch.hppc.IntObjectOpenHashMap;
-import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
@@ -416,8 +415,8 @@ public class JSONRecordReader implements RecordReader {
if (holder == null) {
MajorType type = field.getFieldType();
MaterializedField f = MaterializedField.create(new SchemaPath(field.getFieldName(), ExpressionPosition.UNKNOWN), type);
- ValueVector<?> v = TypeHelper.getNewVector(f, allocator);
- v.allocateNew(batchSize);
+ ValueVector v = TypeHelper.getNewVector(f, allocator);
+ AllocationHelper.allocate(v, batchSize, 50);
holder = new VectorHolder(batchSize, v);
valueVectorMap.put(field, holder);
outputMutator.addField(v);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
new file mode 100644
index 0000000..1631e70
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
@@ -0,0 +1,15 @@
+package org.apache.drill.exec.vector;
+
+public class AllocationHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class);
+
+ public static void allocate(ValueVector v, int valueCount, int bytesPerValue){
+ if(v instanceof FixedWidthVector){
+ ((FixedWidthVector) v).allocateNew(valueCount);
+ }else if(v instanceof VariableWidthVector){
+ ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount);
+ }else{
+ throw new UnsupportedOperationException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index a8678f5..e87e132 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -25,8 +25,6 @@ abstract class BaseValueVector implements ValueVector{
}
abstract class BaseAccessor implements ValueVector.Accessor{
-
-
public void reset(){}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 9d247f5..9a9f438 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -8,6 +8,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
/**
* Bit implements a vector of bit-width values. Elements in the vector are accessed
* by position from the logical start of the vector.
@@ -74,6 +75,32 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
return new Accessor();
}
+ public TransferPair getTransferPair(){
+ return new TransferImpl();
+ }
+
+ public void transferTo(BitVector target){
+ target.data = data;
+ target.data.retain();
+ target.recordCount = recordCount;
+ clear();
+ }
+
+ private class TransferImpl implements TransferPair{
+ BitVector to;
+
+ public TransferImpl(){
+ this.to = new BitVector(getField(), allocator);
+ }
+
+ public BitVector getTo(){
+ return to;
+ }
+
+ public void transfer(){
+ transferTo(to);
+ }
+ }
public class Accessor extends BaseAccessor{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 328182b..27089ac 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -21,9 +21,9 @@ import io.netty.buffer.ByteBuf;
import java.io.Closeable;
-import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
/**
* ValueVectorTypes defines a set of template-generated classes which implement type-specific
@@ -50,6 +50,9 @@ public interface ValueVector extends Closeable {
public void clear();
+ public TransferPair getTransferPair();
+
+
/**
* Return the underlying buffers associated with this vector. Note that this doesn't impact the
* reference counts for this buffer so it only should be used for in-context access. Also note
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
index 007abb3..623af0e 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
@@ -21,18 +21,13 @@ import org.apache.drill.common.expression.parser.ExprParser.parse_return;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.physical.impl.project.Projector;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.TypedFieldId;
-import org.apache.drill.exec.record.vector.Fixed4;
+import org.apache.drill.exec.vector.IntVector;
import org.junit.After;
import org.junit.Test;
-import org.slf4j.ILoggerFactory;
-import org.slf4j.LoggerFactory;
-
-import ch.qos.logback.classic.LoggerContext;
public class ExpressionTest {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionTest.class);
@@ -51,8 +46,8 @@ public class ExpressionTest {
{
batch.getValueVectorId(new SchemaPath("alpha", ExpressionPosition.UNKNOWN));
result = tfid;
- batch.getValueVectorById(tfid.getFieldId(), Fixed4.class);
- result = new Fixed4(null, null);
+ batch.getValueVectorById(tfid.getFieldId(), IntVector.class);
+ result = new IntVector(null, null);
}
};
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
index 093c58f..d125ec0 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
@@ -5,7 +5,7 @@ import org.apache.drill.exec.physical.impl.ScreenCreator.ScreenRoot;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.IterOutcome;
import org.apache.drill.exec.record.RecordBatch.TypedFieldId;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.ValueVector;
public class SimpleRootExec implements RootExec{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleRootExec.class);
@@ -22,7 +22,7 @@ public class SimpleRootExec implements RootExec{
}
- public <T extends ValueVector<T>> T getValueVectorById(SchemaPath path, Class<?> vvClass){
+ public <T extends ValueVector> T getValueVectorById(SchemaPath path, Class<?> vvClass){
TypedFieldId tfid = incoming.getValueVectorId(path);
return incoming.getValueVectorById(tfid.getFieldId(), vvClass);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index 3dc961b..7b002ea 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -17,7 +17,7 @@
******************************************************************************/
package org.apache.drill.exec.physical.impl;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
import java.util.List;
@@ -29,9 +29,9 @@ import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.vector.ValueVector;
import org.junit.Test;
-import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
@@ -57,42 +57,40 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
boolean schemaChanged = batchLoader.load(batch.getHeader().getDef(), batch.getData());
boolean firstColumn = true;
- // print headers.
- if (schemaChanged) {
- System.out.println("\n\n========NEW SCHEMA=========\n\n");
- for (ValueVector<?> value : batchLoader) {
+ // print headers.
+ if (schemaChanged) {
+ System.out.println("\n\n========NEW SCHEMA=========\n\n");
+ for (ValueVector value : batchLoader) {
- if (firstColumn) {
- firstColumn = false;
- } else {
- System.out.print("\t");
- }
- System.out.print(value.getField().getName());
- System.out.print("[");
- System.out.print(value.getField().getType().getMinorType());
- System.out.print("]");
- }
- System.out.println();
+ if (firstColumn) {
+ firstColumn = false;
+ } else {
+ System.out.print("\t");
}
+ System.out.print(value.getField().getName());
+ System.out.print("[");
+ System.out.print(value.getField().getType().getMinorType());
+ System.out.print("]");
+ }
+ System.out.println();
+ }
- for (int i = 0; i < batchLoader.getRecordCount(); i++) {
- boolean first = true;
- recordCount++;
- for (ValueVector<?> value : batchLoader) {
- if (first) {
- first = false;
- } else {
- System.out.print("\t");
- }
- System.out.print(value.getObject(i));
- }
- if(!first) System.out.println();
+ for (int i = 0; i < batchLoader.getRecordCount(); i++) {
+ boolean first = true;
+ recordCount++;
+ for (ValueVector value : batchLoader) {
+ if (first) {
+ first = false;
+ } else {
+ System.out.print("\t");
}
- System.out.print(v.value.getAccessor().getObject(i));
+ System.out.print(value.getAccessor().getObject(i));
}
if(!first) System.out.println();
}
+
+
}
logger.debug("Received results {}", results);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
index 4144a54..e3f03f2 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
@@ -18,9 +18,9 @@ import org.apache.drill.exec.physical.impl.SimpleRootExec;
import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.record.vector.Fixed8;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.vector.BigIntVector;
import org.junit.After;
import org.junit.Test;
@@ -50,13 +50,18 @@ public class TestSimpleProjection {
FragmentContext context = new FragmentContext(bitContext, FragmentHandle.getDefaultInstance(), connection, null, registry);
SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
while(exec.next()){
- Fixed8 c1 = exec.getValueVectorById(new SchemaPath("col1", ExpressionPosition.UNKNOWN), Fixed8.class);
- Fixed8 c2 = exec.getValueVectorById(new SchemaPath("col2", ExpressionPosition.UNKNOWN), Fixed8.class);
+ BigIntVector c1 = exec.getValueVectorById(new SchemaPath("col1", ExpressionPosition.UNKNOWN), BigIntVector.class);
+ BigIntVector c2 = exec.getValueVectorById(new SchemaPath("col2", ExpressionPosition.UNKNOWN), BigIntVector.class);
int x = 0;
- for(int i =0; i < c1.getRecordCount(); i++){
- assertEquals(c1.get(i)+1, c2.get(i));
- x += c1.get(i);
+ BigIntVector.Accessor a1, a2;
+ a1 = c1.getAccessor();
+ a2 = c2.getAccessor();
+
+ for(int i =0; i < c1.getAccessor().getRecordCount(); i++){
+ assertEquals(a1.get(i)+1, a2.get(i));
+ x += a1.get(i);
}
+
System.out.println(x);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestOpenBitSet.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestOpenBitSet.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestOpenBitSet.java
deleted file mode 100644
index 66f69de..0000000
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestOpenBitSet.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.UnpooledByteBufAllocator;
-
-import java.util.BitSet;
-import java.util.Random;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-@Ignore
-public class TestOpenBitSet {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestOpenBitSet.class);
-
- Random random = new Random();
- ByteBufAllocator allocator = UnpooledByteBufAllocator.DEFAULT;
-
- public int atLeast(int val){
- return val + random.nextInt(val);
- }
-
-
- public Random random() {
- return new Random();
- }
-
- void doGet(BitSet a, BufBitSet b) {
- int max = a.size();
- for (int i = 0; i < max; i++) {
- if (a.get(i) != b.get(i)) {
- fail("mismatch: BitSet=[" + i + "]=" + a.get(i));
- }
- if (a.get(i) != b.get((long) i)) {
- fail("mismatch: BitSet=[" + i + "]=" + a.get(i));
- }
- }
- }
-
- void doGetFast(BitSet a, BufBitSet b, int max) {
- for (int i = 0; i < max; i++) {
- if (a.get(i) != b.fastGet(i)) {
- fail("mismatch: BitSet=[" + i + "]=" + a.get(i));
- }
- if (a.get(i) != b.fastGet((long) i)) {
- fail("mismatch: BitSet=[" + i + "]=" + a.get(i));
- }
- }
- }
-
- void doNextSetBit(BitSet a, BufBitSet b) {
- int aa = -1, bb = -1;
- do {
- aa = a.nextSetBit(aa + 1);
- bb = b.nextSetBit(bb + 1);
- assertEquals(aa, bb);
- } while (aa >= 0);
- }
-
- void doNextSetBitLong(BitSet a, BufBitSet b) {
- int aa = -1, bb = -1;
- do {
- aa = a.nextSetBit(aa + 1);
- bb = (int) b.nextSetBit((long) (bb + 1));
- assertEquals(aa, bb);
- } while (aa >= 0);
- }
-
- void doPrevSetBit(BitSet a, BufBitSet b) {
- int aa = a.size() + random().nextInt(100);
- int bb = aa;
- do {
- // aa = a.prevSetBit(aa-1);
- aa--;
- while ((aa >= 0) && (!a.get(aa))) {
- aa--;
- }
- bb = b.prevSetBit(bb - 1);
- assertEquals(aa, bb);
- } while (aa >= 0);
- }
-
- void doPrevSetBitLong(BitSet a, BufBitSet b) {
- int aa = a.size() + random().nextInt(100);
- int bb = aa;
- do {
- // aa = a.prevSetBit(aa-1);
- aa--;
- while ((aa >= 0) && (!a.get(aa))) {
- aa--;
- }
- bb = (int) b.prevSetBit((long) (bb - 1));
- assertEquals(aa, bb);
- } while (aa >= 0);
- }
-
- // test interleaving different OpenBitSetIterator.next()/skipTo()
- void doIterate(BitSet a, BufBitSet b, int mode) {
- // if (mode == 1) doIterate1(a, b);
- // if (mode == 2) doIterate2(a, b);
- }
-
- //
- // void doIterate1(BitSet a, OpenBitSet b) {
- // int aa = -1, bb = -1;
- // OpenBitSetIterator iterator = new OpenBitSetIterator(b);
- // do {
- // aa = a.nextSetBit(aa + 1);
- // bb = random().nextBoolean() ? iterator.nextDoc() : iterator.advance(bb + 1);
- // assertEquals(aa == -1 ? DocIdSetIterator.NO_MORE_DOCS : aa, bb);
- // } while (aa >= 0);
- // }
- //
- // void doIterate2(BitSet a, OpenBitSet b) {
- // int aa = -1, bb = -1;
- // OpenBitSetIterator iterator = new OpenBitSetIterator(b);
- // do {
- // aa = a.nextSetBit(aa + 1);
- // bb = random().nextBoolean() ? iterator.nextDoc() : iterator.advance(bb + 1);
- // assertEquals(aa == -1 ? DocIdSetIterator.NO_MORE_DOCS : aa, bb);
- // } while (aa >= 0);
- // }
-
- void doRandomSets(int maxSize, int iter, int mode) {
- BitSet a0 = null;
- BufBitSet b0 = null;
-
- for (int i = 0; i < iter; i++) {
- int sz = random().nextInt(maxSize);
- BitSet a = new BitSet(sz);
- BufBitSet b = new BufBitSet(sz, allocator);
-
- // test the various ways of setting bits
- if (sz > 0) {
- int nOper = random().nextInt(sz);
- for (int j = 0; j < nOper; j++) {
- int idx;
-
- idx = random().nextInt(sz);
- a.set(idx);
- b.fastSet(idx);
-
- idx = random().nextInt(sz);
- a.set(idx);
- b.fastSet((long) idx);
-
- idx = random().nextInt(sz);
- a.clear(idx);
- b.fastClear(idx);
-
- idx = random().nextInt(sz);
- a.clear(idx);
- b.fastClear((long) idx);
-
- idx = random().nextInt(sz);
- a.flip(idx);
- b.fastFlip(idx);
-
- boolean val = b.flipAndGet(idx);
- boolean val2 = b.flipAndGet(idx);
- assertTrue(val != val2);
-
- idx = random().nextInt(sz);
- a.flip(idx);
- b.fastFlip((long) idx);
-
- val = b.flipAndGet((long) idx);
- val2 = b.flipAndGet((long) idx);
- assertTrue(val != val2);
-
- val = b.getAndSet(idx);
- assertTrue(val2 == val);
- assertTrue(b.get(idx));
-
- if (!val) b.fastClear(idx);
- assertTrue(b.get(idx) == val);
- }
- }
-
- // test that the various ways of accessing the bits are equivalent
- doGet(a, b);
- doGetFast(a, b, sz);
-
- // test ranges, including possible extension
- int fromIndex, toIndex;
- fromIndex = random().nextInt(sz + 80);
- toIndex = fromIndex + random().nextInt((sz >> 1) + 1);
- BitSet aa = (BitSet) a.clone();
- aa.flip(fromIndex, toIndex);
- BufBitSet bb = b.cloneTest();
- bb.flip(fromIndex, toIndex);
-
- doIterate(aa, bb, mode); // a problem here is from flip or doIterate
-
- fromIndex = random().nextInt(sz + 80);
- toIndex = fromIndex + random().nextInt((sz >> 1) + 1);
- aa = (BitSet) a.clone();
- aa.clear(fromIndex, toIndex);
- bb = b.cloneTest();
- bb.clear(fromIndex, toIndex);
-
- doNextSetBit(aa, bb); // a problem here is from clear() or nextSetBit
- doNextSetBitLong(aa, bb);
-
- doPrevSetBit(aa, bb);
- doPrevSetBitLong(aa, bb);
-
- fromIndex = random().nextInt(sz + 80);
- toIndex = fromIndex + random().nextInt((sz >> 1) + 1);
- aa = (BitSet) a.clone();
- aa.set(fromIndex, toIndex);
- bb = b.cloneTest();
- bb.set(fromIndex, toIndex);
-
- doNextSetBit(aa, bb); // a problem here is from set() or nextSetBit
- doNextSetBitLong(aa, bb);
-
- doPrevSetBit(aa, bb);
- doPrevSetBitLong(aa, bb);
-
- if (a0 != null) {
- assertEquals(a.equals(a0), b.equals(b0));
-
- assertEquals(a.cardinality(), b.cardinality());
-
- BitSet a_and = (BitSet) a.clone();
- a_and.and(a0);
- BitSet a_or = (BitSet) a.clone();
- a_or.or(a0);
- BitSet a_xor = (BitSet) a.clone();
- a_xor.xor(a0);
- BitSet a_andn = (BitSet) a.clone();
- a_andn.andNot(a0);
-
- BufBitSet b_and = b.cloneTest();
- assertEquals(b, b_and);
- b_and.and(b0);
- BufBitSet b_or = b.cloneTest();
- b_or.or(b0);
- BufBitSet b_xor = b.cloneTest();
- b_xor.xor(b0);
- BufBitSet b_andn = b.cloneTest();
- b_andn.andNot(b0);
-
- doIterate(a_and, b_and, mode);
- doIterate(a_or, b_or, mode);
- doIterate(a_xor, b_xor, mode);
- doIterate(a_andn, b_andn, mode);
-
- assertEquals(a_and.cardinality(), b_and.cardinality());
- assertEquals(a_or.cardinality(), b_or.cardinality());
- assertEquals(a_xor.cardinality(), b_xor.cardinality());
- assertEquals(a_andn.cardinality(), b_andn.cardinality());
-
- // test non-mutating popcounts
- assertEquals(b_and.cardinality(), BufBitSet.intersectionCount(b, b0));
- assertEquals(b_or.cardinality(), BufBitSet.unionCount(b, b0));
- assertEquals(b_xor.cardinality(), BufBitSet.xorCount(b, b0));
- assertEquals(b_andn.cardinality(), BufBitSet.andNotCount(b, b0));
- }
-
- a0 = a;
- b0 = b;
- }
- }
-
- // large enough to flush obvious bugs, small enough to run in <.5 sec as part of a
- // larger testsuite.
- @Test
- public void testSmall() {
- doRandomSets(atLeast(1200), atLeast(1000), 1);
- doRandomSets(atLeast(1200), atLeast(1000), 2);
- }
-
- // uncomment to run a bigger test (~2 minutes).
- /*
- * public void testBig() { doRandomSets(2000,200000, 1); doRandomSets(2000,200000, 2); }
- */
-
- @Test
- public void testEquals() {
- BufBitSet b1 = new BufBitSet(1111, allocator);
- BufBitSet b2 = new BufBitSet(2222, allocator);
- assertTrue(b1.equals(b2));
- assertTrue(b2.equals(b1));
- b1.set(10);
- assertFalse(b1.equals(b2));
- assertFalse(b2.equals(b1));
- b2.set(10);
- assertTrue(b1.equals(b2));
- assertTrue(b2.equals(b1));
- b2.set(2221);
- assertFalse(b1.equals(b2));
- assertFalse(b2.equals(b1));
- b1.set(2221);
- assertTrue(b1.equals(b2));
- assertTrue(b2.equals(b1));
-
- // try different type of object
- assertFalse(b1.equals(new Object()));
- }
-
- @Test
- public void testHashCodeEquals() {
- BufBitSet bs1 = new BufBitSet(200, allocator);
- BufBitSet bs2 = new BufBitSet(64, allocator);
- bs1.set(3);
- bs2.set(3);
- assertEquals(bs1, bs2);
- assertEquals(bs1.hashCode(), bs2.hashCode());
- }
-
- private BufBitSet makeOpenBitSet(int[] a) {
- BufBitSet bs = new BufBitSet(64, allocator);
- for (int e : a) {
- bs.set(e);
- }
- return bs;
- }
-
- private BitSet makeBitSet(int[] a) {
- BitSet bs = new BitSet();
- for (int e : a) {
- bs.set(e);
- }
- return bs;
- }
-
- private void checkPrevSetBitArray(int[] a) {
- BufBitSet obs = makeOpenBitSet(a);
- BitSet bs = makeBitSet(a);
- doPrevSetBit(bs, obs);
- }
-
- public void testPrevSetBit() {
- checkPrevSetBitArray(new int[] {});
- checkPrevSetBitArray(new int[] { 0 });
- checkPrevSetBitArray(new int[] { 0, 2 });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index 6a1f3ad..0fed756 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -4,8 +4,11 @@ import static org.junit.Assert.assertEquals;
import java.nio.charset.Charset;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.memory.DirectBufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.NullableFloat4Vector;
@@ -22,15 +25,13 @@ public class TestValueVector {
@Test
public void testFixedType() {
// Build a required uint field definition
- SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
- SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ MajorType.Builder typeBuilder = MajorType.newBuilder();
+ FieldDef.Builder defBuilder = FieldDef.newBuilder();
typeBuilder
- .setMinorType(SchemaDefProtos.MinorType.UINT4)
- .setMode(SchemaDefProtos.DataMode.REQUIRED)
+ .setMinorType(MinorType.UINT4)
+ .setMode(DataMode.REQUIRED)
.setWidth(4);
defBuilder
- .setFieldId(1)
- .setParentId(0)
.setMajorType(typeBuilder.build());
MaterializedField field = MaterializedField.create(defBuilder.build());
@@ -58,15 +59,13 @@ public class TestValueVector {
@Test
public void testNullableVarLen2() {
// Build an optional varchar field definition
- SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
- SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ MajorType.Builder typeBuilder = MajorType.newBuilder();
+ FieldDef.Builder defBuilder = FieldDef.newBuilder();
typeBuilder
- .setMinorType(SchemaDefProtos.MinorType.VARCHAR2)
- .setMode(SchemaDefProtos.DataMode.OPTIONAL)
+ .setMinorType(MinorType.VARCHAR2)
+ .setMode(DataMode.OPTIONAL)
.setWidth(2);
defBuilder
- .setFieldId(1)
- .setParentId(0)
.setMajorType(typeBuilder.build());
MaterializedField field = MaterializedField.create(defBuilder.build());
@@ -106,15 +105,13 @@ public class TestValueVector {
@Test
public void testNullableFixedType() {
// Build an optional uint field definition
- SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
- SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ MajorType.Builder typeBuilder = MajorType.newBuilder();
+ FieldDef.Builder defBuilder = FieldDef.newBuilder();
typeBuilder
- .setMinorType(SchemaDefProtos.MinorType.UINT4)
- .setMode(SchemaDefProtos.DataMode.OPTIONAL)
+ .setMinorType(MinorType.UINT4)
+ .setMode(DataMode.OPTIONAL)
.setWidth(4);
defBuilder
- .setFieldId(1)
- .setParentId(0)
.setMajorType(typeBuilder.build());
MaterializedField field = MaterializedField.create(defBuilder.build());
@@ -195,15 +192,13 @@ public class TestValueVector {
@Test
public void testNullableFloat() {
// Build an optional float field definition
- SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
- SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ MajorType.Builder typeBuilder = MajorType.newBuilder();
+ FieldDef.Builder defBuilder = FieldDef.newBuilder();
typeBuilder
- .setMinorType(SchemaDefProtos.MinorType.FLOAT4)
- .setMode(SchemaDefProtos.DataMode.OPTIONAL)
+ .setMinorType(MinorType.FLOAT4)
+ .setMode(DataMode.OPTIONAL)
.setWidth(4);
defBuilder
- .setFieldId(1)
- .setParentId(0)
.setMajorType(typeBuilder.build());
MaterializedField field = MaterializedField.create(defBuilder.build());
@@ -256,15 +251,13 @@ public class TestValueVector {
@Test
public void testBitVector() {
// Build a required boolean field definition
- SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
- SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ MajorType.Builder typeBuilder = MajorType.newBuilder();
+ FieldDef.Builder defBuilder = FieldDef.newBuilder();
typeBuilder
- .setMinorType(SchemaDefProtos.MinorType.BOOLEAN)
- .setMode(SchemaDefProtos.DataMode.REQUIRED)
+ .setMinorType(MinorType.BOOLEAN)
+ .setMode(DataMode.REQUIRED)
.setWidth(4);
defBuilder
- .setFieldId(1)
- .setParentId(0)
.setMajorType(typeBuilder.build());
MaterializedField field = MaterializedField.create(defBuilder.build());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3d41be41/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
index d8966ae..7c9e8f4 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
@@ -19,6 +19,7 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.vector.ValueVector;
import org.junit.Ignore;
import org.junit.Test;
@@ -34,7 +35,7 @@ public class JSONRecordReaderTest {
class MockOutputMutator implements OutputMutator {
List<MaterializedField> removedFields = Lists.newArrayList();
- List<ValueVector<?>> addFields = Lists.newArrayList();
+ List<ValueVector> addFields = Lists.newArrayList();
@Override
public void removeField(MaterializedField field) throws SchemaChangeException {
@@ -42,7 +43,7 @@ public class JSONRecordReaderTest {
}
@Override
- public void addField(ValueVector<?> vector) throws SchemaChangeException {
+ public void addField(ValueVector vector) throws SchemaChangeException {
addFields.add(vector);
}
@@ -54,7 +55,7 @@ public class JSONRecordReaderTest {
return removedFields;
}
- List<ValueVector<?>> getAddFields() {
+ List<ValueVector> getAddFields() {
return addFields;
}
}
@@ -69,7 +70,7 @@ public class JSONRecordReaderTest {
return;
}
- T val = (T) valueVector.getObject(index);
+ T val = (T) valueVector.getAccessor().getObject(index);
if (val instanceof byte[]) {
assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
} else {
@@ -89,15 +90,15 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_1.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector<?>> addFields = mutator.getAddFields();
+ List<ValueVector> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(2, jr.next());
assertEquals(3, addFields.size());
assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
- assertField(addFields.get(1), 0, MinorType.BOOLEAN, 1, "b");
+ assertField(addFields.get(1), 0, MinorType.BOOLEAN, true, "b");
assertField(addFields.get(2), 0, MinorType.VARCHAR4, "hi!".getBytes(UTF_8), "c");
assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
- assertField(addFields.get(1), 1, MinorType.BOOLEAN, 0, "b");
+ assertField(addFields.get(1), 1, MinorType.BOOLEAN, false, "b");
assertField(addFields.get(2), 1, MinorType.VARCHAR4, "drill!".getBytes(UTF_8), "c");
assertEquals(0, jr.next());
@@ -116,7 +117,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_2.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector<?>> addFields = mutator.getAddFields();
+ List<ValueVector> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(3, jr.next());
@@ -124,25 +125,25 @@ public class JSONRecordReaderTest {
assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
assertField(addFields.get(1), 0, MinorType.INT, 1, "b");
assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c");
- assertField(addFields.get(3), 0, MinorType.BOOLEAN, 1, "bool");
+ assertField(addFields.get(3), 0, MinorType.BOOLEAN, true, "bool");
assertField(addFields.get(4), 0, MinorType.VARCHAR4, "test1".getBytes(UTF_8), "str1");
assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
assertField(addFields.get(1), 1, MinorType.INT, 3, "b");
- assertField(addFields.get(3), 1, MinorType.BOOLEAN, 0, "bool");
+ assertField(addFields.get(3), 1, MinorType.BOOLEAN, false, "bool");
assertField(addFields.get(4), 1, MinorType.VARCHAR4, "test2".getBytes(UTF_8), "str1");
assertField(addFields.get(5), 1, MinorType.INT, 4, "d");
assertField(addFields.get(0), 2, MinorType.INT, 12345, "test");
assertField(addFields.get(2), 2, MinorType.FLOAT4, (float) 5.16, "c");
- assertField(addFields.get(3), 2, MinorType.BOOLEAN, 1, "bool");
+ assertField(addFields.get(3), 2, MinorType.BOOLEAN, true, "bool");
assertField(addFields.get(5), 2, MinorType.INT, 6, "d");
assertField(addFields.get(6), 2, MinorType.VARCHAR4, "test3".getBytes(UTF_8), "str2");
assertTrue(mutator.getRemovedFields().isEmpty());
assertEquals(0, jr.next());
}
- @Test
+ @Test @Ignore
public void testChangedSchemaInTwoBatches(@Injectable final FragmentContext context) throws IOException,
ExecutionSetupException {
new Expectations() {
@@ -155,7 +156,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_2.json"), 64); // batch only fits 1
// int
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector<?>> addFields = mutator.getAddFields();
+ List<ValueVector> addFields = mutator.getAddFields();
List<MaterializedField> removedFields = mutator.getRemovedFields();
jr.setup(mutator);
@@ -164,14 +165,14 @@ public class JSONRecordReaderTest {
assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
assertField(addFields.get(1), 0, MinorType.INT, 1, "b");
assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c");
- assertField(addFields.get(3), 0, MinorType.BOOLEAN, 1, "bool");
+ assertField(addFields.get(3), 0, MinorType.BOOLEAN, true, "bool");
assertField(addFields.get(4), 0, MinorType.VARCHAR4, "test1".getBytes(UTF_8), "str1");
assertTrue(removedFields.isEmpty());
assertEquals(1, jr.next());
assertEquals(6, addFields.size());
assertField(addFields.get(0), 0, MinorType.INT, 1234, "test");
assertField(addFields.get(1), 0, MinorType.INT, 3, "b");
- assertField(addFields.get(3), 0, MinorType.BOOLEAN, 0, "bool");
+ assertField(addFields.get(3), 0, MinorType.BOOLEAN, false, "bool");
assertField(addFields.get(4), 0, MinorType.VARCHAR4, "test2".getBytes(UTF_8), "str1");
assertField(addFields.get(5), 0, MinorType.INT, 4, "d");
assertEquals(1, removedFields.size());
@@ -180,7 +181,7 @@ public class JSONRecordReaderTest {
assertEquals(1, jr.next());
assertEquals(8, addFields.size()); // The reappearing of field 'c' is also included
assertField(addFields.get(0), 0, MinorType.INT, 12345, "test");
- assertField(addFields.get(3), 0, MinorType.BOOLEAN, 1, "bool");
+ assertField(addFields.get(3), 0, MinorType.BOOLEAN, true, "bool");
assertField(addFields.get(5), 0, MinorType.INT, 6, "d");
assertField(addFields.get(6), 0, MinorType.FLOAT4, (float) 5.16, "c");
assertField(addFields.get(7), 0, MinorType.VARCHAR4, "test3".getBytes(UTF_8), "str2");
@@ -190,7 +191,7 @@ public class JSONRecordReaderTest {
assertEquals(0, jr.next());
}
- @Test
+ @Test @Ignore
public void testNestedFieldInSameBatch(@Injectable final FragmentContext context) throws ExecutionSetupException {
new Expectations() {
{
@@ -202,7 +203,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_3.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector<?>> addFields = mutator.getAddFields();
+ List<ValueVector> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(2, jr.next());
assertEquals(5, addFields.size());
@@ -210,12 +211,12 @@ public class JSONRecordReaderTest {
assertField(addFields.get(1), 0, MinorType.MAP, null, "a");
assertField(addFields.get(2), 0, MinorType.VARCHAR4, "test".getBytes(UTF_8), "b");
assertField(addFields.get(3), 0, MinorType.MAP, null, "a");
- assertField(addFields.get(4), 0, MinorType.BOOLEAN, 1, "d");
+ assertField(addFields.get(4), 0, MinorType.BOOLEAN, true, "d");
assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
assertField(addFields.get(1), 1, MinorType.MAP, null, "a");
assertField(addFields.get(2), 1, MinorType.VARCHAR4, "test2".getBytes(UTF_8), "b");
assertField(addFields.get(3), 1, MinorType.MAP, null, "a");
- assertField(addFields.get(4), 1, MinorType.BOOLEAN, 0, "d");
+ assertField(addFields.get(4), 1, MinorType.BOOLEAN, true, "d");
assertEquals(0, jr.next());
assertTrue(mutator.getRemovedFields().isEmpty());