You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by js...@apache.org on 2015/02/27 09:01:55 UTC
[1/9] drill git commit: DRILL-2031: Parquet bit column reader fix
Repository: drill
Updated Branches:
refs/heads/master d72d6030e -> 74517f5d9
DRILL-2031: Parquet bit column reader fix
Removes attempted optimized bit reader, left a comment where it was removed as a pointer back to this change set. For now the higher level Parquet interface will be used to ensure reading is correct, a fix of the optimized reader can be pursued at a later date if is is necessary.
Fixed two bugs in the test framework necessary to verify the fix against the complex parquet reader (which uses a similar interface, but it ensures that none of the other columns are read incorrectly).
Fix conflict between parquet bit reader fix and vector reallocation changes.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1d2ed349
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1d2ed349
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1d2ed349
Branch: refs/heads/master
Commit: 1d2ed349699a326165c721257937905e3043418c
Parents: d72d603
Author: Jason Altekruse <al...@gmail.com>
Authored: Mon Jan 19 15:38:46 2015 -0800
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 16:40:53 2015 -0800
----------------------------------------------------------------------
.../store/parquet/columnreaders/BitReader.java | 65 ++++++--------------
.../java/org/apache/drill/DrillTestWrapper.java | 2 +-
.../drill/exec/HyperVectorValueIterator.java | 2 +-
.../physical/impl/writer/TestParquetWriter.java | 6 ++
4 files changed, 26 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/1d2ed349/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java
index 9aabc9c..7416463 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/BitReader.java
@@ -21,6 +21,7 @@ import io.netty.buffer.ByteBuf;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.ValueVector;
import parquet.column.ColumnDescriptor;
@@ -29,10 +30,6 @@ import parquet.hadoop.metadata.ColumnChunkMetaData;
final class BitReader extends ColumnReader {
- private byte currentByte;
- private byte nextByte;
- private ByteBuf bytebuf;
-
BitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
boolean fixedLength, ValueVector v, SchemaElement schemaElement) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
@@ -44,49 +41,23 @@ final class BitReader extends ColumnReader {
recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
- pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
- readStartInBytes = pageReader.readPosInBytes;
- readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
- readLength = (int) Math.ceil(readLengthInBits / 8.0);
-
- bytebuf = pageReader.pageDataByteArray;
- // standard read, using memory mapping
- if (pageReader.bitShift == 0) {
- ((BaseDataValueVector) valueVec).getData().writeBytes(bytebuf,
- (int) readStartInBytes, (int) readLength);
- } else { // read in individual values, because a bitshift is necessary with where the last page or batch ended
-
- vectorData = ((BaseDataValueVector) valueVec).getData();
- nextByte = bytebuf.getByte((int) Math.max(0, Math.ceil(pageReader.valuesRead / 8.0) - 1));
- readLengthInBits = recordsReadInThisIteration + pageReader.bitShift;
-
- int i = 0;
- // read individual bytes with appropriate shifting
- for (; i < (int) readLength; i++) {
- currentByte = nextByte;
- currentByte = (byte) (currentByte >>> pageReader.bitShift);
- // mask the bits about to be added from the next byte
- currentByte = (byte) (currentByte & ParquetRecordReader.startBitMasks[pageReader.bitShift - 1]);
- // if we are not on the last byte
- if ((int) Math.ceil(pageReader.valuesRead / 8.0) + i < pageReader.byteLength) {
- // grab the next byte from the buffer, shift and mask it, and OR it with the leftover bits
- nextByte = bytebuf.getByte((int) Math.ceil(pageReader.valuesRead / 8.0) + i);
- currentByte = (byte) (currentByte | nextByte
- << (8 - pageReader.bitShift)
- & ParquetRecordReader.endBitMasks[8 - pageReader.bitShift - 1]);
- }
- vectorData.setByte(valuesReadInCurrentPass / 8 + i, currentByte);
- }
- vectorData.setIndex(0, (valuesReadInCurrentPass / 8)
- + (int) readLength - 1);
- vectorData.capacity(vectorData.writerIndex() + 1);
- }
-
- // check if the values in this page did not end on a byte boundary, store a number of bits the next page must be
- // shifted by to read all of the values into the vector without leaving space
- if (readLengthInBits % 8 != 0) {
- pageReader.bitShift = (int) readLengthInBits % 8;
- } else {
- pageReader.bitShift = 0;
+ // A more optimized reader for bit columns was removed to fix the bug
+ // DRILL-2031. It attempted to copy large runs of values directly from the
+ // decompressed parquet stream into a BitVector. This was complicated by
+ // parquet not always breaking a page on a row number divisible by 8. In
+ // this case the batch would have to be cut off early or we would have to
+ // copy the next page byte-by-byte with a bit shift to move the values into
+ // the correct position (to make the value vector one contiguous buffer of
+ // data). As page boundaries do not line up across columns, cutting off a
+ // batch at every page boundary of a bit column could be costly with many
+ // such pages, so we opted to try to shift the bits when necessary.
+ //
+ // In the end, this was too much complexity for not enough performance
+ // benefit, for now this reader has been moved to use the higher level value
+ // by value reader provided by the parquet library.
+ for (int i = 0; i < recordsReadInThisIteration; i++){
+ ((BitVector)valueVec).getMutator().setSafe(i + valuesReadInCurrentPass,
+ pageReader.valueReader.readBoolean() ? 1 : 0 );
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/1d2ed349/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
index f06203e..75a91b3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
@@ -299,7 +299,7 @@ public class DrillTestWrapper {
*/
protected void compareOrderedResults() throws Exception {
if (highPerformanceComparison) {
- if (baselineQueryType != null) {
+ if (baselineQueryType == null) {
throw new Exception("Cannot do a high performance comparison without using a baseline file");
}
compareResultsHyperVector();
http://git-wip-us.apache.org/repos/asf/drill/blob/1d2ed349/exec/java-exec/src/test/java/org/apache/drill/exec/HyperVectorValueIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/HyperVectorValueIterator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/HyperVectorValueIterator.java
index d214b7c..9ad72eb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/HyperVectorValueIterator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/HyperVectorValueIterator.java
@@ -80,7 +80,7 @@ public class HyperVectorValueIterator implements Iterator<Object> {
@Override
public Object next() {
- if (currVec == null || indexInCurrentVector == currVec.getValueCapacity()) {
+ if (currVec == null || indexInCurrentVector == currVec.getAccessor().getValueCount()) {
currVec = hyperVector.getValueVectors()[indexInVectorList];
indexInVectorList++;
indexInCurrentVector = 0;
http://git-wip-us.apache.org/repos/asf/drill/blob/1d2ed349/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 6aa3288..7298f28 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -204,6 +204,12 @@ public class TestParquetWriter extends BaseTestQuery {
runTestAndValidate("*", "*", inputTable, "nullable_test");
}
+ @Ignore("Binary file too large for version control, TODO - make available on S3 bucket or similar service")
+ @Test
+ public void testBitError_Drill_2031() throws Exception {
+ compareParquetReadersHyperVector("*", "dfs.`/tmp/wide2/0_0_3.parquet`");
+ }
+
@Test
public void testDecimal() throws Exception {
String selection = "cast(salary as decimal(8,2)) as decimal8, cast(salary as decimal(15,2)) as decimal15, " +
[3/9] drill git commit: DRILL-2013: Part 2 - Improve test for flatten
bug
Posted by js...@apache.org.
DRILL-2013: Part 2 - Improve test for flatten bug
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/f378b342
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/f378b342
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/f378b342
Branch: refs/heads/master
Commit: f378b342cabc6c1e69a1df0855136ef36e523741
Parents: ce0afb1
Author: AdamPD <ad...@pharmadata.net.au>
Authored: Mon Feb 9 12:12:36 2015 +1000
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 16:41:03 2015 -0800
----------------------------------------------------------------------
.../apache/drill/exec/physical/impl/flatten/TestFlatten.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/f378b342/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
index 9518386..dc37079 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -243,7 +243,10 @@ public class TestFlatten extends BaseTestQuery {
@Test
public void testDrill_2013() throws Exception {
- test("select flatten(complex), rownum from cp.`/store/json/test_flatten_mappify2.json` where rownum > 5");
+ testBuilder()
+ .sqlQuery("select flatten(complex), rownum from cp.`/store/json/test_flatten_mappify2.json` where rownum > 5")
+ .expectsEmptyResultSet()
+ .build().run();
}
}
[5/9] drill git commit: DRILL-2294: Prevent collecting intermediate
stats before the operator tree was finished being constructed.
Posted by js...@apache.org.
DRILL-2294: Prevent collecting intermediate stats before the operator tree was finished being constructed.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8100a970
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8100a970
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8100a970
Branch: refs/heads/master
Commit: 8100a970cc958d61359c5b475b0cdfc67d72158b
Parents: a163c06
Author: Jason Altekruse <al...@gmail.com>
Authored: Wed Feb 25 10:29:19 2015 -0800
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 16:41:03 2015 -0800
----------------------------------------------------------------------
.../apache/drill/exec/work/fragment/FragmentExecutor.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/8100a970/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
index 7ccb64e..4ab3cc0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
@@ -64,10 +64,16 @@ public class FragmentExecutor implements Runnable, CancelableQuery, StatusProvid
@Override
public FragmentStatus getStatus() {
- FragmentStatus status = AbstractStatusReporter.getBuilder(context, FragmentState.RUNNING, null, null).build();
+ // If the query is not in a running state, the operator tree is still being constructed and
+ // there is no reason to poll for intermediate results.
+
+ // Previously the call to get the operator stats with the AbstractStatusReporter was happening
+ // before this check. This caused a concurrent modification exception as the list of operator
+ // stats is iterated over while collecting info, and added to while building the operator tree.
if(state.get() != FragmentState.RUNNING_VALUE){
return null;
}
+ FragmentStatus status = AbstractStatusReporter.getBuilder(context, FragmentState.RUNNING, null, null).build();
return status;
}
[8/9] drill git commit: DRILL-1953: alter session set
store.json.all_text_mode does not work as documented
Posted by js...@apache.org.
DRILL-1953: alter session set store.json.all_text_mode does not work as documented
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b49e51ef
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b49e51ef
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b49e51ef
Branch: refs/heads/master
Commit: b49e51ef5e0f925cf3505f33b65309b1b839386d
Parents: 8100a97
Author: adeneche <ad...@gmail.com>
Authored: Tue Feb 3 20:45:54 2015 -0800
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 16:41:04 2015 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/store/mongo/MongoRecordReader.java | 3 +--
.../java/org/apache/drill/exec/server/DrillbitContext.java | 4 ++++
.../apache/drill/exec/store/easy/json/JSONRecordReader.java | 2 +-
.../org/apache/drill/exec/store/json/TestJsonRecordReader.java | 6 ++++++
exec/java-exec/src/test/resources/jsoninput/big_numeric.json | 1 +
5 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/b49e51ef/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
index 5af63ff..da96701 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
@@ -100,8 +100,7 @@ public class MongoRecordReader extends AbstractRecordReader {
Map<String, List<BasicDBObject>> mergedFilters = MongoUtils.mergeFilters(
subScanSpec.getMinFilters(), subScanSpec.getMaxFilters());
buildFilters(subScanSpec.getFilter(), mergedFilters);
- enableAllTextMode = fragmentContext.getDrillbitContext().getOptionManager()
- .getOption(ExecConstants.MONGO_ALL_TEXT_MODE).bool_val;
+ enableAllTextMode = fragmentContext.getOptions().getOption(ExecConstants.MONGO_ALL_TEXT_MODE).bool_val;
init(subScanSpec);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b49e51ef/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 83a89df..0fb10ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -87,6 +87,10 @@ public class DrillbitContext {
return workBus;
}
+ /**
+ * @return the system options manager. It is important to note that this manager only contains options at the
+ * "system" level and not "session" level.
+ */
public SystemOptionManager getOptionManager() {
return systemOptions;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b49e51ef/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
index 557c0f0..c343177 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
@@ -70,7 +70,7 @@ public class JSONRecordReader extends AbstractRecordReader {
this.fileSystem = fileSystem;
this.fragmentContext = fragmentContext;
this.columns = columns;
- this.enableAllTextMode = fragmentContext.getDrillbitContext().getOptionManager().getOption(ExecConstants.JSON_ALL_TEXT_MODE).bool_val;
+ this.enableAllTextMode = fragmentContext.getOptions().getOption(ExecConstants.JSON_ALL_TEXT_MODE).bool_val;
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/b49e51ef/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
index 449f091..c4bfcce 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
@@ -57,4 +57,10 @@ public class TestJsonRecordReader extends BaseTestQuery{
final String sql = "select * from cp.`store/json/value-capacity.json`";
test(sql);
}
+
+ @Test
+ public void testEnableAllTextMode() throws Exception {
+ testNoResult("alter session set `store.json.all_text_mode`= true");
+ test("select * from cp.`jsoninput/big_numeric.json`");
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b49e51ef/exec/java-exec/src/test/resources/jsoninput/big_numeric.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/jsoninput/big_numeric.json b/exec/java-exec/src/test/resources/jsoninput/big_numeric.json
new file mode 100644
index 0000000..df13d30
--- /dev/null
+++ b/exec/java-exec/src/test/resources/jsoninput/big_numeric.json
@@ -0,0 +1 @@
+{ "a1": 14994882832830675451 }
\ No newline at end of file
[7/9] drill git commit: DRILL-1535:Logical plan deserialization error
in Join
Posted by js...@apache.org.
DRILL-1535:Logical plan deserialization error in Join
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a163c06a
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a163c06a
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a163c06a
Branch: refs/heads/master
Commit: a163c06ae581724deb0a1de3f5bd2dfa895cefa7
Parents: aa7eb2a
Author: Yash Sharma <ya...@snapdeal.com>
Authored: Thu Nov 13 00:19:34 2014 +0530
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 16:41:03 2015 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/drill/common/logical/data/Join.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a163c06a/common/src/main/java/org/apache/drill/common/logical/data/Join.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Join.java b/common/src/main/java/org/apache/drill/common/logical/data/Join.java
index df7e583..a3d44f5 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Join.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Join.java
@@ -55,8 +55,8 @@ public class Join extends LogicalOperatorBase {
this(left, right, conditions, resolve(type));
}
-
- public Join(LogicalOperator left, @JsonProperty("right") LogicalOperator right, JoinCondition[] conditions, JoinRelType type) {
+ @JsonCreator
+ public Join(@JsonProperty("left") LogicalOperator left, @JsonProperty("right") LogicalOperator right, @JsonProperty("conditions")JoinCondition[] conditions, @JsonProperty("type") JoinRelType type) {
super();
this.conditions = conditions;
this.left = left;
[9/9] drill git commit: DRILL-2321: FlattenRecordBatch should
transfer vectors honoring output field reference.
Posted by js...@apache.org.
DRILL-2321: FlattenRecordBatch should transfer vectors honoring output field reference.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/74517f5d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/74517f5d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/74517f5d
Branch: refs/heads/master
Commit: 74517f5d95300f94bad08004adb4e1a2817bbbe5
Parents: b49e51e
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Thu Feb 26 11:23:30 2015 -0800
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 22:27:58 2015 -0800
----------------------------------------------------------------------
.../impl/flatten/FlattenRecordBatch.java | 34 ++++++++------------
.../exec/vector/complex/RepeatedMapVector.java | 8 ++---
2 files changed, 17 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/74517f5d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 8a87098..7a5b352 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -262,23 +262,19 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
* the end of one of the other vectors while we are copying the data of the other vectors alongside each new flattened
* value coming out of the repeated field.)
*/
- private TransferPair getFlattenFieldTransferPair() {
- ValueVector flattenField = incoming.getValueAccessorById(
- incoming.getSchema().getColumn(
- incoming.getValueVectorId(
- popConfig.getColumn()).getFieldIds()[0]).getValueClass(),
- incoming.getValueVectorId(popConfig.getColumn()).getFieldIds()).getValueVector();
+ private TransferPair getFlattenFieldTransferPair(FieldReference reference) {
+ final TypedFieldId fieldId = incoming.getValueVectorId(popConfig.getColumn());
+ final Class vectorClass = incoming.getSchema().getColumn(fieldId.getFieldIds()[0]).getValueClass();
+ final ValueVector flattenField = incoming.getValueAccessorById(vectorClass, fieldId.getFieldIds()).getValueVector();
TransferPair tp = null;
- if (flattenField instanceof MapVector) {
- return null;
- } else if (flattenField instanceof RepeatedMapVector) {
- tp = ((RepeatedMapVector)flattenField).getTransferPairToSingleMap();
+ if (flattenField instanceof RepeatedMapVector) {
+ tp = ((RepeatedMapVector)flattenField).getTransferPairToSingleMap(reference);
} else {
ValueVector vvIn = ((RepeatedVector)flattenField).getAccessor().getAllChildValues();
// vvIn may be null because of fast schema return for repeated list vectors
if (vvIn != null) {
- tp = vvIn.getTransferPair();
+ tp = vvIn.getTransferPair(reference);
}
}
return tp;
@@ -293,15 +289,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
final List<TransferPair> transfers = Lists.newArrayList();
final ClassGenerator<Flattener> cg = CodeGenerator.getRoot(Flattener.TEMPLATE_DEFINITION, context.getFunctionRegistry());
- IntOpenHashSet transferFieldIds = new IntOpenHashSet();
+ final IntOpenHashSet transferFieldIds = new IntOpenHashSet();
- NamedExpression namedExpression = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn()));
- LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
- ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
- TypedFieldId id = vectorRead.getFieldId();
- Preconditions.checkNotNull(incoming);
-
- TransferPair tp = getFlattenFieldTransferPair();
+ final NamedExpression flattenExpr = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn()));
+ final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression)ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
+ final TransferPair tp = getFlattenFieldTransferPair(flattenExpr.getRef());
if (tp != null) {
transfers.add(tp);
@@ -314,7 +306,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
ClassifierResult result = new ClassifierResult();
for (int i = 0; i < exprs.size(); i++) {
- namedExpression = exprs.get(i);
+ final NamedExpression namedExpression = exprs.get(i);
result.clear();
String outputName = getRef(namedExpression).getRootSegment().getPath();
@@ -327,7 +319,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
}
}
- expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
final MaterializedField outputField = MaterializedField.create(outputName, expr.getMajorType());
if (collector.hasErrors()) {
throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
http://git-wip-us.apache.org/repos/asf/drill/blob/74517f5d/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index ad8c66f..eb045d0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -159,8 +159,8 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedFixe
}
}
- public TransferPair getTransferPairToSingleMap() {
- return new SingleMapTransferPair(this, getField());
+ public TransferPair getTransferPairToSingleMap(FieldReference reference) {
+ return new SingleMapTransferPair(this, reference);
}
@Override
@@ -183,8 +183,8 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedFixe
private final MapVector to;
private static final MajorType MAP_TYPE = Types.required(MinorType.MAP);
- public SingleMapTransferPair(RepeatedMapVector from, MaterializedField field) {
- this(from, new MapVector(MaterializedField.create(field.getPath(), MAP_TYPE), from.allocator, from.callBack), false);
+ public SingleMapTransferPair(RepeatedMapVector from, SchemaPath path) {
+ this(from, new MapVector(MaterializedField.create(path, MAP_TYPE), from.allocator, from.callBack), false);
}
public SingleMapTransferPair(RepeatedMapVector from, MapVector to) {
[2/9] drill git commit: DRILL-1971: Mongo Storage Plugin slowness
issue
Posted by js...@apache.org.
DRILL-1971: Mongo Storage Plugin slowness issue
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/619c2fbd
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/619c2fbd
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/619c2fbd
Branch: refs/heads/master
Commit: 619c2fbd77adbfd2b28bf79ddd44f298f6a210f4
Parents: f378b34
Author: akumarb2010 <ak...@gmail.com>
Authored: Fri Jan 9 20:49:59 2015 +0530
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 16:41:03 2015 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/store/mongo/MongoRecordReader.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/619c2fbd/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
index 4b73600..5af63ff 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
@@ -191,14 +191,13 @@ public class MongoRecordReader extends AbstractRecordReader {
try {
String errMsg = "Document {} is too big to fit into allocated ValueVector";
- done: for (; rowCount < TARGET_RECORD_COUNT && cursor.hasNext(); rowCount++) {
+ for (; rowCount < TARGET_RECORD_COUNT && cursor.hasNext(); rowCount++) {
writer.setPosition(docCount);
String doc = cursor.next().toString();
jsonReader.setSource(doc.getBytes(Charsets.UTF_8));
- if(jsonReader.write(writer)== JsonReader.ReadState.WRITE_SUCCEED) {
+ if (jsonReader.write(writer) == JsonReader.ReadState.WRITE_SUCCEED) {
docCount++;
- break;
- }else{
+ } else {
if (docCount == 0) {
throw new DrillRuntimeException(errMsg);
}
[4/9] drill git commit: DRILL-2013: Fix flatten when no rows match
filters
Posted by js...@apache.org.
DRILL-2013: Fix flatten when no rows match filters
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ce0afb1d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ce0afb1d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ce0afb1d
Branch: refs/heads/master
Commit: ce0afb1d62b519772084173ed5540cda79a2719b
Parents: 1d2ed34
Author: AdamPD <ad...@pharmadata.net.au>
Authored: Fri Jan 23 17:01:36 2015 +1000
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 16:41:03 2015 -0800
----------------------------------------------------------------------
.../drill/exec/physical/impl/flatten/FlattenRecordBatch.java | 2 +-
.../apache/drill/exec/physical/impl/flatten/TestFlatten.java | 5 +++++
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/ce0afb1d/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 02c154f..8a87098 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -152,7 +152,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
// inside of the the flattener for the current batch
setFlattenVector();
- int childCount = flattener.getFlattenField().getAccessor().getValueCount();
+ int childCount = incomingRecordCount == 0 ? 0 : flattener.getFlattenField().getAccessor().getValueCount();
int outputRecords = flattener.flattenRecords(0, incomingRecordCount, 0);
// TODO - change this to be based on the repeated vector length
if (outputRecords < childCount) {
http://git-wip-us.apache.org/repos/asf/drill/blob/ce0afb1d/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
index b56df84..9518386 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -241,4 +241,9 @@ public class TestFlatten extends BaseTestQuery {
}
+ @Test
+ public void testDrill_2013() throws Exception {
+ test("select flatten(complex), rownum from cp.`/store/json/test_flatten_mappify2.json` where rownum > 5");
+ }
+
}
[6/9] drill git commit: DRILL-1515: For the Web UI users,
if exception is thrown, users are redirected to another page,
which shows the error message
Posted by js...@apache.org.
DRILL-1515: For the Web UI users, if exception is thrown, users are redirected to another page, which shows the error message
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/aa7eb2a9
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/aa7eb2a9
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/aa7eb2a9
Branch: refs/heads/master
Commit: aa7eb2a927faf86a00de3dc0a3cd75862a90a8cc
Parents: 619c2fb
Author: Hsuan-Yi Chu <hs...@usc.edu>
Authored: Mon Jan 5 10:28:29 2015 -0800
Committer: Jason Altekruse <al...@gmail.com>
Committed: Thu Feb 26 16:41:03 2015 -0800
----------------------------------------------------------------------
.../drill/exec/server/rest/QueryResources.java | 9 ++++++--
.../main/resources/rest/query/errorMessage.ftl | 24 ++++++++++++++++++++
2 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/aa7eb2a9/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
index 9ef6676..145a476 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
@@ -68,8 +68,13 @@ public class QueryResources {
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@Produces(MediaType.TEXT_HTML)
public Viewable submitQuery(@FormParam("query") String query, @FormParam("queryType") String queryType) throws Exception {
- final QueryWrapper.QueryResult result = submitQueryJSON(new QueryWrapper(query, queryType));
- return new Viewable("/rest/query/result.ftl", new TabularResult(result));
+ try {
+ final QueryWrapper.QueryResult result = submitQueryJSON(new QueryWrapper(query, queryType));
+ return new Viewable("/rest/query/result.ftl", new TabularResult(result));
+ } catch(Exception | Error e) {
+ logger.error("Query from Web UI Failed", e);
+ return new Viewable("/rest/query/errorMessage.ftl", e);
+ }
}
public static class TabularResult {
http://git-wip-us.apache.org/repos/asf/drill/blob/aa7eb2a9/exec/java-exec/src/main/resources/rest/query/errorMessage.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/query/errorMessage.ftl b/exec/java-exec/src/main/resources/rest/query/errorMessage.ftl
new file mode 100644
index 0000000..dbdcc9e
--- /dev/null
+++ b/exec/java-exec/src/main/resources/rest/query/errorMessage.ftl
@@ -0,0 +1,24 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+ <a href="/queries">back</a><br/>
+ <div class="page-header">
+ </div>
+ <h2> Query Failed: An Error Occurred </h2>
+ ${model}
+</#macro>
+
+<@page_html/>
\ No newline at end of file