You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by sm...@apache.org on 2014/12/02 14:53:35 UTC
[3/4] drill git commit: DRILL-1764: return max value capacity if
vector has no children
DRILL-1764: return max value capacity if vector has no children
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/98a90d6e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/98a90d6e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/98a90d6e
Branch: refs/heads/master
Commit: 98a90d6e8aa261a6fbcc574b6dac2f70a1638938
Parents: 3581a32
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Mon Dec 1 20:52:01 2014 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Tue Dec 2 04:55:11 2014 -0800
----------------------------------------------------------------------
.../exec/store/mongo/MongoRecordReader.java | 2 +-
.../exec/store/easy/json/JSONRecordReader.java | 9 +++--
.../drill/exec/vector/complex/MapVector.java | 2 +-
.../exec/vector/complex/fn/JsonReader.java | 38 ++++++++++----------
.../complex/impl/VectorContainerWriter.java | 8 ++---
.../exec/store/json/TestJsonRecordReader.java | 6 ++++
.../resources/store/json/value-capacity.json | 2 ++
7 files changed, 39 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
index 95c4b64..79abe60 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
@@ -195,7 +195,7 @@ public class MongoRecordReader extends AbstractRecordReader {
writer.setPosition(docCount);
String doc = cursor.next().toString();
jsonReader.setSource(doc.getBytes(Charsets.UTF_8));
- if(jsonReader.write(writer)) {
+ if(jsonReader.write(writer)== JsonReader.ReadState.WRITE_SUCCEED) {
docCount++;
break;
}else{
http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
index 557592b..2582e34 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
@@ -33,6 +33,7 @@ import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.store.AbstractRecordReader;
import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.complex.fn.JsonReader;
+import org.apache.drill.exec.vector.complex.fn.JsonReader.ReadState;
import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -104,13 +105,14 @@ public class JSONRecordReader extends AbstractRecordReader {
writer.reset();
recordCount = 0;
+ ReadState write = null;
// Stopwatch p = new Stopwatch().start();
try{
outside: while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION){
writer.setPosition(recordCount);
- boolean write = jsonReader.write(writer);
+ write = jsonReader.write(writer);
- if(write){
+ if(write == ReadState.WRITE_SUCCEED){
// logger.debug("Wrote record.");
recordCount++;
}else{
@@ -125,6 +127,9 @@ public class JSONRecordReader extends AbstractRecordReader {
writer.setValueCount(recordCount);
// p.stop();
// System.out.println(String.format("Wrote %d records in %dms.", recordCount, p.elapsed(TimeUnit.MILLISECONDS)));
+ if (recordCount == 0 && write == ReadState.WRITE_FAILURE) {
+ throw new IOException("Record was too large to copy into vector.");
+ }
return recordCount;
http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 47b8252..6e26547 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -292,7 +292,7 @@ public class MapVector extends AbstractContainerVector {
@Override
public int getValueCapacity() {
if (this.vectors.isEmpty()) {
- return 0;
+ return Integer.MAX_VALUE;
}
final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index cc06b05..e209917 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -70,7 +70,11 @@ public class JsonReader {
*/
private boolean onReset = false;
- public static enum ReadState {WRITE_FAILURE, END_OF_STREAM, WRITE_SUCCEED}
+ public static enum ReadState {
+ WRITE_FAILURE,
+ END_OF_STREAM,
+ WRITE_SUCCEED
+ }
public JsonReader() throws IOException {
this(null, false);
@@ -131,8 +135,7 @@ public class JsonReader {
}
- public boolean write(ComplexWriter writer) throws JsonParseException, IOException {
-
+ public ReadState write(ComplexWriter writer) throws IOException {
JsonToken t = onReset ? parser.getCurrentToken() : parser.nextToken();
while (!parser.hasCurrentToken() && parser.hasDataAvailable()) {
@@ -140,7 +143,7 @@ public class JsonReader {
}
if(!parser.hasCurrentToken()){
- return false;
+ return ReadState.END_OF_STREAM;
}
if(onReset){
@@ -153,27 +156,26 @@ public class JsonReader {
switch(readState){
case END_OF_STREAM:
- return false;
+ break;
case WRITE_FAILURE:
logger.debug("Ran out of space while writing object, rewinding to object start.");
parser.resetToMark();
onReset = true;
- return false;
-
+ break;
case WRITE_SUCCEED:
- return true;
-
+ break;
+ default:
+ throw new IllegalStateException();
}
- throw new IllegalStateException();
+ return readState;
}
- private ReadState writeToVector(ComplexWriter writer, JsonToken t) throws JsonParseException, IOException {
+ private ReadState writeToVector(ComplexWriter writer, JsonToken t) throws IOException {
if (!writer.ok()) {
return ReadState.WRITE_FAILURE;
}
-
switch (t) {
case START_OBJECT:
writeDataSwitch(writer.rootAsMap());
@@ -198,7 +200,7 @@ public class JsonReader {
}
}
- private void writeDataSwitch(MapWriter w) throws JsonParseException, IOException{
+ private void writeDataSwitch(MapWriter w) throws IOException{
if(this.allTextMode){
writeDataAllText(w, this.selection);
}else{
@@ -206,7 +208,7 @@ public class JsonReader {
}
}
- private void writeDataSwitch(ListWriter w) throws JsonParseException, IOException{
+ private void writeDataSwitch(ListWriter w) throws IOException{
if(this.allTextMode){
writeDataAllText(w);
}else{
@@ -227,7 +229,7 @@ public class JsonReader {
}
}
- private void writeData(MapWriter map, FieldSelection selection) throws JsonParseException, IOException {
+ private void writeData(MapWriter map, FieldSelection selection) throws IOException {
//
map.start();
outside: while(true) {
@@ -299,7 +301,7 @@ public class JsonReader {
}
- private void writeDataAllText(MapWriter map, FieldSelection selection) throws JsonParseException, IOException {
+ private void writeDataAllText(MapWriter map, FieldSelection selection) throws IOException {
//
map.start();
outside: while(true) {
@@ -378,7 +380,7 @@ public class JsonReader {
writer.varChar().writeVarChar(0, prepareVarCharHolder(parser.getText()), workBuf);
}
- private void writeData(ListWriter list) throws JsonParseException, IOException {
+ private void writeData(ListWriter list) throws IOException {
list.start();
outside: while (true) {
if (!list.ok()) {
@@ -431,7 +433,7 @@ public class JsonReader {
}
- private void writeDataAllText(ListWriter list) throws JsonParseException, IOException {
+ private void writeDataAllText(ListWriter list) throws IOException {
list.start();
outside: while (true) {
if (!list.ok()) {
http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
index 6b8a523..36184a7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
@@ -45,11 +45,11 @@ public class VectorContainerWriter extends AbstractFieldWriter implements Comple
@Override
public int getValueCapacity() {
- return mapVector.getValueCapacity();
+ return mapRoot.getValueCapacity();
}
public void checkValueCapacity(){
- inform(mapVector.getValueCapacity() > idx());
+ inform(getValueCapacity() > idx());
}
public MapVector getMapVector() {
@@ -79,10 +79,6 @@ public class VectorContainerWriter extends AbstractFieldWriter implements Comple
mapRoot.setPosition(index);
}
- public MapWriter directMap() {
- return mapRoot;
- }
-
@Override
public void allocate() {
mapRoot.allocate();
http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
index 4bc1ee7..449f091 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
@@ -51,4 +51,10 @@ public class TestJsonRecordReader extends BaseTestQuery{
test("select a[0].b[1] from cp.`jsoninput/nestedArray.json`");
test("select a[1].b[1] from cp.`jsoninput/nestedArray.json`"); // index out of the range. Should return empty list.
}
+
+ @Test
+ public void testEmptyMapDoesNotFailValueCapacityCheck() throws Exception {
+ final String sql = "select * from cp.`store/json/value-capacity.json`";
+ test(sql);
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/test/resources/store/json/value-capacity.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/value-capacity.json b/exec/java-exec/src/test/resources/store/json/value-capacity.json
new file mode 100644
index 0000000..96ff644
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/value-capacity.json
@@ -0,0 +1,2 @@
+{"a":{}}
+{"a":{"b":1}}