You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/08/29 05:43:37 UTC
[4/7] git commit: DRILL-190 (part3) Fixes for integration between
JSON changes, updated CodeGenerator and merge join
DRILL-190 (part3) Fixes for integration between JSON changes, updated CodeGenerator and merge join
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5232b0e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5232b0e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5232b0e1
Branch: refs/heads/master
Commit: 5232b0e1423deecb206e79d5978e4fe7db8197b6
Parents: e0bac2f
Author: Jacques Nadeau <ja...@apache.org>
Authored: Wed Aug 28 17:48:36 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 28 20:37:40 2013 -0700
----------------------------------------------------------------------
.../drill/exec/physical/impl/ScanBatch.java | 2 +-
.../physical/impl/aggregate/InternalBatch.java | 2 +-
.../exec/physical/impl/join/JoinEvaluator.java | 10 ------
.../physical/impl/join/JoinInnerSignature.java | 35 --------------------
.../exec/physical/impl/join/MergeJoinBatch.java | 4 +--
.../partitionsender/OutgoingRecordBatch.java | 2 +-
.../drill/exec/record/AbstractRecordBatch.java | 2 +-
.../drill/exec/record/RecordBatchLoader.java | 2 +-
.../drill/exec/record/VectorContainer.java | 2 +-
.../exec/physical/impl/join/TestMergeJoin.java | 3 +-
.../test/resources/join/merge_single_batch.json | 20 +++++++----
11 files changed, 24 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 192c03c..ae043ec 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -134,7 +134,7 @@ public class ScanBatch implements RecordBatch {
@Override
public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz) {
- return container.getVectorAccessor(fieldId, clazz);
+ return container.getValueAccessorById(fieldId, clazz);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java
index 343dbe5..77dd682 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/InternalBatch.java
@@ -59,7 +59,7 @@ public class InternalBatch implements Iterable<VectorWrapper<?>>{
}
public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz){
- return container.getVectorAccessor(fieldId, clazz);
+ return container.getValueAccessorById(fieldId, clazz);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinEvaluator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinEvaluator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinEvaluator.java
deleted file mode 100644
index beb3e28..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinEvaluator.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.drill.exec.physical.impl.join;
-
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.record.VectorContainer;
-
-public interface JoinEvaluator {
- public abstract void doSetup(FragmentContext context, JoinStatus status, VectorContainer outgoing) throws SchemaChangeException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinInnerSignature.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinInnerSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinInnerSignature.java
deleted file mode 100644
index 1081244..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinInnerSignature.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-
-package org.apache.drill.exec.physical.impl.join;
-
-import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM;
-
-import javax.inject.Named;
-
-import org.apache.drill.exec.compile.sig.CodeGeneratorSignature;
-import org.apache.drill.exec.compile.sig.GeneratorMapping;
-import org.apache.drill.exec.compile.sig.MappingSet;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.record.VectorContainer;
-
-
-public interface JoinInnerSignature extends CodeGeneratorSignature {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index a2b84da..af33ca4 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -317,7 +317,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
JVar vvIn = cg.declareVectorValueSetupAndMember("incomingLeft",
new TypedFieldId(vw.getField().getType(), vectorId));
JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing",
- new TypedFieldId(vw.getField().getType(),vectorId, true));
+ new TypedFieldId(vw.getField().getType(),vectorId));
// todo: check for room in vvOut
cg.getEvalBlock().add(vvOut.invoke("copyFrom")
.arg(COPY_LEFT_MAPPING.getValueReadIndex())
@@ -336,7 +336,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
JVar vvIn = cg.declareVectorValueSetupAndMember("incomingRight",
new TypedFieldId(vw.getField().getType(), vectorId - rightVectorBase));
JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing",
- new TypedFieldId(vw.getField().getType(),vectorId, true));
+ new TypedFieldId(vw.getField().getType(),vectorId));
cg.getEvalBlock().add(vvOut.invoke("copyFrom")
.arg(COPY_RIGHT_MAPPING.getValueReadIndex())
.arg(COPY_RIGHT_MAPPING.getValueWriteIndex())
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
index 0cefc52..e429402 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
@@ -222,7 +222,7 @@ public class OutgoingRecordBatch implements RecordBatch {
@Override
public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz) {
- return vectorContainer.getVectorAccessor(fieldId, clazz);
+ return vectorContainer.getValueAccessorById(fieldId, clazz);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
index a2584b8..ccd2468 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
@@ -65,7 +65,7 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements
@Override
public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz) {
- return container.getVectorAccessor(fieldId, clazz);
+ return container.getValueAccessorById(fieldId, clazz);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
index 4d47404..9ac53f0 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
@@ -140,7 +140,7 @@ public class RecordBatchLoader implements Iterable<VectorWrapper<?>>{
}
public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz){
- return container.getVectorAccessor(fieldId, clazz);
+ return container.getValueAccessorById(fieldId, clazz);
}
public WritableBatch getWritableBatch(){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index 25036fc..e6c8bab 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -108,7 +108,7 @@ public class VectorContainer implements Iterable<VectorWrapper<?>> {
}
@SuppressWarnings("unchecked")
- public <T extends ValueVector> VectorWrapper<T> getVectorAccessor(int fieldId, Class<?> clazz) {
+ public <T extends ValueVector> VectorWrapper<T> getValueAccessorById(int fieldId, Class<?> clazz) {
VectorWrapper<?> va = wrappers.get(fieldId);
assert va != null;
if (va.getVectorClass() != clazz) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
index 38b8225..6aa651b 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
@@ -112,9 +112,10 @@ public class TestMergeJoin {
new NonStrictExpectations(){{
bitContext.getMetrics(); result = new MetricRegistry("test");
bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+ bitContext.getConfig(); result = c;
}};
- PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext));
PhysicalPlan plan = reader.readPhysicalPlan(
Files.toString(
FileUtils.getResourceAsFile("/join/merge_single_batch.json"), Charsets.UTF_8)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5232b0e1/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json
index 0e4f79d..ad33d26 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json
@@ -10,16 +10,24 @@
{
@id:1,
pop:"json-sub-scan",
- entries:[
- {url: "#{LEFT_FILE}"}
- ]
+ readEntries:[
+ {path: "#{LEFT_FILE}"}
+ ],
+ engineConfig:{
+ "type":"json",
+ "dfsName" : "file:///"
+ }
},
{
@id:2,
pop:"json-sub-scan",
- entries:[
- {url: "#{RIGHT_FILE}"}
- ]
+ readEntries:[
+ {path: "#{RIGHT_FILE}"}
+ ],
+ engineConfig:{
+ "type":"json",
+ "dfsName" : "file:///"
+ }
},
{
@id: 3,