You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2017/09/05 04:38:57 UTC
[3/4] drill git commit: DRILL-4264: Allow field names to include dots
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
index f7fd1d6..e5ba98f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
@@ -293,7 +293,8 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
continue;
}
keyExprs[i] = expr;
- final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
+ final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(),
+ expr.getMajorType());
@SuppressWarnings("resource")
final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
keyOutputIds[i] = container.add(vector);
@@ -309,7 +310,8 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
continue;
}
- final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
+ final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(),
+ expr.getMajorType());
@SuppressWarnings("resource")
ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
TypedFieldId id = container.add(vector);
@@ -425,17 +427,17 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
case FOUR_BYTE: {
JVar var = g.declareClassField("sv4_", g.getModel()._ref(SelectionVector4.class));
g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector4"));
- g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
+ g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));
return;
}
case NONE: {
- g.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));;
+ g.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));
return;
}
case TWO_BYTE: {
JVar var = g.declareClassField("sv2_", g.getModel()._ref(SelectionVector2.class));
g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector2"));
- g.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));;
+ g.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));
return;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
index 436480e..387dad1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -196,7 +196,8 @@ public class ChainedHashTable {
*/
for (NamedExpression ne : htConfig.getKeyExprsBuild()) {
LogicalExpression expr = keyExprsBuild[i];
- final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
+ final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(),
+ expr.getMajorType());
@SuppressWarnings("resource")
ValueVector vv = TypeHelper.getNewVector(outputField, allocator);
htKeyFieldIds[i] = htContainerOrig.add(vv);
@@ -318,8 +319,7 @@ public class ChainedHashTable {
* aggregate. For join we need to hash everything as double (both for distribution and for comparison) but
* for aggregation we can avoid the penalty of casting to double
*/
- LogicalExpression hashExpression = HashPrelUtil.getHashExpression(Arrays.asList(keyExprs),
- incomingProbe != null ? true : false);
+ LogicalExpression hashExpression = HashPrelUtil.getHashExpression(Arrays.asList(keyExprs), incomingProbe != null);
final LogicalExpression materializedExpr = ExpressionTreeMaterializer.materializeAndCheckErrors(hashExpression, batch, context.getFunctionRegistry());
HoldingContainer hash = cg.addExpr(materializedExpr);
cg.getEvalBlock()._return(hash.getValue());
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 8fd9441..e64e919 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -253,8 +253,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
}
private FieldReference getRef(NamedExpression e) {
- final FieldReference ref = e.getRef();
- return ref;
+ return e.getRef();
}
/**
@@ -333,8 +332,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
ClassifierResult result = new ClassifierResult();
- for (int i = 0; i < exprs.size(); i++) {
- final NamedExpression namedExpression = exprs.get(i);
+ for (NamedExpression namedExpression : exprs) {
result.clear();
String outputName = getRef(namedExpression).getRootSegment().getPath();
@@ -408,10 +406,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
List<NamedExpression> exprs = Lists.newArrayList();
for (MaterializedField field : incoming.getSchema()) {
- if (field.getPath().equals(popConfig.getColumn().getAsUnescapedPath())) {
+ String fieldName = field.getName();
+ if (fieldName.equals(popConfig.getColumn().getRootSegmentPath())) {
continue;
}
- exprs.add(new NamedExpression(SchemaPath.getSimplePath(field.getPath()), new FieldReference(field.getPath())));
+ exprs.add(new NamedExpression(SchemaPath.getSimplePath(fieldName), new FieldReference(fieldName)));
}
return exprs;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 72b8833..1f74ba1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -477,7 +477,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
outputType = inputType;
}
- final ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getPath(), outputType));
+ final ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getName(), outputType));
if (v instanceof AbstractContainerVector) {
vv.getValueVector().makeTransferPair(v);
v.clear();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index c351517..e599702 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -400,7 +400,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
} else {
outputType = inputType;
}
- MaterializedField newField = MaterializedField.create(w.getField().getPath(), outputType);
+ MaterializedField newField = MaterializedField.create(w.getField().getName(), outputType);
ValueVector v = container.addOrGet(newField);
if (v instanceof AbstractContainerVector) {
w.getValueVector().makeTransferPair(v);
@@ -417,7 +417,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
} else {
outputType = inputType;
}
- MaterializedField newField = MaterializedField.create(w.getField().getPath(), outputType);
+ MaterializedField newField = MaterializedField.create(w.getField().getName(), outputType);
ValueVector v = container.addOrGet(newField);
if (v instanceof AbstractContainerVector) {
w.getValueVector().makeTransferPair(v);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
index 8336e86..35cc710 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
@@ -317,7 +317,7 @@ public class NestedLoopJoinBatch extends AbstractRecordBatch<NestedLoopJoinPOP>
outputType = inputType;
}
- MaterializedField newField = MaterializedField.create(field.getPath(), outputType);
+ MaterializedField newField = MaterializedField.create(field.getName(), outputType);
container.addOrGet(newField);
JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer",
@@ -384,7 +384,7 @@ public class NestedLoopJoinBatch extends AbstractRecordBatch<NestedLoopJoinPOP>
} else {
outputType = inputType;
}
- MaterializedField newField = MaterializedField.create(vectorWrapper.getField().getPath(), outputType);
+ MaterializedField newField = MaterializedField.create(vectorWrapper.getField().getName(), outputType);
ValueVector valueVector = container.addOrGet(newField);
if (valueVector instanceof AbstractContainerVector) {
vectorWrapper.getValueVector().makeTransferPair(valueVector);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index fede487..63133d4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -431,11 +431,10 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
int i = 0;
for (Ordering od : orderings) {
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), incoming, collector, context.getFunctionRegistry());
- SchemaPath schemaPath = SchemaPath.getSimplePath("f" + i++);
TypeProtos.MajorType.Builder builder = TypeProtos.MajorType.newBuilder().mergeFrom(expr.getMajorType())
.clearMode().setMode(TypeProtos.DataMode.REQUIRED);
TypeProtos.MajorType newType = builder.build();
- MaterializedField outputField = MaterializedField.create(schemaPath.getAsUnescapedPath(), newType);
+ MaterializedField outputField = MaterializedField.create("f" + i++, newType);
if (collector.hasErrors()) {
throw new SchemaChangeException(String.format(
"Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
@@ -533,7 +532,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
// if this now that all the batches on the queue are processed, we begin processing the incoming batches. For the
// first one
// we need to generate a new schema, even if the outcome is IterOutcome.OK After that we can reuse the schema.
- if (this.startedUnsampledBatches == false) {
+ if (!this.startedUnsampledBatches) {
this.startedUnsampledBatches = true;
if (upstream == IterOutcome.OK) {
upstream = IterOutcome.OK_NEW_SCHEMA;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
index 589754f..3afa852 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -99,9 +99,9 @@ public class ProducerConsumerBatch extends AbstractRecordBatch<ProducerConsumer>
final MaterializedField field = schema.getColumn(i);
final MajorType type = field.getType();
final ValueVector vOut = container.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()),
- container.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds()).getValueVector();
+ container.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds()).getValueVector();
final ValueVector vIn = newContainer.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()),
- newContainer.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds()).getValueVector();
+ newContainer.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds()).getValueVector();
final TransferPair tp = vIn.makeTransferPair(vOut);
tp.transfer();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 6baf070..9a72fcb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -202,7 +202,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
this.recordCount = remainderIndex;
} else {
setValueCount(incomingRecordCount);
- for(final VectorWrapper<?> v: incoming) {
+ for (final VectorWrapper<?> v: incoming) {
v.clear();
}
this.recordCount = outputRecords;
@@ -331,8 +331,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final ClassifierResult result = new ClassifierResult();
final boolean classify = isClassificationNeeded(exprs);
- for (int i = 0; i < exprs.size(); i++) {
- final NamedExpression namedExpression = exprs.get(i);
+ for (NamedExpression namedExpression : exprs) {
result.clear();
if (classify && namedExpression.getExpr() instanceof SchemaPath) {
@@ -341,15 +340,15 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
if (result.isStar) {
// The value indicates which wildcard we are processing now
final Integer value = result.prefixMap.get(result.prefix);
- if (value != null && value.intValue() == 1) {
+ if (value != null && value == 1) {
int k = 0;
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- if (k > result.outputNames.size()-1) {
+ if (k > result.outputNames.size() - 1) {
assert false;
}
final String name = result.outputNames.get(k++); // get the renamed column names
- if (name == EMPTY_STRING) {
+ if (name.isEmpty()) {
continue;
}
@@ -362,16 +361,16 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final TransferPair tp = vvIn.makeTransferPair(vvOut);
transfers.add(tp);
}
- } else if (value != null && value.intValue() > 1) { // subsequent wildcards should do a copy of incoming valuevectors
+ } else if (value != null && value > 1) { // subsequent wildcards should do a copy of incoming valuevectors
int k = 0;
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final SchemaPath originalPath = SchemaPath.getSimplePath(vvIn.getField().getPath());
- if (k > result.outputNames.size()-1) {
+ final SchemaPath originalPath = SchemaPath.getSimplePath(vvIn.getField().getName());
+ if (k > result.outputNames.size() - 1) {
assert false;
}
final String name = result.outputNames.get(k++); // get the renamed column names
- if (name == EMPTY_STRING) {
+ if (name.isEmpty()) {
continue;
}
@@ -379,7 +378,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
continue;
}
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry() );
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry());
if (collector.hasErrors()) {
throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
}
@@ -387,7 +386,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final MaterializedField outputField = MaterializedField.create(name, expr.getMajorType());
final ValueVector vv = container.addOrGet(outputField, callBack);
allocationVectors.add(vv);
- final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
+ final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName()));
final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
}
@@ -406,14 +405,14 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
boolean isMatched = false;
for (int j = 0; j < result.outputNames.size(); j++) {
- if (!result.outputNames.get(j).equals(EMPTY_STRING)) {
+ if (!result.outputNames.get(j).isEmpty()) {
outputName = result.outputNames.get(j);
isMatched = true;
break;
}
}
- if(!isMatched) {
+ if (!isMatched) {
continue;
}
}
@@ -437,7 +436,8 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
Preconditions.checkNotNull(incoming);
final FieldReference ref = getRef(namedExpression);
- final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getAsUnescapedPath(), vectorRead.getMajorType()), callBack);
+ final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getLastSegment().getNameSegment().getPath(),
+ vectorRead.getMajorType()), callBack);
final TransferPair tp = vvIn.makeTransferPair(vvOut);
transfers.add(tp);
transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
@@ -463,7 +463,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
// need to do evaluation.
final ValueVector vector = container.addOrGet(outputField, callBack);
allocationVectors.add(vector);
- final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
+ final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName()));
final boolean useSetSafe = !(vector instanceof FixedWidthVector);
final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe);
final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
@@ -510,17 +510,19 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final List<NamedExpression> exprs = Lists.newArrayList();
for (final MaterializedField field : incoming.getSchema()) {
+ String fieldName = field.getName();
if (Types.isComplex(field.getType()) || Types.isRepeated(field.getType())) {
- final LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON", SchemaPath.getSimplePath(field.getPath()), ExpressionPosition.UNKNOWN);
+ final LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON",
+ SchemaPath.getSimplePath(fieldName), ExpressionPosition.UNKNOWN);
final String castFuncName = CastFunctions.getCastFunc(MinorType.VARCHAR);
final List<LogicalExpression> castArgs = Lists.newArrayList();
castArgs.add(convertToJson); //input_expr
// implicitly casting to varchar, since we don't know actual source length, cast to undefined length, which will preserve source length
castArgs.add(new ValueExpressions.LongExpression(Types.MAX_VARCHAR_LENGTH, null));
final FunctionCall castCall = new FunctionCall(castFuncName, castArgs, ExpressionPosition.UNKNOWN);
- exprs.add(new NamedExpression(castCall, new FieldReference(field.getPath())));
+ exprs.add(new NamedExpression(castCall, new FieldReference(fieldName)));
} else {
- exprs.add(new NamedExpression(SchemaPath.getSimplePath(field.getPath()), new FieldReference(field.getPath())));
+ exprs.add(new NamedExpression(SchemaPath.getSimplePath(fieldName), new FieldReference(fieldName)));
}
}
return exprs;
@@ -528,8 +530,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
private boolean isClassificationNeeded(final List<NamedExpression> exprs) {
boolean needed = false;
- for (int i = 0; i < exprs.size(); i++) {
- final NamedExpression ex = exprs.get(i);
+ for (NamedExpression ex : exprs) {
if (!(ex.getExpr() instanceof SchemaPath)) {
continue;
}
@@ -570,7 +571,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
* @param origName the original input name of the column
* @param result the data structure to keep track of the used names and decide what output name should be
* to ensure uniqueness
- * @Param allowDupsWithRename if the original name has been used, is renaming allowed to ensure output name unique
+ * @param allowDupsWithRename if the original name has been used, is renaming allowed to ensure output name unique
*/
private void addToResultMaps(final String origName, final ClassifierResult result, final boolean allowDupsWithRename) {
String name = origName;
@@ -610,7 +611,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
boolean exprIsFirstWildcard = false;
if (exprContainsStar) {
result.isStar = true;
- final Integer value = (Integer) result.prefixMap.get(exprPrefix);
+ final Integer value = result.prefixMap.get(exprPrefix);
if (value == null) {
final Integer n = 1;
result.prefixMap.put(exprPrefix, n);
@@ -634,9 +635,9 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
assert(components.length == 2);
final String prefix = components[0];
result.outputNames = Lists.newArrayList();
- for(final VectorWrapper<?> wrapper : incoming) {
+ for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String name = vvIn.getField().getPath();
+ final String name = vvIn.getField().getName();
// add the prefix to the incoming column name
final String newName = prefix + StarColumnHelper.PREFIX_DELIMITER + name;
@@ -656,7 +657,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String incomingName = vvIn.getField().getPath();
+ final String incomingName = vvIn.getField().getName();
// get the prefix of the name
final String[] nameComponents = incomingName.split(StarColumnHelper.PREFIX_DELIMITER, 2);
// if incoming valuevector does not have a prefix, ignore it since this expression is not referencing it
@@ -666,10 +667,9 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
}
final String namePrefix = nameComponents[0];
if (exprPrefix.equalsIgnoreCase(namePrefix)) {
- final String newName = incomingName;
- if (!result.outputMap.containsKey(newName)) {
- result.outputNames.set(k, newName);
- result.outputMap.put(newName, newName);
+ if (!result.outputMap.containsKey(incomingName)) {
+ result.outputNames.set(k, incomingName);
+ result.outputMap.put(incomingName, incomingName);
}
}
k++;
@@ -679,7 +679,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
if (exprContainsStar) {
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String incomingName = vvIn.getField().getPath();
+ final String incomingName = vvIn.getField().getName();
if (refContainsStar) {
addToResultMaps(incomingName, result, true); // allow dups since this is likely top-level project
} else {
@@ -698,11 +698,11 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
}
// input is wildcard and it is not the first wildcard
- else if(exprIsStar) {
+ else if (exprIsStar) {
result.outputNames = Lists.newArrayList();
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String incomingName = vvIn.getField().getPath();
+ final String incomingName = vvIn.getField().getName();
addToResultMaps(incomingName, result, true); // allow dups since this is likely top-level project
}
}
@@ -723,7 +723,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String name = vvIn.getField().getPath();
+ final String name = vvIn.getField().getName();
final String[] components = name.split(StarColumnHelper.PREFIX_DELIMITER, 2);
if (components.length <= 1) {
k++;
@@ -760,7 +760,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
result.outputNames = Lists.newArrayList();
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String incomingName = vvIn.getField().getPath();
+ final String incomingName = vvIn.getField().getName();
if (expr.getPath().equalsIgnoreCase(incomingName)) { // case insensitive matching of field name.
final String newName = ref.getPath();
addToResultMaps(newName, result, true);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
index e6a0dd4..5afe66b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
@@ -103,7 +103,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
try {
IterOutcome upstream = unionAllInput.nextBatch();
logger.debug("Upstream of Union-All: {}", upstream);
- switch(upstream) {
+ switch (upstream) {
case NONE:
case OUT_OF_MEMORY:
case STOP:
@@ -114,7 +114,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
case OK:
IterOutcome workOutcome = doWork();
- if(workOutcome != IterOutcome.OK) {
+ if (workOutcome != IterOutcome.OK) {
return workOutcome;
} else {
return upstream;
@@ -164,9 +164,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
transfers.clear();
// If both sides of Union-All are empty
- if(unionAllInput.isBothSideEmpty()) {
- for(int i = 0; i < outputFields.size(); ++i) {
- final String colName = outputFields.get(i).getPath();
+ if (unionAllInput.isBothSideEmpty()) {
+ for (MaterializedField materializedField : outputFields) {
+ final String colName = materializedField.getName();
final MajorType majorType = MajorType.newBuilder()
.setMinorType(MinorType.INT)
.setMode(DataMode.OPTIONAL)
@@ -186,12 +186,12 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// Uncomment out this line to debug the generated code.
// cg.getCodeGenerator().saveCodeForDebugging(true);
int index = 0;
- for(VectorWrapper<?> vw : current) {
+ for (VectorWrapper<?> vw : current) {
ValueVector vvIn = vw.getValueVector();
// get the original input column names
- SchemaPath inputPath = SchemaPath.getSimplePath(vvIn.getField().getPath());
+ SchemaPath inputPath = SchemaPath.getSimplePath(vvIn.getField().getName());
// get the renamed column names
- SchemaPath outputPath = SchemaPath.getSimplePath(outputFields.get(index).getPath());
+ SchemaPath outputPath = SchemaPath.getSimplePath(outputFields.get(index).getName());
final ErrorCollector collector = new ErrorCollectorImpl();
// According to input data names, Minortypes, Datamodes, choose to
@@ -202,19 +202,20 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// Transfer column
MajorType outputFieldType = outputFields.get(index).getType();
- MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), outputFieldType);
+ MaterializedField outputField = MaterializedField.create(outputPath.getLastSegment().getNameSegment().getPath(),
+ outputFieldType);
/*
todo: Fix if condition when DRILL-4824 is merged
If condition should be changed to:
- `if (outputFields.get(index).getPath().equals(inputPath.getAsUnescapedPath())) {`
+ `if (outputFields.get(index).getName().equals(inputPath.getRootSegmentPath())) {`
DRILL-5419 has changed condition to correct one but this caused regression (DRILL-5521).
Root cause is missing indication of child column in map types when it is null.
DRILL-4824 is re-working json reader implementation, including map types and will fix this problem.
Reverting condition to previous one to avoid regression till DRILL-4824 is merged.
Unit test - TestJsonReader.testKvgenWithUnionAll().
*/
- if (outputFields.get(index).getPath().equals(inputPath)) {
+ if (outputFields.get(index).getName().equals(inputPath)) {
ValueVector vvOut = container.addOrGet(outputField);
TransferPair tp = vvIn.makeTransferPair(vvOut);
transfers.add(tp);
@@ -227,7 +228,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
ValueVector vv = container.addOrGet(outputField, callBack);
allocationVectors.add(vv);
- TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
+ TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName()));
ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
cg.addExpr(write);
}
@@ -240,7 +241,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// If the inputs' DataMode is required and the outputs' DataMode is not required
// cast to the one with the least restriction
- if(vvIn.getField().getType().getMode() == DataMode.REQUIRED
+ if (vvIn.getField().getType().getMode() == DataMode.REQUIRED
&& outputFields.get(index).getType().getMode() != DataMode.REQUIRED) {
expr = ExpressionTreeMaterializer.convertToNullableType(expr, vvIn.getField().getType().getMinorType(), context.getFunctionRegistry(), collector);
if (collector.hasErrors()) {
@@ -250,17 +251,18 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// If two inputs' MinorTypes are different,
// Insert a cast before the Union operation
- if(vvIn.getField().getType().getMinorType() != outputFields.get(index).getType().getMinorType()) {
+ if (vvIn.getField().getType().getMinorType() != outputFields.get(index).getType().getMinorType()) {
expr = ExpressionTreeMaterializer.addCastExpression(expr, outputFields.get(index).getType(), context.getFunctionRegistry(), collector);
if (collector.hasErrors()) {
throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
}
}
- final MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), expr.getMajorType());
+ final MaterializedField outputField = MaterializedField.create(outputPath.getLastSegment().getNameSegment().getPath(),
+ expr.getMajorType());
ValueVector vector = container.addOrGet(outputField, callBack);
allocationVectors.add(vector);
- TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
+ TypedFieldId fid = container.getValueVectorId(outputPath);
boolean useSetSafe = !(vector instanceof FixedWidthVector);
ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe);
@@ -272,12 +274,12 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
unionall = context.getImplementationClass(cg.getCodeGenerator());
unionall.setup(context, current, this, transfers);
- if(!schemaAvailable) {
+ if (!schemaAvailable) {
container.buildSchema(BatchSchema.SelectionVectorMode.NONE);
schemaAvailable = true;
}
- if(!doAlloc()) {
+ if (!doAlloc()) {
return IterOutcome.OUT_OF_MEMORY;
}
@@ -298,7 +300,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// This method is used by inner class to clear the current record batch
private void clearCurrentRecordBatch() {
- for(VectorWrapper<?> v: current) {
+ for (VectorWrapper<?> v: current) {
v.clear();
}
}
@@ -333,9 +335,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
}
public IterOutcome nextBatch() throws SchemaChangeException {
- if(upstream == RecordBatch.IterOutcome.NOT_YET) {
+ if (upstream == RecordBatch.IterOutcome.NOT_YET) {
IterOutcome iterLeft = leftSide.nextBatch();
- switch(iterLeft) {
+ switch (iterLeft) {
case OK_NEW_SCHEMA:
/*
* If the first few record batches are all empty,
@@ -345,7 +347,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
* Thus, while-loop is necessary to skip those empty batches.
*/
whileLoop:
- while(leftSide.getRecordBatch().getRecordCount() == 0) {
+ while (leftSide.getRecordBatch().getRecordCount() == 0) {
iterLeft = leftSide.nextBatch();
switch(iterLeft) {
@@ -380,11 +382,11 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
}
IterOutcome iterRight = rightSide.nextBatch();
- switch(iterRight) {
+ switch (iterRight) {
case OK_NEW_SCHEMA:
// Unless there is no record batch on the left side of the inputs,
// always start processing from the left side.
- if(leftIsFinish) {
+ if (leftIsFinish) {
unionAllRecordBatch.setCurrentRecordBatch(rightSide.getRecordBatch());
} else {
unionAllRecordBatch.setCurrentRecordBatch(leftSide.getRecordBatch());
@@ -402,9 +404,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
* Thus, while-loop is necessary to skip those empty batches.
*/
whileLoop:
- while(rightSide.getRecordBatch().getRecordCount() == 0) {
+ while (rightSide.getRecordBatch().getRecordCount() == 0) {
iterRight = rightSide.nextBatch();
- switch(iterRight) {
+ switch (iterRight) {
case STOP:
case OUT_OF_MEMORY:
return iterRight;
@@ -425,7 +427,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
}
}
- if(leftIsFinish && rightIsFinish) {
+ if (leftIsFinish && rightIsFinish) {
setBothSideEmpty(true);
}
@@ -446,19 +448,19 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
upstream = IterOutcome.OK_NEW_SCHEMA;
return upstream;
} else {
- if(isBothSideEmpty()) {
+ if (isBothSideEmpty()) {
return IterOutcome.NONE;
}
unionAllRecordBatch.clearCurrentRecordBatch();
- if(leftIsFinish && rightIsFinish) {
+ if (leftIsFinish && rightIsFinish) {
upstream = IterOutcome.NONE;
return upstream;
- } else if(leftIsFinish) {
+ } else if (leftIsFinish) {
IterOutcome iterOutcome = rightSide.nextBatch();
- switch(iterOutcome) {
+ switch (iterOutcome) {
case NONE:
rightIsFinish = true;
// fall through
@@ -468,7 +470,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
return upstream;
case OK_NEW_SCHEMA:
- if(!rightSide.getRecordBatch().getSchema().equals(rightSchema)) {
+ if (!rightSide.getRecordBatch().getSchema().equals(rightSchema)) {
throw new SchemaChangeException("Schema change detected in the right input of Union-All. This is not currently supported");
}
iterOutcome = IterOutcome.OK;
@@ -481,9 +483,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
default:
throw new IllegalStateException(String.format("Unknown state %s.", upstream));
}
- } else if(rightIsFinish) {
+ } else if (rightIsFinish) {
IterOutcome iterOutcome = leftSide.nextBatch();
- switch(iterOutcome) {
+ switch (iterOutcome) {
case STOP:
case OUT_OF_MEMORY:
case NONE:
@@ -501,14 +503,14 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
} else {
IterOutcome iterOutcome = leftSide.nextBatch();
- switch(iterOutcome) {
+ switch (iterOutcome) {
case STOP:
case OUT_OF_MEMORY:
upstream = iterOutcome;
return upstream;
case OK_NEW_SCHEMA:
- if(!leftSide.getRecordBatch().getSchema().equals(leftSchema)) {
+ if (!leftSide.getRecordBatch().getSchema().equals(leftSchema)) {
throw new SchemaChangeException("Schema change detected in the left input of Union-All. This is not currently supported");
}
@@ -549,10 +551,10 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
* types are nullable integer
*/
private void inferOutputFields() {
- if(!leftIsFinish && !rightIsFinish) {
+ if (!leftIsFinish && !rightIsFinish) {
// Both sides are non-empty
inferOutputFieldsBothSide();
- } else if(!rightIsFinish) {
+ } else if (!rightIsFinish) {
// Left side is non-empty
// While use left side's column names as output column names,
// use right side's column types as output column types.
@@ -585,7 +587,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
if (hasSameTypeAndMode(leftField, rightField)) {
MajorType.Builder builder = MajorType.newBuilder().setMinorType(leftField.getType().getMinorType()).setMode(leftField.getDataMode());
builder = Types.calculateTypePrecisionAndScale(leftField.getType(), rightField.getType(), builder);
- outputFields.add(MaterializedField.create(leftField.getPath(), builder.build()));
+ outputFields.add(MaterializedField.create(leftField.getName(), builder.build()));
} else {
// If the output type is not the same,
// cast the column of one of the table to a data type which is the Least Restrictive
@@ -612,7 +614,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
dataModes.add(rightField.getType().getMode());
builder.setMode(TypeCastRules.getLeastRestrictiveDataMode(dataModes));
- outputFields.add(MaterializedField.create(leftField.getPath(), builder.build()));
+ outputFields.add(MaterializedField.create(leftField.getName(), builder.build()));
}
++index;
}
@@ -624,20 +626,19 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
outputFields = Lists.newArrayList();
final List<String> outputColumnNames = Lists.newArrayList();
- final Iterator<MaterializedField> iterForNames = schemaForNames.iterator();
- while(iterForNames.hasNext()) {
- outputColumnNames.add(iterForNames.next().getPath());
+ for (MaterializedField materializedField : schemaForNames) {
+ outputColumnNames.add(materializedField.getName());
}
final Iterator<MaterializedField> iterForTypes = schemaForTypes.iterator();
- for(int i = 0; iterForTypes.hasNext(); ++i) {
+ for (int i = 0; iterForTypes.hasNext(); ++i) {
MaterializedField field = iterForTypes.next();
outputFields.add(MaterializedField.create(outputColumnNames.get(i), field.getType()));
}
}
public List<MaterializedField> getOutputFields() {
- if(outputFields == null) {
+ if (outputFields == null) {
throw new NullPointerException("Output fields have not been inferred");
}
@@ -670,11 +671,11 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
}
public IterOutcome nextBatch() {
- if(upstream == IterOutcome.NONE) {
+ if (upstream == IterOutcome.NONE) {
throw new IllegalStateException(String.format("Unknown state %s.", upstream));
}
- if(upstream == IterOutcome.NOT_YET) {
+ if (upstream == IterOutcome.NOT_YET) {
upstream = unionAllRecordBatch.next(recordBatch);
return upstream;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index 39c662f..e0cfc7a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -678,7 +678,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
for (BatchGroup group : batchGroupList) {
vectors[i++] = group.getValueAccessorById(
field.getValueClass(),
- group.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds())
+ group.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds())
.getValueVector();
}
cont.add(vectors);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
index 6ec8862..2d82cc1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
@@ -310,7 +310,7 @@ public class PriorityQueueCopierWrapper extends BaseSortWrapper {
for (BatchGroup group : batchGroupList) {
vectors[i++] = group.getValueAccessorById(
field.getValueClass(),
- group.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds())
+ group.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds())
.getValueVector();
}
cont.add(vectors);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
index 9a3ef96..837ddd8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -160,7 +160,7 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule {
metaContext = ((FormatSelection)selection).getSelection().getMetaContext();
}
- RexNode condition = null;
+ RexNode condition;
if (projectRel == null) {
condition = filterRel.getCondition();
} else {
@@ -244,7 +244,7 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule {
for (int partitionColumnIndex : BitSets.toIter(partitionColumnBitSet)) {
SchemaPath column = SchemaPath.getSimplePath(fieldNameMap.get(partitionColumnIndex));
MajorType type = descriptor.getVectorType(column, settings);
- MaterializedField field = MaterializedField.create(column.getAsUnescapedPath(), type);
+ MaterializedField field = MaterializedField.create(column.getLastSegment().getNameSegment().getPath(), type);
ValueVector v = TypeHelper.getNewVector(field, allocator);
v.allocateNew();
vectors[partitionColumnIndex] = v;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
index 3801cb5..9a8483b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -88,7 +88,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp
final Map<String, ValueVector> oldFields = Maps.newHashMap();
for(final VectorWrapper<?> wrapper : container) {
final ValueVector vector = wrapper.getValueVector();
- oldFields.put(vector.getField().getPath(), vector);
+ oldFields.put(vector.getField().getName(), vector);
}
final VectorContainer newVectors = new VectorContainer();
@@ -97,7 +97,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp
int bufOffset = 0;
for(final SerializedField field : fields) {
final MaterializedField fieldDef = MaterializedField.create(field);
- ValueVector vector = oldFields.remove(fieldDef.getPath());
+ ValueVector vector = oldFields.remove(fieldDef.getName());
if (vector == null) {
// Field did not exist previously--is schema change.
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java
index 28f5bf2..e1a1031 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -53,7 +53,7 @@ public class SchemaUtil {
for (BatchSchema s : schemas) {
for (MaterializedField field : s) {
- SchemaPath path = SchemaPath.getSimplePath(field.getPath());
+ SchemaPath path = SchemaPath.getSimplePath(field.getName());
Set<MinorType> currentTypes = typeSetMap.get(path);
if (currentTypes == null) {
currentTypes = Sets.newHashSet();
@@ -64,9 +64,7 @@ public class SchemaUtil {
throw new RuntimeException("Schema change not currently supported for schemas with complex types");
}
if (newType == MinorType.UNION) {
- for (MinorType subType : field.getType().getSubTypeList()) {
- currentTypes.add(subType);
- }
+ currentTypes.addAll(field.getType().getSubTypeList());
} else {
currentTypes.add(newType);
}
@@ -82,10 +80,11 @@ public class SchemaUtil {
for (MinorType t : types) {
builder.addSubType(t);
}
- MaterializedField field = MaterializedField.create(path.getAsUnescapedPath(), builder.build());
+ MaterializedField field = MaterializedField.create(path.getLastSegment().getNameSegment().getPath(), builder.build());
fields.add(field);
} else {
- MaterializedField field = MaterializedField.create(path.getAsUnescapedPath(), Types.optional(types.iterator().next()));
+ MaterializedField field = MaterializedField.create(path.getLastSegment().getNameSegment().getPath(),
+ Types.optional(types.iterator().next()));
fields.add(field);
}
}
@@ -162,12 +161,12 @@ public class SchemaUtil {
if (w.isHyper()) {
isHyper = true;
final ValueVector[] vvs = w.getValueVectors();
- vectorMap.put(vvs[0].getField().getPath(), vvs);
+ vectorMap.put(vvs[0].getField().getName(), vvs);
} else {
assert !isHyper;
@SuppressWarnings("resource")
final ValueVector v = w.getValueVector();
- vectorMap.put(v.getField().getPath(), v);
+ vectorMap.put(v.getField().getName(), v);
}
}
@@ -175,7 +174,7 @@ public class SchemaUtil {
for (MaterializedField field : toSchema) {
if (isHyper) {
- final ValueVector[] vvs = (ValueVector[]) vectorMap.remove(field.getPath());
+ final ValueVector[] vvs = (ValueVector[]) vectorMap.remove(field.getName());
final ValueVector[] vvsOut;
if (vvs == null) {
vvsOut = new ValueVector[1];
@@ -189,7 +188,7 @@ public class SchemaUtil {
c.add(vvsOut);
} else {
@SuppressWarnings("resource")
- final ValueVector v = (ValueVector) vectorMap.remove(field.getPath());
+ final ValueVector v = (ValueVector) vectorMap.remove(field.getName());
c.add(coerceVector(v, c, field, recordCount, allocator));
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index 99353ea..63cac7d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -136,7 +136,7 @@ public class VectorContainer implements VectorAccessible {
@SuppressWarnings({ "resource", "unchecked" })
public <T extends ValueVector> T addOrGet(final MaterializedField field, final SchemaChangeCallBack callBack) {
- final TypedFieldId id = getValueVectorId(SchemaPath.getSimplePath(field.getPath()));
+ final TypedFieldId id = getValueVectorId(SchemaPath.getSimplePath(field.getName()));
final ValueVector vector;
final Class<?> clazz = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getType().getMode());
if (id != null) {
@@ -206,12 +206,12 @@ public class VectorContainer implements VectorAccessible {
*/
public static VectorContainer canonicalize(VectorContainer original) {
VectorContainer vc = new VectorContainer();
- List<VectorWrapper<?>> canonicalWrappers = new ArrayList<VectorWrapper<?>>(original.wrappers);
+ List<VectorWrapper<?>> canonicalWrappers = new ArrayList<>(original.wrappers);
// Sort list of VectorWrapper alphabetically based on SchemaPath.
Collections.sort(canonicalWrappers, new Comparator<VectorWrapper<?>>() {
@Override
public int compare(VectorWrapper<?> v1, VectorWrapper<?> v2) {
- return v1.getField().getPath().compareTo(v2.getField().getPath());
+ return v1.getField().getName().compareTo(v2.getField().getName());
}
});
@@ -255,9 +255,7 @@ public class VectorContainer implements VectorAccessible {
schema = null;
Class<?> clazz = hyperVector[0].getClass();
ValueVector[] c = (ValueVector[]) Array.newInstance(clazz, hyperVector.length);
- for (int i = 0; i < hyperVector.length; i++) {
- c[i] = hyperVector[i];
- }
+ System.arraycopy(hyperVector, 0, c, 0, hyperVector.length);
// todo: work with a merged schema.
wrappers.add(HyperVectorWrapper.create(hyperVector[0].getField(), c, releasable));
}
@@ -283,7 +281,7 @@ public class VectorContainer implements VectorAccessible {
for (VectorWrapper<?> w : wrappers){
if (!w.isHyper() && old == w.getValueVector()) {
w.clear();
- wrappers.set(i, new SimpleVectorWrapper<ValueVector>(newVector));
+ wrappers.set(i, new SimpleVectorWrapper<>(newVector));
return;
}
i++;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
index 62c6efd..bcce9eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
@@ -99,7 +99,7 @@ public class WebUserConnection extends AbstractDisposableUserClientConnection im
// TODO: Clean: DRILL-2933: That load(...) no longer throws
// SchemaChangeException, so check/clean catch clause below.
for (int i = 0; i < loader.getSchema().getFieldCount(); ++i) {
- columns.add(loader.getSchema().getColumn(i).getPath());
+ columns.add(loader.getSchema().getColumn(i).getName());
}
for (int i = 0; i < rows; ++i) {
final Map<String, String> record = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java
index ccd622b..fa8121e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java
@@ -95,7 +95,7 @@ public class ColumnExplorer {
*/
public static boolean isPartitionColumn(OptionManager optionManager, SchemaPath column){
String partitionDesignator = optionManager.getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL).string_val;
- String path = column.getAsUnescapedPath();
+ String path = column.getRootSegmentPath();
return isPartitionColumn(partitionDesignator, path);
}
@@ -187,7 +187,7 @@ public class ColumnExplorer {
selectedImplicitColumns.putAll(allImplicitColumns);
} else {
for (SchemaPath column : columns) {
- String path = column.getAsUnescapedPath();
+ String path = column.getRootSegmentPath();
if (isPartitionColumn(partitionDesignator, path)) {
selectedPartitionColumns.add(Integer.parseInt(path.substring(partitionDesignator.length())));
} else if (allImplicitColumns.get(path) != null) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
index 22fb48c..6262dce 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -138,7 +138,7 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void,
public ExprNode visitCastExpression(CastExpression e, Void value) throws RuntimeException {
if (e.getInput() instanceof FieldReference) {
FieldReference fieldRef = (FieldReference) e.getInput();
- String field = fieldRef.getAsUnescapedPath().toUpperCase();
+ String field = fieldRef.getRootSegmentPath().toUpperCase();
if (field.equals(CATS_COL_CATALOG_NAME)
|| field.equals(SCHS_COL_SCHEMA_NAME)
|| field.equals(SHRD_COL_TABLE_NAME)
@@ -158,7 +158,7 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void,
@Override
public ExprNode visitSchemaPath(SchemaPath path, Void value) throws RuntimeException {
- String field = path.getAsUnescapedPath().toUpperCase();
+ String field = path.getRootSegmentPath().toUpperCase();
if (field.equals(CATS_COL_CATALOG_NAME)
|| field.equals(SCHS_COL_SCHEMA_NAME)
|| field.equals(SHRD_COL_TABLE_NAME)
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
index a33f46a..4a8c5f3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
@@ -80,10 +80,12 @@ import com.google.common.collect.Maps;
import javax.annotation.Nullable;
+import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.SUPPORTED_VERSIONS;
import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V1;
import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V2;
import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3;
import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3_1;
+import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3_2;
public class Metadata {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Metadata.class);
@@ -234,7 +236,8 @@ public class Metadata {
childFiles.add(file);
}
}
- ParquetTableMetadata_v3 parquetTableMetadata = new ParquetTableMetadata_v3(V3_1, DrillVersionInfo.getVersion());
+ ParquetTableMetadata_v3 parquetTableMetadata = new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(),
+ DrillVersionInfo.getVersion());
if (childFiles.size() > 0) {
List<ParquetFileMetadata_v3 > childFilesMetadata =
getParquetFileMetadata_v3(parquetTableMetadata, childFiles);
@@ -308,7 +311,8 @@ public class Metadata {
*/
private ParquetTableMetadata_v3 getParquetTableMetadata(List<FileStatus> fileStatuses)
throws IOException {
- ParquetTableMetadata_v3 tableMetadata = new ParquetTableMetadata_v3(V3_1, DrillVersionInfo.getVersion());
+ ParquetTableMetadata_v3 tableMetadata = new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(),
+ DrillVersionInfo.getVersion());
List<ParquetFileMetadata_v3> fileMetadataList = getParquetFileMetadata_v3(tableMetadata, fileStatuses);
tableMetadata.files = fileMetadataList;
tableMetadata.directories = new ArrayList<String>();
@@ -695,7 +699,8 @@ public class Metadata {
@JsonSubTypes.Type(value = ParquetTableMetadata_v1.class, name = V1),
@JsonSubTypes.Type(value = ParquetTableMetadata_v2.class, name = V2),
@JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3),
- @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_1)
+ @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_1),
+ @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_2)
})
public static abstract class ParquetTableMetadataBase {
@@ -1421,7 +1426,7 @@ public class Metadata {
}
- @JsonTypeName(V3_1)
+ @JsonTypeName(V3_2)
public static class ParquetTableMetadata_v3 extends ParquetTableMetadataBase {
@JsonProperty(value = "metadata_version", access = JsonProperty.Access.WRITE_ONLY) private String metadataVersion;
/*
@@ -1654,16 +1659,20 @@ public class Metadata {
}
private static class Key {
- private String[] name;
+ private SchemaPath name;
private int hashCode = 0;
public Key(String[] name) {
- this.name = name;
+ this.name = SchemaPath.getCompoundPath(name);
+ }
+
+ public Key(SchemaPath name) {
+ this.name = new SchemaPath(name);
}
@Override public int hashCode() {
if (hashCode == 0) {
- hashCode = Arrays.hashCode(name);
+ hashCode = name.hashCode();
}
return hashCode;
}
@@ -1676,20 +1685,11 @@ public class Metadata {
return false;
}
final Key other = (Key) obj;
- return Arrays.equals(this.name, other.name);
+ return this.name.equals(other.name);
}
@Override public String toString() {
- String s = null;
- for (String namePart : name) {
- if (s != null) {
- s += ".";
- s += namePart;
- } else {
- s = namePart;
- }
- }
- return s;
+ return name.toString();
}
public static class DeSerializer extends KeyDeserializer {
@@ -1701,6 +1701,10 @@ public class Metadata {
@Override
public Object deserializeKey(String key, com.fasterxml.jackson.databind.DeserializationContext ctxt)
throws IOException, com.fasterxml.jackson.core.JsonProcessingException {
+ // key string should contain '`' char if the field was serialized as SchemaPath object
+ if (key.contains("`")) {
+ return new Key(SchemaPath.parseFromString(key));
+ }
return new Key(key.split("\\."));
}
}
@@ -1884,8 +1888,8 @@ public class Metadata {
filesWithRelativePaths.add(new ParquetFileMetadata_v3(
relativize(baseDir, file.getPath()), file.length, file.rowGroups));
}
- return new ParquetTableMetadata_v3(V3_1, tableMetadataWithAbsolutePaths, filesWithRelativePaths,
- directoriesWithRelativePaths, DrillVersionInfo.getVersion());
+ return new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(), tableMetadataWithAbsolutePaths,
+ filesWithRelativePaths, directoriesWithRelativePaths, DrillVersionInfo.getVersion());
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java
index bc6fd70..5ceadcd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java
@@ -134,6 +134,13 @@ public class MetadataVersion implements Comparable<MetadataVersion> {
public static final String V3_1 = "3.1";
/**
+ * Version 3.2: An array with the components of the field name in
+ * {@link Metadata.ColumnTypeMetadata_v3.Key} class is replaced by the SchemaPath.<br>
+ * See DRILL-4264
+ */
+ public static final String V3_2 = "3.2";
+
+ /**
* All historical versions of the Drill metadata cache files. In case of introducing a new parquet metadata version
* please follow the {@link MetadataVersion#FORMAT}.
*/
@@ -141,7 +148,8 @@ public class MetadataVersion implements Comparable<MetadataVersion> {
new MetadataVersion(V1),
new MetadataVersion(V2),
new MetadataVersion(V3),
- new MetadataVersion(V3_1)
+ new MetadataVersion(V3_1),
+ new MetadataVersion(V3_2)
);
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index 1d4d161..9b84e81 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -199,7 +199,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
private void newSchema() throws IOException {
List<Type> types = Lists.newArrayList();
for (MaterializedField field : batchSchema) {
- if (field.getPath().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) {
+ if (field.getName().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) {
continue;
}
types.add(getType(field));
@@ -230,7 +230,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
private PrimitiveType getPrimitiveType(MaterializedField field) {
MinorType minorType = field.getType().getMinorType();
- String name = field.getLastName();
+ String name = field.getName();
PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
Repetition repetition = ParquetTypeHelper.getRepetitionForDataMode(field.getDataMode());
OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
@@ -248,7 +248,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
for (MaterializedField childField : field.getChildren()) {
types.add(getType(childField));
}
- return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getLastName(), types);
+ return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getName(), types);
case LIST:
throw new UnsupportedOperationException("Unsupported type " + minorType);
default:
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
index bbdf246..d49a416 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
@@ -59,12 +59,12 @@ public class ParquetColumnMetadata {
se = schemaElements.get(column.getPath()[0]);
type = ParquetToDrillTypeConverter.toMajorType(column.getType(), se.getType_length(),
getDataMode(column), se, options);
- field = MaterializedField.create(toFieldName(column.getPath()), type);
+ field = MaterializedField.create(toFieldName(column.getPath()).getLastSegment().getNameSegment().getPath(), type);
length = getDataTypeLength();
}
- private String toFieldName(String[] paths) {
- return SchemaPath.getCompoundPath(paths).getAsUnescapedPath();
+ private SchemaPath toFieldName(String[] paths) {
+ return SchemaPath.getCompoundPath(paths);
}
private TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
index ab4b1b8..9814b53 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
@@ -108,7 +108,6 @@ public class ParquetSchema {
* Build the schema for this read as a combination of the schema specified in
* the Parquet footer and the list of columns selected in the query.
*
- * @param footer Parquet metadata
* @param batchSize target size of the batch, in rows
* @throws Exception if anything goes wrong
*/
@@ -188,7 +187,7 @@ public class ParquetSchema {
/**
* Determine if a Parquet field is selected for the query. It is selected
* either if this is a star query (we want all columns), or the column
- * appers in the select list.
+ * appears in the select list.
*
* @param field the Parquet column expressed as as Drill field.
* @return true if the column is to be included in the scan, false
@@ -205,7 +204,7 @@ public class ParquetSchema {
int i = 0;
for (SchemaPath expr : selectedCols) {
- if ( field.getPath().equalsIgnoreCase(expr.getAsUnescapedPath())) {
+ if (field.getName().equalsIgnoreCase(expr.getRootSegmentPath())) {
columnsFound[i] = true;
return true;
}
@@ -243,8 +242,9 @@ public class ParquetSchema {
*/
private NullableIntVector createMissingColumn(SchemaPath col, OutputMutator output) throws SchemaChangeException {
- MaterializedField field = MaterializedField.create(col.getAsUnescapedPath(),
- Types.optional(TypeProtos.MinorType.INT));
+ // col.toExpr() is used here as field name since we don't want to see these fields in the existing maps
+ MaterializedField field = MaterializedField.create(col.toExpr(),
+ Types.optional(TypeProtos.MinorType.INT));
return (NullableIntVector) output.addField(field,
TypeHelper.getValueVectorClass(TypeProtos.MinorType.INT, DataMode.OPTIONAL));
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 68d3bbb..9e66f6d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -208,27 +208,28 @@ public class DrillParquetReader extends AbstractRecordReader {
try {
this.operatorContext = context;
schema = footer.getFileMetaData().getSchema();
- MessageType projection = null;
+ MessageType projection;
if (isStarQuery()) {
projection = schema;
} else {
- columnsNotFound=new ArrayList<SchemaPath>();
+ columnsNotFound = new ArrayList<>();
projection = getProjection(schema, getColumns(), columnsNotFound);
- if(projection == null){
+ if (projection == null) {
projection = schema;
}
- if(columnsNotFound!=null && columnsNotFound.size()>0) {
+ if (columnsNotFound != null && columnsNotFound.size() > 0) {
nullFilledVectors = new ArrayList<>();
- for(SchemaPath col: columnsNotFound){
+ for (SchemaPath col: columnsNotFound) {
+ // col.toExpr() is used here as field name since we don't want to see these fields in the existing maps
nullFilledVectors.add(
- (NullableIntVector)output.addField(MaterializedField.create(col.getAsUnescapedPath(),
+ (NullableIntVector) output.addField(MaterializedField.create(col.toExpr(),
org.apache.drill.common.types.Types.optional(TypeProtos.MinorType.INT)),
(Class<? extends ValueVector>) TypeHelper.getValueVectorClass(TypeProtos.MinorType.INT,
TypeProtos.DataMode.OPTIONAL)));
}
- if(columnsNotFound.size()==getColumns().size()){
- noColumnsFound=true;
+ if (columnsNotFound.size() == getColumns().size()) {
+ noColumnsFound = true;
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
index cb5183e..7e32a4d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -34,21 +34,18 @@ import com.google.common.collect.Lists;
public class BatchPrinter {
public static void printHyperBatch(VectorAccessible batch, SelectionVector4 sv4) {
List<String> columns = Lists.newArrayList();
- List<ValueVector> vectors = Lists.newArrayList();
- int numBatches = 0;
for (VectorWrapper vw : batch) {
- columns.add(vw.getValueVectors()[0].getField().getPath());
- numBatches = vw.getValueVectors().length;
+ columns.add(vw.getValueVectors()[0].getField().getName());
}
int width = columns.size();
for (int j = 0; j < sv4.getCount(); j++) {
- if (j%50 == 0) {
+ if (j % 50 == 0) {
System.out.println(StringUtils.repeat("-", width * 17 + 1));
for (String column : columns) {
System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
}
System.out.printf("|\n");
- System.out.println(StringUtils.repeat("-", width*17 + 1));
+ System.out.println(StringUtils.repeat("-", width * 17 + 1));
}
for (VectorWrapper vw : batch) {
Object o = vw.getValueVectors()[sv4.get(j) >>> 16].getAccessor().getObject(sv4.get(j) & 65535);
@@ -60,7 +57,7 @@ public class BatchPrinter {
} else {
value = o.toString();
}
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
+ System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14));
}
System.out.printf("|\n");
}
@@ -71,32 +68,31 @@ public class BatchPrinter {
List<String> columns = Lists.newArrayList();
List<ValueVector> vectors = Lists.newArrayList();
for (VectorWrapper vw : batch) {
- columns.add(vw.getValueVector().getField().getPath());
+ columns.add(vw.getValueVector().getField().getName());
vectors.add(vw.getValueVector());
}
int width = columns.size();
int rows = vectors.get(0).getMetadata().getValueCount();
for (int row = 0; row < rows; row++) {
- if (row%50 == 0) {
+ if (row % 50 == 0) {
System.out.println(StringUtils.repeat("-", width * 17 + 1));
for (String column : columns) {
System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
}
System.out.printf("|\n");
- System.out.println(StringUtils.repeat("-", width*17 + 1));
+ System.out.println(StringUtils.repeat("-", width * 17 + 1));
}
for (ValueVector vv : vectors) {
Object o = vv.getAccessor().getObject(row);
String value;
if (o == null) {
value = "null";
- } else
- if (o instanceof byte[]) {
+ } else if (o instanceof byte[]) {
value = new String((byte[]) o);
} else {
value = o.toString();
}
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
+ System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14));
}
System.out.printf("|\n");
}
@@ -106,19 +102,19 @@ public class BatchPrinter {
List<String> columns = Lists.newArrayList();
List<ValueVector> vectors = Lists.newArrayList();
for (VectorWrapper vw : batch) {
- columns.add(vw.getValueVector().getField().getPath());
+ columns.add(vw.getValueVector().getField().getName());
vectors.add(vw.getValueVector());
}
int width = columns.size();
int rows = vectors.get(0).getMetadata().getValueCount();
for (int i = 0; i < rows; i++) {
- if (i%50 == 0) {
+ if (i % 50 == 0) {
System.out.println(StringUtils.repeat("-", width * 17 + 1));
for (String column : columns) {
System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
}
System.out.printf("|\n");
- System.out.println(StringUtils.repeat("-", width*17 + 1));
+ System.out.println(StringUtils.repeat("-", width * 17 + 1));
}
int row = sv2.getIndex(i);
for (ValueVector vv : vectors) {
@@ -126,13 +122,12 @@ public class BatchPrinter {
String value;
if (o == null) {
value = "null";
- } else
- if (o instanceof byte[]) {
+ } else if (o instanceof byte[]) {
value = new String((byte[]) o);
} else {
value = o.toString();
}
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
+ System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14));
}
System.out.printf("|\n");
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index b150179..d836bfc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -40,12 +40,12 @@ public class VectorUtil {
System.out.println(rows + " row(s):");
List<String> columns = Lists.newArrayList();
for (VectorWrapper<?> vw : va) {
- columns.add(vw.getValueVector().getField().getPath());
+ columns.add(vw.getValueVector().getField().getName());
}
int width = columns.size();
for (String column : columns) {
- System.out.printf("%s%s",column, column == columns.get(width - 1) ? "\n" : delimiter);
+ System.out.printf("%s%s",column, column.equals(columns.get(width - 1)) ? "\n" : delimiter);
}
for (int row = 0; row < rows; row++) {
int columnCounter = 0;
@@ -54,8 +54,8 @@ public class VectorUtil {
Object o ;
try{
o = vw.getValueVector().getAccessor().getObject(row);
- }catch(Exception e){
- throw new RuntimeException("failure while trying to read column " + vw.getField().getPath());
+ } catch (Exception e) {
+ throw new RuntimeException("failure while trying to read column " + vw.getField().getName());
}
if (o == null) {
//null value
@@ -83,7 +83,7 @@ public class VectorUtil {
if (includeHeader) {
List<String> columns = Lists.newArrayList();
for (VectorWrapper<?> vw : va) {
- columns.add(vw.getValueVector().getField().getPath());
+ columns.add(vw.getValueVector().getField().getName());
}
formattedResults.append(Joiner.on(delimiter).join(columns));
@@ -134,7 +134,7 @@ public class VectorUtil {
width += columnWidth + 2;
formats.add("| %-" + columnWidth + "s");
MaterializedField field = vw.getValueVector().getField();
- columns.add(field.getPath() + "<" + field.getType().getMinorType() + "(" + field.getType().getMode() + ")" + ">");
+ columns.add(field.getName() + "<" + field.getType().getMinorType() + "(" + field.getType().getMode() + ")" + ">");
columnIndex++;
}