You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2017/09/05 04:38:55 UTC
[1/4] drill git commit: DRILL-4264: Allow field names to include dots
Repository: drill
Updated Branches:
refs/heads/master e57514aad -> d105950a7
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t2.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t2.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t2.requires_replace.txt
new file mode 100644
index 0000000..b611d13
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t2.requires_replace.txt
@@ -0,0 +1,76 @@
+{
+ "metadata_version" : "3.1",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ ],
+ "drillVersion" : "1.12.0-SNAPSHOT"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_directories_with_absolute_paths.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_directories_with_absolute_paths.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_directories_with_absolute_paths.requires_replace.txt
deleted file mode 100644
index 8a9989d..0000000
--- a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_directories_with_absolute_paths.requires_replace.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-{
- "directories" : [ "file:REPLACED_IN_TEST/absolute_paths_metadata/t1", "file:REPLACED_IN_TEST/absolute_paths_metadata/t2" ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths.requires_replace.txt
deleted file mode 100644
index e3734f3..0000000
--- a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths.requires_replace.txt
+++ /dev/null
@@ -1,108 +0,0 @@
-{
- "metadata_version" : "v3",
- "columnTypeInfo" : {
- "n_name" : {
- "name" : [ "n_name" ],
- "primitiveType" : "BINARY",
- "originalType" : "UTF8",
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_nationkey" : {
- "name" : [ "n_nationkey" ],
- "primitiveType" : "INT32",
- "originalType" : null,
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_regionkey" : {
- "name" : [ "n_regionkey" ],
- "primitiveType" : "INT32",
- "originalType" : null,
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_comment" : {
- "name" : [ "n_comment" ],
- "primitiveType" : "BINARY",
- "originalType" : "UTF8",
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- }
- },
- "files" : [ {
- "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t1/0_0_0.parquet",
- "length" : 2424,
- "rowGroups" : [ {
- "start" : 4,
- "length" : 1802,
- "rowCount" : 25,
- "hostAffinity" : {
- "localhost" : 1.0
- },
- "columns" : [ {
- "name" : [ "n_nationkey" ],
- "minValue" : 0,
- "maxValue" : 24,
- "nulls" : 0
- }, {
- "name" : [ "n_name" ],
- "minValue" : "ALGERIA",
- "maxValue" : "VIETNAM",
- "nulls" : 0
- }, {
- "name" : [ "n_regionkey" ],
- "minValue" : 0,
- "maxValue" : 4,
- "nulls" : 0
- }, {
- "name" : [ "n_comment" ],
- "minValue" : " haggle. carefully final deposits detect slyly agai",
- "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
- "nulls" : 0
- } ]
- } ]
- }, {
- "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t2/0_0_0.parquet",
- "length" : 2424,
- "rowGroups" : [ {
- "start" : 4,
- "length" : 1802,
- "rowCount" : 25,
- "hostAffinity" : {
- "localhost" : 1.0
- },
- "columns" : [ {
- "name" : [ "n_nationkey" ],
- "minValue" : 0,
- "maxValue" : 24,
- "nulls" : 0
- }, {
- "name" : [ "n_name" ],
- "minValue" : "ALGERIA",
- "maxValue" : "VIETNAM",
- "nulls" : 0
- }, {
- "name" : [ "n_regionkey" ],
- "minValue" : 0,
- "maxValue" : 4,
- "nulls" : 0
- }, {
- "name" : [ "n_comment" ],
- "minValue" : " haggle. carefully final deposits detect slyly agai",
- "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
- "nulls" : 0
- } ]
- } ]
- } ],
- "directories" : [ "file:REPLACED_IN_TEST/absolute_paths_metadata/t1", "file:REPLACED_IN_TEST/absolute_paths_metadata/t2" ],
- "drillVersion" : "1.11.0"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t1.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t1.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t1.requires_replace.txt
deleted file mode 100644
index 62a8c80..0000000
--- a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t1.requires_replace.txt
+++ /dev/null
@@ -1,76 +0,0 @@
-{
- "metadata_version" : "v3",
- "columnTypeInfo" : {
- "n_name" : {
- "name" : [ "n_name" ],
- "primitiveType" : "BINARY",
- "originalType" : "UTF8",
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_nationkey" : {
- "name" : [ "n_nationkey" ],
- "primitiveType" : "INT32",
- "originalType" : null,
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_regionkey" : {
- "name" : [ "n_regionkey" ],
- "primitiveType" : "INT32",
- "originalType" : null,
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_comment" : {
- "name" : [ "n_comment" ],
- "primitiveType" : "BINARY",
- "originalType" : "UTF8",
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- }
- },
- "files" : [ {
- "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t1/0_0_0.parquet",
- "length" : 2424,
- "rowGroups" : [ {
- "start" : 4,
- "length" : 1802,
- "rowCount" : 25,
- "hostAffinity" : {
- "localhost" : 1.0
- },
- "columns" : [ {
- "name" : [ "n_nationkey" ],
- "minValue" : 0,
- "maxValue" : 24,
- "nulls" : 0
- }, {
- "name" : [ "n_name" ],
- "minValue" : "ALGERIA",
- "maxValue" : "VIETNAM",
- "nulls" : 0
- }, {
- "name" : [ "n_regionkey" ],
- "minValue" : 0,
- "maxValue" : 4,
- "nulls" : 0
- }, {
- "name" : [ "n_comment" ],
- "minValue" : " haggle. carefully final deposits detect slyly agai",
- "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
- "nulls" : 0
- } ]
- } ]
- } ],
- "directories" : [ ],
- "drillVersion" : "1.11.0"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t2.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t2.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t2.requires_replace.txt
deleted file mode 100644
index b70c8fa..0000000
--- a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t2.requires_replace.txt
+++ /dev/null
@@ -1,76 +0,0 @@
-{
- "metadata_version" : "v3",
- "columnTypeInfo" : {
- "n_name" : {
- "name" : [ "n_name" ],
- "primitiveType" : "BINARY",
- "originalType" : "UTF8",
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_nationkey" : {
- "name" : [ "n_nationkey" ],
- "primitiveType" : "INT32",
- "originalType" : null,
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_regionkey" : {
- "name" : [ "n_regionkey" ],
- "primitiveType" : "INT32",
- "originalType" : null,
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- },
- "n_comment" : {
- "name" : [ "n_comment" ],
- "primitiveType" : "BINARY",
- "originalType" : "UTF8",
- "precision" : 0,
- "scale" : 0,
- "repetitionLevel" : 0,
- "definitionLevel" : 0
- }
- },
- "files" : [ {
- "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t2/0_0_0.parquet",
- "length" : 2424,
- "rowGroups" : [ {
- "start" : 4,
- "length" : 1802,
- "rowCount" : 25,
- "hostAffinity" : {
- "localhost" : 1.0
- },
- "columns" : [ {
- "name" : [ "n_nationkey" ],
- "minValue" : 0,
- "maxValue" : 24,
- "nulls" : 0
- }, {
- "name" : [ "n_name" ],
- "minValue" : "ALGERIA",
- "maxValue" : "VIETNAM",
- "nulls" : 0
- }, {
- "name" : [ "n_regionkey" ],
- "minValue" : 0,
- "maxValue" : 4,
- "nulls" : 0
- }, {
- "name" : [ "n_comment" ],
- "minValue" : " haggle. carefully final deposits detect slyly agai",
- "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
- "nulls" : 0
- } ]
- } ]
- } ],
- "directories" : [ ],
- "drillVersion" : "1.11.0"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json b/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
index 290d896..095ef11 100644
--- a/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
+++ b/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
@@ -1,16 +1,21 @@
{
- "notexists" : null,
- "id" : 4
+ "notexists1" : null,
+ "id" : 4,
+ "notexists2" : null
} {
- "notexists" : null,
- "id" : 6
+ "notexists1" : null,
+ "id" : 6,
+ "notexists2" : null
} {
- "notexists" : null,
- "id" : 17
+ "notexists1" : null,
+ "id" : 17,
+ "notexists2" : null
} {
- "notexists" : null,
- "id" : 17
+ "notexists1" : null,
+ "id" : 17,
+ "notexists2" : null
} {
- "notexists" : null,
- "id" : 8
+ "notexists1" : null,
+ "id" : 8,
+ "notexists2" : null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillColumnMetaDataList.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillColumnMetaDataList.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillColumnMetaDataList.java
index 79007bb..90dcafe 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillColumnMetaDataList.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillColumnMetaDataList.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -118,9 +118,9 @@ public class DrillColumnMetaDataList extends BasicList<ColumnMetaData>{
new ArrayList<>(schema.getFieldCount());
for (int colOffset = 0; colOffset < schema.getFieldCount(); colOffset++) {
final MaterializedField field = schema.getColumn(colOffset);
- Class<?> objectClass = getObjectClasses.get( colOffset );
+ Class<?> objectClass = getObjectClasses.get(colOffset);
- final String columnName = field.getPath();
+ final String columnName = field.getName();
final MajorType rpcDataType = field.getType();
final AvaticaType bundledSqlDataType = getAvaticaType(rpcDataType);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillColumnMetaDataListTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillColumnMetaDataListTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillColumnMetaDataListTest.java
index 9bd8502..be21d3b 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillColumnMetaDataListTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillColumnMetaDataListTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -68,13 +68,13 @@ public class DrillColumnMetaDataListTest {
// Create mock columns
final MaterializedField exampleIntField = mock(MaterializedField.class);
MajorType exampleIntType = MajorType.newBuilder().setMinorType(MinorType.INT).build();
- when(exampleIntField.getPath()).thenReturn("/path/to/testInt");
+ when(exampleIntField.getName()).thenReturn("/path/to/testInt");
when(exampleIntField.getType()).thenReturn(exampleIntType);
when(exampleIntField.getDataMode()).thenReturn(DataMode.OPTIONAL);
final MaterializedField exampleStringField = mock(MaterializedField.class);
MajorType exampleStringType = MajorType.newBuilder().setMinorType(MinorType.VARCHAR).build();
- when(exampleStringField.getPath()).thenReturn("/path/to/testString");
+ when(exampleStringField.getName()).thenReturn("/path/to/testString");
when(exampleStringField.getType()).thenReturn(exampleStringType);
when(exampleStringField.getDataMode()).thenReturn(DataMode.REQUIRED);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index e8cdcf8..bbda672 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -220,7 +220,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
@Override
public void load(SerializedField metadata, DrillBuf buffer) {
- Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
+ Preconditions.checkArgument(this.field.getName().equals(metadata.getNamePart().getName()),
+ "The field %s doesn't match the provided metadata %s.", this.field, metadata);
final int actualLength = metadata.getBufferLength();
final int valueCount = metadata.getValueCount();
final int expectedLength = valueCount * VALUE_WIDTH;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index bc1ec3a..1ecedc6 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -47,7 +47,7 @@ public class MaterializedField {
this.children = children;
}
- public static MaterializedField create(SerializedField serField){
+ public static MaterializedField create(SerializedField serField) {
LinkedHashSet<MaterializedField> children = new LinkedHashSet<>();
for (SerializedField sf : serField.getChildList()) {
children.add(MaterializedField.create(sf));
@@ -66,7 +66,7 @@ public class MaterializedField {
return serializedFieldBuilder.build();
}
- public SerializedField.Builder getAsBuilder(){
+ public SerializedField.Builder getAsBuilder() {
return SerializedField.newBuilder()
.setMajorType(type)
.setNamePart(NamePart.newBuilder().setName(name).build());
@@ -82,7 +82,7 @@ public class MaterializedField {
return newField;
}
- public void addChild(MaterializedField field){
+ public void addChild(MaterializedField field) {
children.add(field);
}
@@ -106,47 +106,16 @@ public class MaterializedField {
return new MaterializedField(name, type, newChildren);
}
-// public String getLastName(){
-// PathSegment seg = key.path.getRootSegment();
-// while (seg.getChild() != null) {
-// seg = seg.getChild();
-// }
-// return seg.getNameSegment().getPath();
-// }
-
// TODO: rewrite without as direct match rather than conversion then match.
- public boolean matches(SerializedField field){
+ public boolean matches(SerializedField field) {
MaterializedField f = create(field);
return f.equals(this);
}
- public static MaterializedField create(String name, MajorType type){
+ public static MaterializedField create(String name, MajorType type) {
return new MaterializedField(name, type, new LinkedHashSet<MaterializedField>());
}
-// public String getName(){
-// StringBuilder sb = new StringBuilder();
-// boolean first = true;
-// for(NamePart np : def.getNameList()){
-// if(np.getType() == Type.ARRAY){
-// sb.append("[]");
-// }else{
-// if(first){
-// first = false;
-// }else{
-// sb.append(".");
-// }
-// sb.append('`');
-// sb.append(np.getName());
-// sb.append('`');
-//
-// }
-// }
-// return sb.toString();
-// }
-
- public String getPath() { return getName(); }
- public String getLastName() { return getName(); }
public String getName() { return name; }
public int getWidth() { return type.getWidth(); }
public MajorType getType() { return type; }
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index 2179829..2b6fde0 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -71,13 +71,13 @@ public abstract class BaseValueVector implements ValueVector {
return field;
}
- public MaterializedField getField(String ref){
+ public MaterializedField getField(String ref) {
return getField().withPath(ref);
}
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return getTransferPair(getField().getPath(), allocator);
+ return getTransferPair(getField().getName(), allocator);
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 4d7098b..f879fc4 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -221,7 +221,8 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
@Override
public void load(SerializedField metadata, DrillBuf buffer) {
- Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
+ Preconditions.checkArgument(this.field.getName().equals(metadata.getNamePart().getName()),
+ "The field %s doesn't match the provided metadata %s.", this.field, metadata);
final int valueCount = metadata.getValueCount();
final int expectedLength = getSizeFromCount(valueCount);
final int actualLength = metadata.getBufferLength();
@@ -280,7 +281,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
if (target.data != null) {
target.data.release();
}
- target.data = (DrillBuf) data.slice(firstByte, byteSize);
+ target.data = data.slice(firstByte, byteSize);
target.data.retain(1);
} else {
// Copy data
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
index dc8928e..7f1a3a6 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -49,7 +49,7 @@ public class VectorDescriptor {
}
public String getName() {
- return field.getLastName();
+ return field.getName();
}
public Collection<MaterializedField> getChildren() {
@@ -57,7 +57,7 @@ public class VectorDescriptor {
}
public boolean hasName() {
- return getName() != DEFAULT_NAME;
+ return !DEFAULT_NAME.equals(getName());
}
public VectorDescriptor withName(final String name) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleAccessor.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleAccessor.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleAccessor.java
index 2ebb32c..ea9b869 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleAccessor.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleAccessor.java
@@ -48,8 +48,20 @@ public interface TupleAccessor {
MaterializedField column(int index);
+ /**
+ * Returns {@code MaterializedField} instance from schema using the name specified in param.
+ *
+ * @param name name of the column in the schema
+ * @return {@code MaterializedField} instance
+ */
MaterializedField column(String name);
+ /**
+ * Returns index of the column in the schema with name specified in param.
+ *
+ * @param name name of the column in the schema
+ * @return index of the column in the schema
+ */
int columnIndex(String name);
int count();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java
index 57425af..acca767 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleReader.java
@@ -27,6 +27,13 @@ package org.apache.drill.exec.vector.accessor;
public interface TupleReader extends TupleAccessor {
ColumnReader column(int colIndex);
+
+ /**
+ * Returns column reader for the column with name specified in param.
+ *
+ * @param colName name of the column in the schema
+ * @return column reader
+ */
ColumnReader column(String colName);
Object get(int colIndex);
String getAsString(int colIndex);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java
index 59eca79..563734e 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java
@@ -29,6 +29,13 @@ package org.apache.drill.exec.vector.accessor;
public interface TupleWriter extends TupleAccessor {
ColumnWriter column(int colIndex);
+
+ /**
+ * Returns column writer for the column with name specified in param.
+ *
+ * @param colName name of the column in the schema
+ * @return column writer
+ */
ColumnWriter column(String colName);
void set(int colIndex, Object value);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
index 2c5baa3..d5b66f9 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -88,7 +88,7 @@ public abstract class AbstractContainerVector implements ValueVector {
@Nullable
@Override
public String apply(MaterializedField field) {
- return Preconditions.checkNotNull(field).getLastName();
+ return Preconditions.checkNotNull(field).getName();
}
}));
}
@@ -96,9 +96,9 @@ public abstract class AbstractContainerVector implements ValueVector {
/**
* Clears out all underlying child vectors.
*/
- @Override
+ @Override
public void close() {
- for (ValueVector vector:(Iterable<ValueVector>)this) {
+ for (ValueVector vector : this) {
vector.close();
}
}
@@ -112,14 +112,14 @@ public abstract class AbstractContainerVector implements ValueVector {
}
MajorType getLastPathType() {
- if((this.getField().getType().getMinorType() == MinorType.LIST &&
+ if ((this.getField().getType().getMinorType() == MinorType.LIST &&
this.getField().getType().getMode() == DataMode.REPEATED)) { // Use Repeated scalar type instead of Required List.
VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
ValueVector v = vord.vector;
- if (! (v instanceof AbstractContainerVector)) {
+ if (!(v instanceof AbstractContainerVector)) {
return v.getField().getType();
}
- } else if (this.getField().getType().getMinorType() == MinorType.MAP &&
+ } else if (this.getField().getType().getMinorType() == MinorType.MAP &&
this.getField().getType().getMode() == DataMode.REPEATED) { // Use Required Map
return this.getField().getType().toBuilder().setMode(DataMode.REQUIRED).build();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
index 30db41e..6b60471 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -43,7 +43,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
// Maintains a map with key as field name and value is the vector itself
- private final MapWithOrdinal<String, ValueVector> vectors = new MapWithOrdinal<>();
+ private final MapWithOrdinal<String, ValueVector> vectors = new MapWithOrdinal<>();
protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
super(field.clone(), allocator, callBack);
@@ -51,7 +51,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
// create the hierarchy of the child vectors based on the materialized field
for (MaterializedField child : clonedField.getChildren()) {
if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
- final String fieldName = child.getLastName();
+ final String fieldName = child.getName();
final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
putVector(fieldName, v);
}
@@ -60,7 +60,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
@Override
public void close() {
- for(final ValueVector valueVector : vectors.values()) {
+ for (final ValueVector valueVector : vectors.values()) {
valueVector.close();
}
vectors.clear();
@@ -135,7 +135,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
if (create) {
final T vector = (T) BasicTypeHelper.getNewVector(name, allocator, type, callBack);
putChild(name, vector);
- if (callBack!=null) {
+ if (callBack != null) {
callBack.doWork();
}
return vector;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
index c61fd00..9569946 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
@@ -14,7 +14,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- ******************************************************************************/
+ */
package org.apache.drill.exec.vector.complex;
import com.google.common.collect.ObjectArrays;
@@ -259,7 +259,7 @@ public class ListVector extends BaseRepeatedValueVector {
}
public UnionVector promoteToUnion() {
- MaterializedField newField = MaterializedField.create(getField().getPath(), Types.optional(MinorType.UNION));
+ MaterializedField newField = MaterializedField.create(getField().getName(), Types.optional(MinorType.UNION));
UnionVector vector = new UnionVector(newField, allocator, null);
replaceDataVector(vector);
reader = new UnionListReader(this);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index f9ff58d..19c910b 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -105,7 +105,7 @@ public class MapVector extends AbstractMapVector {
@Override
public void setInitialCapacity(int numRecords) {
- for (final ValueVector v : (Iterable<ValueVector>) this) {
+ for (final ValueVector v : this) {
v.setInitialCapacity(numRecords);
}
}
@@ -116,7 +116,7 @@ public class MapVector extends AbstractMapVector {
return 0;
}
long buffer = 0;
- for (final ValueVector v : (Iterable<ValueVector>)this) {
+ for (final ValueVector v : this) {
buffer += v.getBufferSize();
}
@@ -130,7 +130,7 @@ public class MapVector extends AbstractMapVector {
}
long bufferSize = 0;
- for (final ValueVector v : (Iterable<ValueVector>) this) {
+ for (final ValueVector v : this) {
bufferSize += v.getBufferSizeFor(valueCount);
}
@@ -147,7 +147,7 @@ public class MapVector extends AbstractMapVector {
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return new MapTransferPair(this, getField().getPath(), allocator);
+ return new MapTransferPair(this, getField().getName(), allocator);
}
@Override
@@ -268,11 +268,11 @@ public class MapVector extends AbstractMapVector {
for (final SerializedField child : fields) {
final MaterializedField fieldDef = MaterializedField.create(child);
- ValueVector vector = getChild(fieldDef.getLastName());
+ ValueVector vector = getChild(fieldDef.getName());
if (vector == null) {
// if we arrive here, we didn't have a matching vector.
vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
- putChild(fieldDef.getLastName(), vector);
+ putChild(fieldDef.getName(), vector);
}
if (child.getValueCount() == 0) {
vector.clear();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 999e29e..ae1f0b1 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -88,7 +88,7 @@ public class RepeatedMapVector extends AbstractMapVector
@Override
public void setInitialCapacity(int numRecords) {
offsets.setInitialCapacity(numRecords + 1);
- for(final ValueVector v : (Iterable<ValueVector>) this) {
+ for (final ValueVector v : this) {
v.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
}
}
@@ -159,7 +159,7 @@ public class RepeatedMapVector extends AbstractMapVector
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return new RepeatedMapTransferPair(this, getField().getPath(), allocator);
+ return new RepeatedMapTransferPair(this, getField().getName(), allocator);
}
@Override
@@ -442,11 +442,11 @@ public class RepeatedMapVector extends AbstractMapVector
for (int i = 1; i < children.size(); i++) {
final SerializedField child = children.get(i);
final MaterializedField fieldDef = MaterializedField.create(child);
- ValueVector vector = getChild(fieldDef.getLastName());
+ ValueVector vector = getChild(fieldDef.getName());
if (vector == null) {
// if we arrive here, we didn't have a matching vector.
vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
- putChild(fieldDef.getLastName(), vector);
+ putChild(fieldDef.getName(), vector);
}
final int vectorLength = child.getBufferLength();
vector.load(child, buffer.slice(bufOffset, vectorLength));
@@ -486,7 +486,7 @@ public class RepeatedMapVector extends AbstractMapVector
final Map<String, Object> vv = Maps.newLinkedHashMap();
for (final MaterializedField field : getField().getChildren()) {
if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
- fieldName = field.getLastName();
+ fieldName = field.getName();
final Object value = getChild(fieldName).getAccessor().getObject(i);
if (value != null) {
vv.put(fieldName, value);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
index dbbd092..10ac551 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/PromotableWriter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -149,7 +149,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
}
private FieldWriter promoteToUnion() {
- String name = vector.getField().getLastName();
+ String name = vector.getField().getName();
TransferPair tp = vector.getTransferPair(vector.getField().getType().getMinorType().name().toLowerCase(), vector.getAllocator());
tp.transfer();
if (parentContainer != null) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java b/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java
index ba3bf7a..967ca9e 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/FieldReference.java
@@ -51,19 +51,8 @@ public class FieldReference extends SchemaPath {
}
}
- private void checkSimpleString(CharSequence value) {
- if (value.toString().contains(".")) {
- throw new UnsupportedOperationException(
- String.format(
- "Unhandled field reference \"%s\"; a field reference identifier"
- + " must not have the form of a qualified name (i.e., with \".\").",
- value));
- }
- }
-
public FieldReference(CharSequence value) {
this(value, ExpressionPosition.UNKNOWN);
- checkSimpleString(value);
}
/**
@@ -87,7 +76,6 @@ public class FieldReference extends SchemaPath {
super(new NameSegment(value), pos);
if (check) {
checkData();
- checkSimpleString(value);
}
}
@@ -117,7 +105,7 @@ public class FieldReference extends SchemaPath {
JsonProcessingException {
String ref = this._parseString(jp, ctxt);
ref = ref.replace("`", "");
- return new FieldReference(ref, ExpressionPosition.UNKNOWN, false);
+ return new FieldReference(ref, ExpressionPosition.UNKNOWN, true);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java b/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
index 744a07f..16bb255 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,11 +17,15 @@
*/
package org.apache.drill.common.expression;
-public abstract class PathSegment{
+public abstract class PathSegment {
- PathSegment child;
+ private PathSegment child;
- int hash;
+ private int hash;
+
+ public PathSegment(PathSegment child) {
+ this.child = child;
+ }
public abstract PathSegment cloneWithNewChild(PathSegment segment);
@Override
@@ -35,13 +39,13 @@ public abstract class PathSegment{
}
public ArraySegment(int index, PathSegment child) {
- this.child = child;
+ super(child);
this.index = index;
- assert index >=0;
+ assert index >= 0;
}
public ArraySegment(PathSegment child) {
- this.child = child;
+ super(child);
this.index = -1;
}
@@ -50,6 +54,7 @@ public abstract class PathSegment{
}
public ArraySegment(int index) {
+ super(null);
if (index < 0 ) {
throw new IllegalArgumentException();
}
@@ -100,8 +105,8 @@ public abstract class PathSegment{
@Override
public PathSegment clone() {
PathSegment seg = index < 0 ? new ArraySegment(null) : new ArraySegment(index);
- if (child != null) {
- seg.setChild(child.clone());
+ if (getChild() != null) {
+ seg.setChild(getChild().clone());
}
return seg;
}
@@ -109,8 +114,8 @@ public abstract class PathSegment{
@Override
public ArraySegment cloneWithNewChild(PathSegment newChild) {
ArraySegment seg = index < 0 ? new ArraySegment(null) : new ArraySegment(index);
- if (child != null) {
- seg.setChild(child.cloneWithNewChild(newChild));
+ if (getChild() != null) {
+ seg.setChild(getChild().cloneWithNewChild(newChild));
} else {
seg.setChild(newChild);
}
@@ -123,11 +128,12 @@ public abstract class PathSegment{
private final String path;
public NameSegment(CharSequence n, PathSegment child) {
- this.child = child;
+ super(child);
this.path = n.toString();
}
public NameSegment(CharSequence n) {
+ super(null);
this.path = n.toString();
}
@@ -180,8 +186,8 @@ public abstract class PathSegment{
@Override
public NameSegment clone() {
NameSegment s = new NameSegment(this.path);
- if (child != null) {
- s.setChild(child.clone());
+ if (getChild() != null) {
+ s.setChild(getChild().clone());
}
return s;
}
@@ -189,8 +195,8 @@ public abstract class PathSegment{
@Override
public NameSegment cloneWithNewChild(PathSegment newChild) {
NameSegment s = new NameSegment(this.path);
- if (child != null) {
- s.setChild(child.cloneWithNewChild(newChild));
+ if (getChild() != null) {
+ s.setChild(getChild().cloneWithNewChild(newChild));
} else {
s.setChild(newChild);
}
@@ -230,7 +236,7 @@ public abstract class PathSegment{
int h = hash;
if (h == 0) {
h = segmentHashCode();
- h = 31*h + ((child == null) ? 0 : child.hashCode());
+ h = h + ((child == null) ? 0 : 31 * child.hashCode());
hash = h;
}
return h;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 026fb09..8854e15 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -69,9 +69,6 @@ public class SchemaPath extends LogicalExpressionBase {
public SchemaPath(String simpleName, ExpressionPosition pos) {
super(pos);
this.rootSegment = new NameSegment(simpleName);
- if (simpleName.contains(".")) {
- throw new IllegalStateException("This is deprecated and only supports simpe paths.");
- }
}
@@ -115,6 +112,37 @@ public class SchemaPath extends LogicalExpressionBase {
}
/**
+ * Parses input string using the same rules which are used for the field in the query.
+ * If a string contains dot outside back-ticks, or there are no backticks in the string,
+ * will be created {@link SchemaPath} with the {@link NameSegment}
+ * which contains one else {@link NameSegment}, etc.
+ * If a string contains [] then {@link ArraySegment} will be created.
+ *
+ * @param expr input string to be parsed
+ * @return {@link SchemaPath} instance
+ */
+ public static SchemaPath parseFromString(String expr) {
+ if (expr == null || expr.isEmpty()) {
+ return null;
+ }
+ try {
+ ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+ ExprParser parser = new ExprParser(tokens);
+
+ parse_return ret = parser.parse();
+
+ if (ret.e instanceof SchemaPath) {
+ return (SchemaPath) ret.e;
+ } else {
+ throw new IllegalStateException("Schema path is not a valid format.");
+ }
+ } catch (RecognitionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
* A simple is a path where there are no repeated elements outside the lowest level of the path.
* @return Whether this path is a simple path.
*/
@@ -227,25 +255,13 @@ public class SchemaPath extends LogicalExpressionBase {
return ExpressionStringBuilder.toString(this);
}
- public String getAsUnescapedPath() {
- StringBuilder sb = new StringBuilder();
- PathSegment seg = getRootSegment();
- if (seg.isArray()) {
- throw new IllegalStateException("Drill doesn't currently support top level arrays");
- }
- sb.append(seg.getNameSegment().getPath());
-
- while ( (seg = seg.getChild()) != null) {
- if (seg.isNamed()) {
- sb.append('.');
- sb.append(seg.getNameSegment().getPath());
- } else {
- sb.append('[');
- sb.append(seg.getArraySegment().getIndex());
- sb.append(']');
- }
- }
- return sb.toString();
+ /**
+ * Returns path string of {@code rootSegment}
+ *
+ * @return path string of {@code rootSegment}
+ */
+ public String getRootSegmentPath() {
+ return rootSegment.getPath();
}
public static class De extends StdDeserializer<SchemaPath> {
@@ -256,32 +272,7 @@ public class SchemaPath extends LogicalExpressionBase {
@Override
public SchemaPath deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
- String expr = jp.getText();
-
- if (expr == null || expr.isEmpty()) {
- return null;
- }
- try {
- // logger.debug("Parsing expression string '{}'", expr);
- ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
- CommonTokenStream tokens = new CommonTokenStream(lexer);
- ExprParser parser = new ExprParser(tokens);
-
- //TODO: move functionregistry and error collector to injectables.
- //ctxt.findInjectableValue(valueId, forProperty, beanInstance)
- parse_return ret = parser.parse();
-
- // ret.e.resolveAndValidate(expr, errorCollector);
- if (ret.e instanceof SchemaPath) {
- return (SchemaPath) ret.e;
- } else {
- throw new IllegalStateException("Schema path is not a valid format.");
- }
- } catch (RecognitionException e) {
- throw new RuntimeException(e);
- }
+ return parseFromString(jp.getText());
}
-
}
-
}
[4/4] drill git commit: DRILL-4264: Allow field names to include dots
Posted by am...@apache.org.
DRILL-4264: Allow field names to include dots
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/d105950a
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/d105950a
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/d105950a
Branch: refs/heads/master
Commit: d105950a7a9fb2ff3acd072ee65a51ef1fca120e
Parents: e57514a
Author: Volodymyr Vysotskyi <vv...@gmail.com>
Authored: Wed Jul 5 19:08:59 2017 +0000
Committer: Aman Sinha <as...@maprtech.com>
Committed: Mon Sep 4 19:23:49 2017 -0700
----------------------------------------------------------------------
.../db/binary/CompareFunctionsProcessor.java | 547 -------------------
.../mapr/db/binary/MapRDBFilterBuilder.java | 12 +-
.../binary/MaprDBCompareFunctionsProcessor.java | 65 +++
.../mapr/db/json/JsonConditionBuilder.java | 28 +-
.../store/hbase/CompareFunctionsProcessor.java | 461 ++++++++--------
.../exec/store/hbase/HBaseFilterBuilder.java | 6 +-
...onvertHiveParquetScanToDrillParquetScan.java | 4 +-
.../hive/HiveDrillNativeScanBatchCreator.java | 6 +-
.../drill/exec/store/kudu/KuduRecordReader.java | 6 +-
.../exec/store/kudu/KuduRecordWriterImpl.java | 4 +-
.../exec/store/mongo/MongoFilterBuilder.java | 4 +-
.../templates/EventBasedRecordWriter.java | 6 +-
.../templates/StringOutputRecordWriter.java | 4 +-
.../org/apache/drill/exec/client/DumpCat.java | 26 +-
.../drill/exec/physical/impl/ScanBatch.java | 12 +-
.../impl/TopN/PriorityQueueTemplate.java | 2 +-
.../impl/aggregate/StreamingAggBatch.java | 12 +-
.../physical/impl/common/ChainedHashTable.java | 6 +-
.../impl/flatten/FlattenRecordBatch.java | 11 +-
.../exec/physical/impl/join/HashJoinBatch.java | 2 +-
.../exec/physical/impl/join/MergeJoinBatch.java | 4 +-
.../physical/impl/join/NestedLoopJoinBatch.java | 4 +-
.../OrderedPartitionRecordBatch.java | 5 +-
.../impl/producer/ProducerConsumerBatch.java | 6 +-
.../impl/project/ProjectRecordBatch.java | 70 +--
.../impl/union/UnionAllRecordBatch.java | 95 ++--
.../physical/impl/xsort/ExternalSortBatch.java | 2 +-
.../managed/PriorityQueueCopierWrapper.java | 2 +-
.../logical/partition/PruneScanRule.java | 6 +-
.../drill/exec/record/RecordBatchLoader.java | 6 +-
.../apache/drill/exec/record/SchemaUtil.java | 21 +-
.../drill/exec/record/VectorContainer.java | 12 +-
.../exec/server/rest/WebUserConnection.java | 2 +-
.../apache/drill/exec/store/ColumnExplorer.java | 4 +-
.../store/ischema/InfoSchemaFilterBuilder.java | 6 +-
.../drill/exec/store/parquet/Metadata.java | 44 +-
.../exec/store/parquet/MetadataVersion.java | 10 +-
.../exec/store/parquet/ParquetRecordWriter.java | 6 +-
.../columnreaders/ParquetColumnMetadata.java | 6 +-
.../parquet/columnreaders/ParquetSchema.java | 10 +-
.../exec/store/parquet2/DrillParquetReader.java | 19 +-
.../apache/drill/exec/util/BatchPrinter.java | 35 +-
.../org/apache/drill/exec/util/VectorUtil.java | 14 +-
.../drill/exec/vector/complex/FieldIdUtil.java | 4 +-
.../drill/exec/vector/complex/MapUtility.java | 4 +-
.../java/org/apache/drill/BaseTestQuery.java | 5 +
.../java/org/apache/drill/DrillTestWrapper.java | 29 +-
.../java/org/apache/drill/PlanTestBase.java | 4 +-
.../java/org/apache/drill/TestUnionAll.java | 42 +-
.../expr/TestSchemaPathMaterialization.java | 21 +-
.../exec/fn/impl/TestAggregateFunctions.java | 33 +-
.../exec/physical/impl/TestOptiqPlans.java | 12 +-
.../physical/impl/TestSimpleFragmentRun.java | 8 +-
.../impl/join/TestHashJoinAdvanced.java | 43 +-
.../exec/physical/impl/join/TestMergeJoin.java | 10 +-
.../impl/mergereceiver/TestMergingReceiver.java | 14 +-
.../physical/unit/MiniPlanUnitTestBase.java | 5 +-
.../exec/record/TestMaterializedField.java | 7 +-
.../drill/exec/record/vector/TestLoad.java | 16 +-
.../drill/exec/store/TestOutputMutator.java | 4 +-
.../store/parquet/ParquetResultListener.java | 20 +-
.../exec/store/parquet/TestParquetComplex.java | 10 +-
.../store/parquet/TestParquetMetadataCache.java | 93 +++-
.../store/parquet/TestParquetPhysicalPlan.java | 8 +-
.../vector/complex/writer/TestJsonReader.java | 29 +
.../apache/drill/test/rowSet/RowSetPrinter.java | 2 +-
.../drill/test/rowSet/test/RowSetTest.java | 290 +++++-----
.../metadata_directories.requires_replace.txt | 3 +
.../V3/metadata_table.requires_replace.txt | 108 ++++
.../V3/metadata_table_t1.requires_replace.txt | 76 +++
.../V3/metadata_table_t2.requires_replace.txt | 76 +++
.../metadata_directories.requires_replace.txt | 3 +
.../v3_1/metadata_table.requires_replace.txt | 108 ++++
.../v3_1/metadata_table_t1.requires_replace.txt | 76 +++
.../v3_1/metadata_table_t2.requires_replace.txt | 76 +++
...ies_with_absolute_paths.requires_replace.txt | 3 -
...ble_with_absolute_paths.requires_replace.txt | 108 ----
..._with_absolute_paths_t1.requires_replace.txt | 76 ---
..._with_absolute_paths_t2.requires_replace.txt | 76 ---
.../store/parquet/complex/baseline8.json | 25 +-
.../jdbc/impl/DrillColumnMetaDataList.java | 6 +-
.../drill/jdbc/DrillColumnMetaDataListTest.java | 6 +-
.../codegen/templates/FixedValueVectors.java | 3 +-
.../drill/exec/record/MaterializedField.java | 41 +-
.../drill/exec/vector/BaseValueVector.java | 4 +-
.../org/apache/drill/exec/vector/BitVector.java | 5 +-
.../drill/exec/vector/VectorDescriptor.java | 6 +-
.../exec/vector/accessor/TupleAccessor.java | 12 +
.../drill/exec/vector/accessor/TupleReader.java | 7 +
.../drill/exec/vector/accessor/TupleWriter.java | 7 +
.../vector/complex/AbstractContainerVector.java | 14 +-
.../exec/vector/complex/AbstractMapVector.java | 10 +-
.../drill/exec/vector/complex/ListVector.java | 4 +-
.../drill/exec/vector/complex/MapVector.java | 12 +-
.../exec/vector/complex/RepeatedMapVector.java | 10 +-
.../vector/complex/impl/PromotableWriter.java | 4 +-
.../drill/common/expression/FieldReference.java | 14 +-
.../drill/common/expression/PathSegment.java | 40 +-
.../drill/common/expression/SchemaPath.java | 87 ++-
99 files changed, 1710 insertions(+), 1722 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java
deleted file mode 100644
index a83abf3..0000000
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/CompareFunctionsProcessor.java
+++ /dev/null
@@ -1,547 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.mapr.db.binary;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-
-import org.apache.drill.common.expression.CastExpression;
-import org.apache.drill.common.expression.ConvertExpression;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DateExpression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
-import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
-import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
-import org.apache.hadoop.hbase.util.Order;
-import org.apache.hadoop.hbase.util.PositionedByteRange;
-import org.apache.hadoop.hbase.util.SimplePositionedMutableByteRange;
-
-import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.PrefixFilter;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpression, RuntimeException> {
- private byte[] value;
- private boolean success;
- private boolean isEqualityFn;
- private SchemaPath path;
- private String functionName;
- private boolean sortOrderAscending;
-
- // Fields for row-key prefix comparison
- // If the query is on row-key prefix, we cannot use a standard template to identify startRow, stopRow and filter
- // Hence, we use these local variables(set depending upon the encoding type in user query)
- private boolean isRowKeyPrefixComparison;
- byte[] rowKeyPrefixStartRow;
- byte[] rowKeyPrefixStopRow;
- Filter rowKeyPrefixFilter;
-
- public static boolean isCompareFunction(String functionName) {
- return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName);
- }
-
- public static CompareFunctionsProcessor process(FunctionCall call, boolean nullComparatorSupported) {
- String functionName = call.getName();
- LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null;
- CompareFunctionsProcessor evaluator = new CompareFunctionsProcessor(functionName);
-
- if (valueArg != null) { // binary function
- if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) {
- LogicalExpression swapArg = valueArg;
- valueArg = nameArg;
- nameArg = swapArg;
- evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName);
- }
- evaluator.success = nameArg.accept(evaluator, valueArg);
- } else if (nullComparatorSupported && call.args.get(0) instanceof SchemaPath) {
- evaluator.success = true;
- evaluator.path = (SchemaPath) nameArg;
- }
-
- return evaluator;
- }
-
- public CompareFunctionsProcessor(String functionName) {
- this.success = false;
- this.functionName = functionName;
- this.isEqualityFn = COMPARE_FUNCTIONS_TRANSPOSE_MAP.containsKey(functionName)
- && COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName).equals(functionName);
- this.isRowKeyPrefixComparison = false;
- this.sortOrderAscending = true;
- }
-
- public byte[] getValue() {
- return value;
- }
-
- public boolean isSuccess() {
- return success;
- }
-
- public SchemaPath getPath() {
- return path;
- }
-
- public String getFunctionName() {
- return functionName;
- }
-
- public boolean isRowKeyPrefixComparison() {
- return isRowKeyPrefixComparison;
- }
-
- public byte[] getRowKeyPrefixStartRow() {
- return rowKeyPrefixStartRow;
- }
-
- public byte[] getRowKeyPrefixStopRow() {
- return rowKeyPrefixStopRow;
- }
-
- public Filter getRowKeyPrefixFilter() {
- return rowKeyPrefixFilter;
- }
-
- public boolean isSortOrderAscending() {
- return sortOrderAscending;
- }
-
- @Override
- public Boolean visitCastExpression(CastExpression e, LogicalExpression valueArg) throws RuntimeException {
- if (e.getInput() instanceof CastExpression || e.getInput() instanceof SchemaPath) {
- return e.getInput().accept(this, valueArg);
- }
- return false;
- }
-
- @Override
- public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression valueArg) throws RuntimeException {
- if (e.getConvertFunction() == ConvertExpression.CONVERT_FROM) {
-
- String encodingType = e.getEncodingType();
- int prefixLength = 0;
-
- // Handle scan pruning in the following scenario:
- // The row-key is a composite key and the CONVERT_FROM() function has byte_substr() as input function which is
- // querying for the first few bytes of the row-key(start-offset 1)
- // Example WHERE clause:
- // CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'DATE_EPOCH_BE') < DATE '2015-06-17'
- if (e.getInput() instanceof FunctionCall) {
-
- // We can prune scan range only for big-endian encoded data
- if (encodingType.endsWith("_BE") == false) {
- return false;
- }
-
- FunctionCall call = (FunctionCall)e.getInput();
- String functionName = call.getName();
- if (!functionName.equalsIgnoreCase("byte_substr")) {
- return false;
- }
-
- LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg1 = call.args.size() >= 2 ? call.args.get(1) : null;
- LogicalExpression valueArg2 = call.args.size() >= 3 ? call.args.get(2) : null;
-
- if (((nameArg instanceof SchemaPath) == false) ||
- (valueArg1 == null) || ((valueArg1 instanceof IntExpression) == false) ||
- (valueArg2 == null) || ((valueArg2 instanceof IntExpression) == false)) {
- return false;
- }
-
- boolean isRowKey = ((SchemaPath)nameArg).getAsUnescapedPath().equals(DrillHBaseConstants.ROW_KEY);
- int offset = ((IntExpression)valueArg1).getInt();
-
- if (!isRowKey || (offset != 1)) {
- return false;
- }
-
- this.path = (SchemaPath)nameArg;
- prefixLength = ((IntExpression)valueArg2).getInt();
- this.isRowKeyPrefixComparison = true;
- return visitRowKeyPrefixConvertExpression(e, prefixLength, valueArg);
- }
-
- if (e.getInput() instanceof SchemaPath) {
- ByteBuf bb = null;
-
- switch (encodingType) {
- case "INT_BE":
- case "INT":
- case "UINT_BE":
- case "UINT":
- case "UINT4_BE":
- case "UINT4":
- if (valueArg instanceof IntExpression
- && (isEqualityFn || encodingType.startsWith("U"))) {
- bb = newByteBuf(4, encodingType.endsWith("_BE"));
- bb.writeInt(((IntExpression)valueArg).getInt());
- }
- break;
- case "BIGINT_BE":
- case "BIGINT":
- case "UINT8_BE":
- case "UINT8":
- if (valueArg instanceof LongExpression
- && (isEqualityFn || encodingType.startsWith("U"))) {
- bb = newByteBuf(8, encodingType.endsWith("_BE"));
- bb.writeLong(((LongExpression)valueArg).getLong());
- }
- break;
- case "FLOAT":
- if (valueArg instanceof FloatExpression && isEqualityFn) {
- bb = newByteBuf(4, true);
- bb.writeFloat(((FloatExpression)valueArg).getFloat());
- }
- break;
- case "DOUBLE":
- if (valueArg instanceof DoubleExpression && isEqualityFn) {
- bb = newByteBuf(8, true);
- bb.writeDouble(((DoubleExpression)valueArg).getDouble());
- }
- break;
- case "TIME_EPOCH":
- case "TIME_EPOCH_BE":
- if (valueArg instanceof TimeExpression) {
- bb = newByteBuf(8, encodingType.endsWith("_BE"));
- bb.writeLong(((TimeExpression)valueArg).getTime());
- }
- break;
- case "DATE_EPOCH":
- case "DATE_EPOCH_BE":
- if (valueArg instanceof DateExpression) {
- bb = newByteBuf(8, encodingType.endsWith("_BE"));
- bb.writeLong(((DateExpression)valueArg).getDate());
- }
- break;
- case "BOOLEAN_BYTE":
- if (valueArg instanceof BooleanExpression) {
- bb = newByteBuf(1, false /* does not matter */);
- bb.writeByte(((BooleanExpression)valueArg).getBoolean() ? 1 : 0);
- }
- break;
- case "DOUBLE_OB":
- case "DOUBLE_OBD":
- if (valueArg instanceof DoubleExpression) {
- bb = newByteBuf(9, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br,
- ((DoubleExpression)valueArg).getDouble(), Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br,
- ((DoubleExpression)valueArg).getDouble(), Order.ASCENDING);
- }
- }
- break;
- case "FLOAT_OB":
- case "FLOAT_OBD":
- if (valueArg instanceof FloatExpression) {
- bb = newByteBuf(5, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br,
- ((FloatExpression)valueArg).getFloat(), Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br,
- ((FloatExpression)valueArg).getFloat(), Order.ASCENDING);
- }
- }
- break;
- case "BIGINT_OB":
- case "BIGINT_OBD":
- if (valueArg instanceof LongExpression) {
- bb = newByteBuf(9, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br,
- ((LongExpression)valueArg).getLong(), Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br,
- ((LongExpression)valueArg).getLong(), Order.ASCENDING);
- }
- }
- break;
- case "INT_OB":
- case "INT_OBD":
- if (valueArg instanceof IntExpression) {
- bb = newByteBuf(5, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br,
- ((IntExpression)valueArg).getInt(), Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br,
- ((IntExpression)valueArg).getInt(), Order.ASCENDING);
- }
- }
- break;
- case "UTF8_OB":
- case "UTF8_OBD":
- if (valueArg instanceof QuotedString) {
- int stringLen = ((QuotedString) valueArg).value.getBytes(Charsets.UTF_8).length;
- bb = newByteBuf(stringLen + 2, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, stringLen + 2);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br,
- ((QuotedString)valueArg).value, Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br,
- ((QuotedString)valueArg).value, Order.ASCENDING);
- }
- }
- break;
- case "UTF8":
- // let visitSchemaPath() handle this.
- return e.getInput().accept(this, valueArg);
- }
-
- if (bb != null) {
- this.value = bb.array();
- this.path = (SchemaPath)e.getInput();
- return true;
- }
- }
- }
- return false;
- }
-
- private Boolean visitRowKeyPrefixConvertExpression(ConvertExpression e,
- int prefixLength, LogicalExpression valueArg) {
- String encodingType = e.getEncodingType();
- rowKeyPrefixStartRow = HConstants.EMPTY_START_ROW;
- rowKeyPrefixStopRow = HConstants.EMPTY_START_ROW;
- rowKeyPrefixFilter = null;
-
- if ((encodingType.compareTo("UINT4_BE") == 0) ||
- (encodingType.compareTo("UINT_BE") == 0)) {
- if (prefixLength != 4) {
- throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix");
- }
-
- int val;
- if ((valueArg instanceof IntExpression) == false) {
- return false;
- }
-
- val = ((IntExpression)valueArg).getInt();
-
- // For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >=
- switch (functionName) {
- case "equal":
- rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(4).putInt(val).array());
- rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array();
- rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array();
- return true;
- case "greater_than_or_equal_to":
- rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array();
- return true;
- case "greater_than":
- rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val + 1).array();
- return true;
- case "less_than_or_equal_to":
- rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array();
- return true;
- case "less_than":
- rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val).array();
- return true;
- }
-
- return false;
- }
-
- if ((encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) ||
- (encodingType.compareTo("TIME_EPOCH_BE") == 0) ||
- (encodingType.compareTo("UINT8_BE") == 0)) {
-
- if (prefixLength != 8) {
- throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix");
- }
-
- long val;
- if (encodingType.compareTo("TIME_EPOCH_BE") == 0) {
- if ((valueArg instanceof TimeExpression) == false) {
- return false;
- }
-
- val = ((TimeExpression)valueArg).getTime();
- } else if (encodingType.compareTo("UINT8_BE") == 0){
- if ((valueArg instanceof LongExpression) == false) {
- return false;
- }
-
- val = ((LongExpression)valueArg).getLong();
- } else if (encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) {
- if ((valueArg instanceof TimeStampExpression) == false) {
- return false;
- }
-
- val = ((TimeStampExpression)valueArg).getTimeStamp();
- } else {
- // Should not reach here.
- return false;
- }
-
- // For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >=
- switch (functionName) {
- case "equal":
- rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(8).putLong(val).array());
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array();
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array();
- return true;
- case "greater_than_or_equal_to":
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array();
- return true;
- case "greater_than":
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val + 1).array();
- return true;
- case "less_than_or_equal_to":
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array();
- return true;
- case "less_than":
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val).array();
- return true;
- }
-
- return false;
- }
-
- if (encodingType.compareTo("DATE_EPOCH_BE") == 0) {
- if ((valueArg instanceof DateExpression) == false) {
- return false;
- }
-
- if (prefixLength != 8) {
- throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix");
- }
-
- final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24;
- long dateToSet;
- // For DATE encoding, the operators that we push-down are =, <>, <, <=, >, >=
- switch (functionName) {
- case "equal":
- long startDate = ((DateExpression)valueArg).getDate();
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(startDate).array();
- long stopDate = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY;
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(stopDate).array();
- return true;
- case "greater_than_or_equal_to":
- dateToSet = ((DateExpression)valueArg).getDate();
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
- return true;
- case "greater_than":
- dateToSet = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY;
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
- return true;
- case "less_than_or_equal_to":
- dateToSet = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY;
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
- return true;
- case "less_than":
- dateToSet = ((DateExpression)valueArg).getDate();
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
- return true;
- }
-
- return false;
- }
-
- return false;
- }
-
- @Override
- public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) throws RuntimeException {
- return false;
- }
-
- @Override
- public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) throws RuntimeException {
- if (valueArg instanceof QuotedString) {
- this.value = ((QuotedString) valueArg).value.getBytes(Charsets.UTF_8);
- this.path = path;
- return true;
- }
- return false;
- }
-
- private static ByteBuf newByteBuf(int size, boolean bigEndian) {
- return Unpooled.wrappedBuffer(new byte[size])
- .order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN)
- .writerIndex(0);
- }
-
- private static final ImmutableSet<Class<? extends LogicalExpression>> VALUE_EXPRESSION_CLASSES;
- static {
- ImmutableSet.Builder<Class<? extends LogicalExpression>> builder = ImmutableSet.builder();
- VALUE_EXPRESSION_CLASSES = builder
- .add(BooleanExpression.class)
- .add(DateExpression.class)
- .add(DoubleExpression.class)
- .add(FloatExpression.class)
- .add(IntExpression.class)
- .add(LongExpression.class)
- .add(QuotedString.class)
- .add(TimeExpression.class)
- .build();
- }
-
- private static final ImmutableMap<String, String> COMPARE_FUNCTIONS_TRANSPOSE_MAP;
- static {
- ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
- COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder
- // unary functions
- .put("isnotnull", "isnotnull")
- .put("isNotNull", "isNotNull")
- .put("is not null", "is not null")
- .put("isnull", "isnull")
- .put("isNull", "isNull")
- .put("is null", "is null")
- // binary functions
- .put("like", "like")
- .put("equal", "equal")
- .put("not_equal", "not_equal")
- .put("greater_than_or_equal_to", "less_than_or_equal_to")
- .put("greater_than", "less_than")
- .put("less_than_or_equal_to", "greater_than_or_equal_to")
- .put("less_than", "greater_than")
- .build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java
index 3aba1e7..240d6b0 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -95,7 +95,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
String functionName = call.getName();
ImmutableList<LogicalExpression> args = call.args;
- if (CompareFunctionsProcessor.isCompareFunction(functionName)) {
+ if (MaprDBCompareFunctionsProcessor.isCompareFunction(functionName)) {
/*
* HBASE-10848: Bug in HBase versions (0.94.[0-18], 0.96.[0-2], 0.98.[0-1])
* causes a filter with NullComparator to fail. Enable only if specified in
@@ -105,7 +105,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
nullComparatorSupported = groupScan.getHBaseConf().getBoolean("drill.hbase.supports.null.comparator", false);
}
- CompareFunctionsProcessor processor = CompareFunctionsProcessor.process(call, nullComparatorSupported);
+ MaprDBCompareFunctionsProcessor processor = MaprDBCompareFunctionsProcessor.createFunctionsProcessorInstance(call, nullComparatorSupported);
if (processor.isSuccess()) {
nodeScanSpec = createHBaseScanSpec(call, processor);
}
@@ -156,12 +156,12 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, newFilter);
}
- private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsProcessor processor) {
+ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, MaprDBCompareFunctionsProcessor processor) {
String functionName = processor.getFunctionName();
SchemaPath field = processor.getPath();
byte[] fieldValue = processor.getValue();
boolean sortOrderAscending = processor.isSortOrderAscending();
- boolean isRowKey = field.getAsUnescapedPath().equals(ROW_KEY);
+ boolean isRowKey = field.getRootSegmentPath().equals(ROW_KEY);
if (!(isRowKey
|| (!field.getRootSegment().isLastPath()
&& field.getRootSegment().getChild().isLastPath()
@@ -338,7 +338,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
}
private HBaseScanSpec createRowKeyPrefixScanSpec(FunctionCall call,
- CompareFunctionsProcessor processor) {
+ MaprDBCompareFunctionsProcessor processor) {
byte[] startRow = processor.getRowKeyPrefixStartRow();
byte[] stopRow = processor.getRowKeyPrefixStopRow();
Filter filter = processor.getRowKeyPrefixFilter();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MaprDBCompareFunctionsProcessor.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MaprDBCompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MaprDBCompareFunctionsProcessor.java
new file mode 100644
index 0000000..5e17c27
--- /dev/null
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MaprDBCompareFunctionsProcessor.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.mapr.db.binary;
+
+import com.google.common.base.Charsets;
+import io.netty.buffer.ByteBuf;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.ValueExpressions;
+import org.apache.drill.exec.store.hbase.CompareFunctionsProcessor;
+import org.apache.hadoop.hbase.util.Order;
+import org.apache.hadoop.hbase.util.PositionedByteRange;
+import org.apache.hadoop.hbase.util.SimplePositionedMutableByteRange;
+
+class MaprDBCompareFunctionsProcessor extends CompareFunctionsProcessor {
+
+ public MaprDBCompareFunctionsProcessor(String functionName) {
+ super(functionName);
+ }
+
+ public static MaprDBCompareFunctionsProcessor createFunctionsProcessorInstance(FunctionCall call, boolean nullComparatorSupported) {
+ String functionName = call.getName();
+ MaprDBCompareFunctionsProcessor evaluator = new MaprDBCompareFunctionsProcessor(functionName);
+
+ return createFunctionsProcessorInstanceInternal(call, nullComparatorSupported, evaluator);
+ }
+
+ @Override
+ protected ByteBuf getByteBuf(LogicalExpression valueArg, String encodingType) {
+ switch (encodingType) {
+ case "UTF8_OB":
+ case "UTF8_OBD":
+ if (valueArg instanceof ValueExpressions.QuotedString) {
+ int stringLen = ((ValueExpressions.QuotedString) valueArg).value.getBytes(Charsets.UTF_8).length;
+ ByteBuf bb = newByteBuf(stringLen + 2, true);
+ PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, stringLen + 2);
+ if (encodingType.endsWith("_OBD")) {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br, ((ValueExpressions.QuotedString) valueArg).value,
+ Order.DESCENDING);
+ setSortOrderAscending(false);
+ } else {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br, ((ValueExpressions.QuotedString) valueArg).value,
+ Order.ASCENDING);
+ }
+ return bb;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
index 16802ad..e8a8b6e 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -166,66 +166,66 @@ public class JsonConditionBuilder extends AbstractExprVisitor<JsonScanSpec, Void
switch (functionName) {
case "equal":
cond = MapRDB.newCondition();
- setIsCondition(cond, field.getAsUnescapedPath(), Op.EQUAL, fieldValue);
+ setIsCondition(cond, field.getRootSegmentPath(), Op.EQUAL, fieldValue);
cond.build();
break;
case "not_equal":
cond = MapRDB.newCondition();
- setIsCondition(cond, field.getAsUnescapedPath(), Op.NOT_EQUAL, fieldValue);
+ setIsCondition(cond, field.getRootSegmentPath(), Op.NOT_EQUAL, fieldValue);
cond.build();
break;
case "less_than":
cond = MapRDB.newCondition();
- setIsCondition(cond, field.getAsUnescapedPath(), Op.LESS, fieldValue);
+ setIsCondition(cond, field.getRootSegmentPath(), Op.LESS, fieldValue);
cond.build();
break;
case "less_than_or_equal_to":
cond = MapRDB.newCondition();
- setIsCondition(cond, field.getAsUnescapedPath(), Op.LESS_OR_EQUAL, fieldValue);
+ setIsCondition(cond, field.getRootSegmentPath(), Op.LESS_OR_EQUAL, fieldValue);
cond.build();
break;
case "greater_than":
cond = MapRDB.newCondition();
- setIsCondition(cond, field.getAsUnescapedPath(), Op.GREATER, fieldValue);
+ setIsCondition(cond, field.getRootSegmentPath(), Op.GREATER, fieldValue);
cond.build();
break;
case "greater_than_or_equal_to":
cond = MapRDB.newCondition();
- setIsCondition(cond, field.getAsUnescapedPath(), Op.GREATER_OR_EQUAL, fieldValue);
+ setIsCondition(cond, field.getRootSegmentPath(), Op.GREATER_OR_EQUAL, fieldValue);
cond.build();
break;
case "isnull":
- cond = MapRDB.newCondition().notExists(field.getAsUnescapedPath()).build();
+ cond = MapRDB.newCondition().notExists(field.getRootSegmentPath()).build();
break;
case "isnotnull":
- cond = MapRDB.newCondition().exists(field.getAsUnescapedPath()).build();
+ cond = MapRDB.newCondition().exists(field.getRootSegmentPath()).build();
break;
case "istrue":
- cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.EQUAL, true).build();
+ cond = MapRDB.newCondition().is(field.getRootSegmentPath(), Op.EQUAL, true).build();
break;
case "isnotfalse":
- cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.NOT_EQUAL, false).build();
+ cond = MapRDB.newCondition().is(field.getRootSegmentPath(), Op.NOT_EQUAL, false).build();
break;
case "isfalse":
- cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.EQUAL, false).build();
+ cond = MapRDB.newCondition().is(field.getRootSegmentPath(), Op.EQUAL, false).build();
break;
case "isnottrue":
- cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.NOT_EQUAL, true).build();
+ cond = MapRDB.newCondition().is(field.getRootSegmentPath(), Op.NOT_EQUAL, true).build();
break;
case "like":
- cond = MapRDB.newCondition().like(field.getAsUnescapedPath(), fieldValue.getString()).build();
+ cond = MapRDB.newCondition().like(field.getRootSegmentPath(), fieldValue.getString()).build();
break;
default:
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
index 09e7be7..0672b53 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -50,7 +50,7 @@ import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
-class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpression, RuntimeException> {
+public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpression, RuntimeException> {
private byte[] value;
private boolean success;
private boolean isEqualityFn;
@@ -62,31 +62,37 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
// If the query is on row-key prefix, we cannot use a standard template to identify startRow, stopRow and filter
// Hence, we use these local variables(set depending upon the encoding type in user query)
private boolean isRowKeyPrefixComparison;
- byte[] rowKeyPrefixStartRow;
- byte[] rowKeyPrefixStopRow;
- Filter rowKeyPrefixFilter;
+ private byte[] rowKeyPrefixStartRow;
+ private byte[] rowKeyPrefixStopRow;
+ private Filter rowKeyPrefixFilter;
public static boolean isCompareFunction(String functionName) {
return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName);
}
- public static CompareFunctionsProcessor process(FunctionCall call, boolean nullComparatorSupported) {
+ public static CompareFunctionsProcessor createFunctionsProcessorInstance(FunctionCall call, boolean nullComparatorSupported) {
String functionName = call.getName();
- LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null;
CompareFunctionsProcessor evaluator = new CompareFunctionsProcessor(functionName);
+ return createFunctionsProcessorInstanceInternal(call, nullComparatorSupported, evaluator);
+ }
+
+ protected static <T extends CompareFunctionsProcessor> T createFunctionsProcessorInstanceInternal(FunctionCall call,
+ boolean nullComparatorSupported,
+ T evaluator) {
+ LogicalExpression nameArg = call.args.get(0);
+ LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null;
if (valueArg != null) { // binary function
if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) {
LogicalExpression swapArg = valueArg;
valueArg = nameArg;
nameArg = swapArg;
- evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName);
+ evaluator.setFunctionName(COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(evaluator.getFunctionName()));
}
- evaluator.success = nameArg.accept(evaluator, valueArg);
+ evaluator.setSuccess(nameArg.accept(evaluator, valueArg));
} else if (nullComparatorSupported && call.args.get(0) instanceof SchemaPath) {
- evaluator.success = true;
- evaluator.path = (SchemaPath) nameArg;
+ evaluator.setSuccess(true);
+ evaluator.setPath((SchemaPath) nameArg);
}
return evaluator;
@@ -109,14 +115,26 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
return success;
}
+ protected void setSuccess(boolean success) {
+ this.success = success;
+ }
+
public SchemaPath getPath() {
return path;
}
+ protected void setPath(SchemaPath path) {
+ this.path = path;
+ }
+
public String getFunctionName() {
return functionName;
}
+ protected void setFunctionName(String functionName) {
+ this.functionName = functionName;
+ }
+
public boolean isRowKeyPrefixComparison() {
return isRowKeyPrefixComparison;
}
@@ -137,6 +155,10 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
return sortOrderAscending;
}
+ protected void setSortOrderAscending(boolean sortOrderAscending) {
+ this.sortOrderAscending = sortOrderAscending;
+ }
+
@Override
public Boolean visitCastExpression(CastExpression e, LogicalExpression valueArg) throws RuntimeException {
if (e.getInput() instanceof CastExpression || e.getInput() instanceof SchemaPath) {
@@ -147,10 +169,10 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
@Override
public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression valueArg) throws RuntimeException {
- if (e.getConvertFunction() == ConvertExpression.CONVERT_FROM) {
+ if (ConvertExpression.CONVERT_FROM.equals(e.getConvertFunction())) {
String encodingType = e.getEncodingType();
- int prefixLength = 0;
+ int prefixLength;
// Handle scan pruning in the following scenario:
// The row-key is a composite key and the CONVERT_FROM() function has byte_substr() as input function which is
@@ -160,7 +182,7 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
if (e.getInput() instanceof FunctionCall) {
// We can prune scan range only for big-endian encoded data
- if (encodingType.endsWith("_BE") == false) {
+ if (!encodingType.endsWith("_BE")) {
return false;
}
@@ -174,21 +196,21 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
LogicalExpression valueArg1 = call.args.size() >= 2 ? call.args.get(1) : null;
LogicalExpression valueArg2 = call.args.size() >= 3 ? call.args.get(2) : null;
- if (((nameArg instanceof SchemaPath) == false) ||
- (valueArg1 == null) || ((valueArg1 instanceof IntExpression) == false) ||
- (valueArg2 == null) || ((valueArg2 instanceof IntExpression) == false)) {
+ if (!(nameArg instanceof SchemaPath)
+ || (valueArg1 == null) || !(valueArg1 instanceof IntExpression)
+ || (valueArg2 == null) || !(valueArg2 instanceof IntExpression)) {
return false;
}
- boolean isRowKey = ((SchemaPath)nameArg).getAsUnescapedPath().equals(DrillHBaseConstants.ROW_KEY);
- int offset = ((IntExpression)valueArg1).getInt();
+ boolean isRowKey = ((SchemaPath) nameArg).getRootSegmentPath().equals(DrillHBaseConstants.ROW_KEY);
+ int offset = ((IntExpression) valueArg1).getInt();
- if (!isRowKey || (offset != 1)) {
+ if (!isRowKey || offset != 1) {
return false;
}
- this.path = (SchemaPath)nameArg;
- prefixLength = ((IntExpression)valueArg2).getInt();
+ this.path = (SchemaPath) nameArg;
+ prefixLength = ((IntExpression) valueArg2).getInt();
this.isRowKeyPrefixComparison = true;
return visitRowKeyPrefixConvertExpression(e, prefixLength, valueArg);
}
@@ -196,123 +218,125 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
if (e.getInput() instanceof SchemaPath) {
ByteBuf bb = null;
switch (encodingType) {
- case "INT_BE":
- case "INT":
- case "UINT_BE":
- case "UINT":
- case "UINT4_BE":
- case "UINT4":
- if (valueArg instanceof IntExpression
- && (isEqualityFn || encodingType.startsWith("U"))) {
- bb = newByteBuf(4, encodingType.endsWith("_BE"));
- bb.writeInt(((IntExpression)valueArg).getInt());
- }
- break;
- case "BIGINT_BE":
- case "BIGINT":
- case "UINT8_BE":
- case "UINT8":
- if (valueArg instanceof LongExpression
- && (isEqualityFn || encodingType.startsWith("U"))) {
- bb = newByteBuf(8, encodingType.endsWith("_BE"));
- bb.writeLong(((LongExpression)valueArg).getLong());
- }
- break;
- case "FLOAT":
- if (valueArg instanceof FloatExpression && isEqualityFn) {
- bb = newByteBuf(4, true);
- bb.writeFloat(((FloatExpression)valueArg).getFloat());
- }
- break;
- case "DOUBLE":
- if (valueArg instanceof DoubleExpression && isEqualityFn) {
- bb = newByteBuf(8, true);
- bb.writeDouble(((DoubleExpression)valueArg).getDouble());
- }
- break;
- case "TIME_EPOCH":
- case "TIME_EPOCH_BE":
- if (valueArg instanceof TimeExpression) {
- bb = newByteBuf(8, encodingType.endsWith("_BE"));
- bb.writeLong(((TimeExpression)valueArg).getTime());
- }
- break;
- case "DATE_EPOCH":
- case "DATE_EPOCH_BE":
- if (valueArg instanceof DateExpression) {
- bb = newByteBuf(8, encodingType.endsWith("_BE"));
- bb.writeLong(((DateExpression)valueArg).getDate());
- }
- break;
- case "BOOLEAN_BYTE":
- if (valueArg instanceof BooleanExpression) {
- bb = newByteBuf(1, false /* does not matter */);
- bb.writeByte(((BooleanExpression)valueArg).getBoolean() ? 1 : 0);
- }
- break;
- case "DOUBLE_OB":
- case "DOUBLE_OBD":
- if (valueArg instanceof DoubleExpression) {
- bb = newByteBuf(9, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br,
- ((DoubleExpression)valueArg).getDouble(), Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br,
- ((DoubleExpression)valueArg).getDouble(), Order.ASCENDING);
+ case "INT_BE":
+ case "INT":
+ case "UINT_BE":
+ case "UINT":
+ case "UINT4_BE":
+ case "UINT4":
+ if (valueArg instanceof IntExpression
+ && (isEqualityFn || encodingType.startsWith("U"))) {
+ bb = newByteBuf(4, encodingType.endsWith("_BE"));
+ bb.writeInt(((IntExpression) valueArg).getInt());
+ }
+ break;
+ case "BIGINT_BE":
+ case "BIGINT":
+ case "UINT8_BE":
+ case "UINT8":
+ if (valueArg instanceof LongExpression
+ && (isEqualityFn || encodingType.startsWith("U"))) {
+ bb = newByteBuf(8, encodingType.endsWith("_BE"));
+ bb.writeLong(((LongExpression) valueArg).getLong());
+ }
+ break;
+ case "FLOAT":
+ if (valueArg instanceof FloatExpression && isEqualityFn) {
+ bb = newByteBuf(4, true);
+ bb.writeFloat(((FloatExpression) valueArg).getFloat());
+ }
+ break;
+ case "DOUBLE":
+ if (valueArg instanceof DoubleExpression && isEqualityFn) {
+ bb = newByteBuf(8, true);
+ bb.writeDouble(((DoubleExpression) valueArg).getDouble());
}
- }
- break;
- case "FLOAT_OB":
- case "FLOAT_OBD":
- if (valueArg instanceof FloatExpression) {
- bb = newByteBuf(5, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br,
- ((FloatExpression)valueArg).getFloat(), Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br,
- ((FloatExpression)valueArg).getFloat(), Order.ASCENDING);
+ break;
+ case "TIME_EPOCH":
+ case "TIME_EPOCH_BE":
+ if (valueArg instanceof TimeExpression) {
+ bb = newByteBuf(8, encodingType.endsWith("_BE"));
+ bb.writeLong(((TimeExpression) valueArg).getTime());
}
- }
- break;
- case "BIGINT_OB":
- case "BIGINT_OBD":
- if (valueArg instanceof LongExpression) {
- bb = newByteBuf(9, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br,
- ((LongExpression)valueArg).getLong(), Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br,
- ((LongExpression)valueArg).getLong(), Order.ASCENDING);
+ break;
+ case "DATE_EPOCH":
+ case "DATE_EPOCH_BE":
+ if (valueArg instanceof DateExpression) {
+ bb = newByteBuf(8, encodingType.endsWith("_BE"));
+ bb.writeLong(((DateExpression) valueArg).getDate());
}
- }
- break;
- case "INT_OB":
- case "INT_OBD":
- if (valueArg instanceof IntExpression) {
- bb = newByteBuf(5, true);
- PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5);
- if (encodingType.endsWith("_OBD")) {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br,
- ((IntExpression)valueArg).getInt(), Order.DESCENDING);
- this.sortOrderAscending = false;
- } else {
- org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br,
- ((IntExpression)valueArg).getInt(), Order.ASCENDING);
+ break;
+ case "BOOLEAN_BYTE":
+ if (valueArg instanceof BooleanExpression) {
+ bb = newByteBuf(1, false /* does not matter */);
+ bb.writeByte(((BooleanExpression) valueArg).getBoolean() ? 1 : 0);
}
- }
- break;
- case "UTF8":
- // let visitSchemaPath() handle this.
- return e.getInput().accept(this, valueArg);
+ break;
+ case "DOUBLE_OB":
+ case "DOUBLE_OBD":
+ if (valueArg instanceof DoubleExpression) {
+ bb = newByteBuf(9, true);
+ PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9);
+ if (encodingType.endsWith("_OBD")) {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br,
+ ((DoubleExpression) valueArg).getDouble(), Order.DESCENDING);
+ this.sortOrderAscending = false;
+ } else {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br,
+ ((DoubleExpression) valueArg).getDouble(), Order.ASCENDING);
+ }
+ }
+ break;
+ case "FLOAT_OB":
+ case "FLOAT_OBD":
+ if (valueArg instanceof FloatExpression) {
+ bb = newByteBuf(5, true);
+ PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5);
+ if (encodingType.endsWith("_OBD")) {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br,
+ ((FloatExpression) valueArg).getFloat(), Order.DESCENDING);
+ this.sortOrderAscending = false;
+ } else {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br,
+ ((FloatExpression) valueArg).getFloat(), Order.ASCENDING);
+ }
+ }
+ break;
+ case "BIGINT_OB":
+ case "BIGINT_OBD":
+ if (valueArg instanceof LongExpression) {
+ bb = newByteBuf(9, true);
+ PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9);
+ if (encodingType.endsWith("_OBD")) {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br,
+ ((LongExpression) valueArg).getLong(), Order.DESCENDING);
+ this.sortOrderAscending = false;
+ } else {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br,
+ ((LongExpression) valueArg).getLong(), Order.ASCENDING);
+ }
+ }
+ break;
+ case "INT_OB":
+ case "INT_OBD":
+ if (valueArg instanceof IntExpression) {
+ bb = newByteBuf(5, true);
+ PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5);
+ if (encodingType.endsWith("_OBD")) {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br,
+ ((IntExpression) valueArg).getInt(), Order.DESCENDING);
+ this.sortOrderAscending = false;
+ } else {
+ org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br,
+ ((IntExpression) valueArg).getInt(), Order.ASCENDING);
+ }
+ }
+ break;
+ case "UTF8":
+ // let visitSchemaPath() handle this.
+ return e.getInput().accept(this, valueArg);
+ default:
+ bb = getByteBuf(valueArg, encodingType);
}
if (bb != null) {
@@ -325,53 +349,57 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
return false;
}
+ protected ByteBuf getByteBuf(LogicalExpression valueArg, String encodingType) {
+ return null;
+ }
+
private Boolean visitRowKeyPrefixConvertExpression(ConvertExpression e,
- int prefixLength, LogicalExpression valueArg) {
+ int prefixLength, LogicalExpression valueArg) {
String encodingType = e.getEncodingType();
rowKeyPrefixStartRow = HConstants.EMPTY_START_ROW;
- rowKeyPrefixStopRow = HConstants.EMPTY_START_ROW;
- rowKeyPrefixFilter = null;
+ rowKeyPrefixStopRow = HConstants.EMPTY_START_ROW;
+ rowKeyPrefixFilter = null;
- if ((encodingType.compareTo("UINT4_BE") == 0) ||
- (encodingType.compareTo("UINT_BE") == 0)) {
+ if ((encodingType.compareTo("UINT4_BE") == 0)
+ || (encodingType.compareTo("UINT_BE") == 0)) {
if (prefixLength != 4) {
throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix");
}
int val;
- if ((valueArg instanceof IntExpression) == false) {
+ if (!(valueArg instanceof IntExpression)) {
return false;
}
- val = ((IntExpression)valueArg).getInt();
+ val = ((IntExpression) valueArg).getInt();
// For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >=
switch (functionName) {
- case "equal":
- rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(4).putInt(val).array());
- rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array();
- rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array();
- return true;
- case "greater_than_or_equal_to":
- rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array();
- return true;
- case "greater_than":
- rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val + 1).array();
- return true;
- case "less_than_or_equal_to":
- rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array();
- return true;
- case "less_than":
- rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val).array();
- return true;
+ case "equal":
+ rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(4).putInt(val).array());
+ rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array();
+ rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array();
+ return true;
+ case "greater_than_or_equal_to":
+ rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array();
+ return true;
+ case "greater_than":
+ rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val + 1).array();
+ return true;
+ case "less_than_or_equal_to":
+ rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array();
+ return true;
+ case "less_than":
+ rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val).array();
+ return true;
}
return false;
}
- if ((encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) ||
- (encodingType.compareTo("TIME_EPOCH_BE") == 0) ||
- (encodingType.compareTo("UINT8_BE") == 0)) {
+ if ((encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0)
+ || (encodingType.compareTo("TIME_EPOCH_BE") == 0)
+ || (encodingType.compareTo("UINT8_BE") == 0)) {
if (prefixLength != 8) {
throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix");
@@ -379,23 +407,23 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
long val;
if (encodingType.compareTo("TIME_EPOCH_BE") == 0) {
- if ((valueArg instanceof TimeExpression) == false) {
+ if (!(valueArg instanceof TimeExpression)) {
return false;
}
- val = ((TimeExpression)valueArg).getTime();
- } else if (encodingType.compareTo("UINT8_BE") == 0){
- if ((valueArg instanceof LongExpression) == false) {
+ val = ((TimeExpression) valueArg).getTime();
+ } else if (encodingType.compareTo("UINT8_BE") == 0) {
+ if (!(valueArg instanceof LongExpression)) {
return false;
}
- val = ((LongExpression)valueArg).getLong();
+ val = ((LongExpression) valueArg).getLong();
} else if (encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) {
- if ((valueArg instanceof TimeStampExpression) == false) {
+ if (!(valueArg instanceof TimeStampExpression)) {
return false;
}
- val = ((TimeStampExpression)valueArg).getTimeStamp();
+ val = ((TimeStampExpression) valueArg).getTimeStamp();
} else {
// Should not reach here.
return false;
@@ -403,30 +431,30 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
// For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >=
switch (functionName) {
- case "equal":
- rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(8).putLong(val).array());
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array();
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array();
- return true;
- case "greater_than_or_equal_to":
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array();
- return true;
- case "greater_than":
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val + 1).array();
- return true;
- case "less_than_or_equal_to":
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array();
- return true;
- case "less_than":
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val).array();
- return true;
+ case "equal":
+ rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(8).putLong(val).array());
+ rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array();
+ rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array();
+ return true;
+ case "greater_than_or_equal_to":
+ rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array();
+ return true;
+ case "greater_than":
+ rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val + 1).array();
+ return true;
+ case "less_than_or_equal_to":
+ rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array();
+ return true;
+ case "less_than":
+ rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val).array();
+ return true;
}
return false;
}
if (encodingType.compareTo("DATE_EPOCH_BE") == 0) {
- if ((valueArg instanceof DateExpression) == false) {
+ if (!(valueArg instanceof DateExpression)) {
return false;
}
@@ -434,41 +462,41 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix");
}
- final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24;
+ final long MILLISECONDS_IN_A_DAY = 1000 * 60 * 60 * 24;
long dateToSet;
// For DATE encoding, the operators that we push-down are =, <>, <, <=, >, >=
switch (functionName) {
- case "equal":
- long startDate = ((DateExpression)valueArg).getDate();
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(startDate).array();
- long stopDate = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY;
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(stopDate).array();
- return true;
- case "greater_than_or_equal_to":
- dateToSet = ((DateExpression)valueArg).getDate();
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
- return true;
- case "greater_than":
- dateToSet = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY;
- rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
- return true;
- case "less_than_or_equal_to":
- dateToSet = ((DateExpression)valueArg).getDate() + MILLISECONDS_IN_A_DAY;
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
- return true;
- case "less_than":
- dateToSet = ((DateExpression)valueArg).getDate();
- rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
- return true;
+ case "equal":
+ long startDate = ((DateExpression) valueArg).getDate();
+ rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(startDate).array();
+ long stopDate = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY;
+ rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(stopDate).array();
+ return true;
+ case "greater_than_or_equal_to":
+ dateToSet = ((DateExpression) valueArg).getDate();
+ rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
+ return true;
+ case "greater_than":
+ dateToSet = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY;
+ rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
+ return true;
+ case "less_than_or_equal_to":
+ dateToSet = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY;
+ rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
+ return true;
+ case "less_than":
+ dateToSet = ((DateExpression) valueArg).getDate();
+ rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
+ return true;
}
return false;
- }
+ }
- return false;
-}
+ return false;
+ }
-@Override
+ @Override
public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) throws RuntimeException {
return false;
}
@@ -483,7 +511,7 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
return false;
}
- private static ByteBuf newByteBuf(int size, boolean bigEndian) {
+ protected static ByteBuf newByteBuf(int size, boolean bigEndian) {
return Unpooled.wrappedBuffer(new byte[size])
.order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN)
.writerIndex(0);
@@ -525,5 +553,4 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
.put("less_than", "greater_than")
.build();
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
index 394e128..8d2e8ff 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -99,7 +99,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
nullComparatorSupported = groupScan.getHBaseConf().getBoolean("drill.hbase.supports.null.comparator", false);
}
- CompareFunctionsProcessor processor = CompareFunctionsProcessor.process(call, nullComparatorSupported);
+ CompareFunctionsProcessor processor = CompareFunctionsProcessor.createFunctionsProcessorInstance(call, nullComparatorSupported);
if (processor.isSuccess()) {
nodeScanSpec = createHBaseScanSpec(call, processor);
}
@@ -155,7 +155,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
SchemaPath field = processor.getPath();
byte[] fieldValue = processor.getValue();
boolean sortOrderAscending = processor.isSortOrderAscending();
- boolean isRowKey = field.getAsUnescapedPath().equals(ROW_KEY);
+ boolean isRowKey = field.getRootSegmentPath().equals(ROW_KEY);
if (!(isRowKey
|| (!field.getRootSegment().isLastPath()
&& field.getRootSegment().getChild().isLastPath()
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
index bb59600..b1b966a 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -233,7 +233,7 @@ public class ConvertHiveParquetScanToDrillParquetScan extends StoragePluginOptim
// unlike above where we expanded the '*'. HiveScan and related (subscan) can handle '*'.
final List<SchemaPath> nativeScanCols = Lists.newArrayList();
for(SchemaPath colName : hiveScanRel.getColumns()) {
- final String partitionCol = partitionColMapping.get(colName.getAsUnescapedPath());
+ final String partitionCol = partitionColMapping.get(colName.getRootSegmentPath());
if (partitionCol != null) {
nativeScanCols.add(SchemaPath.getSimplePath(partitionCol));
} else {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java
index 66f41e2..6c10d25 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeScanBatchCreator.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -80,10 +80,10 @@ public class HiveDrillNativeScanBatchCreator implements BatchCreator<HiveDrillNa
newColumns = Lists.newArrayList();
Pattern pattern = Pattern.compile(String.format("%s[0-9]+", partitionDesignator));
for (SchemaPath column : columns) {
- Matcher m = pattern.matcher(column.getAsUnescapedPath());
+ Matcher m = pattern.matcher(column.getRootSegmentPath());
if (m.matches()) {
selectedPartitionColumns.add(
- Integer.parseInt(column.getAsUnescapedPath().substring(partitionDesignator.length())));
+ Integer.parseInt(column.getRootSegmentPath().substring(partitionDesignator.length())));
} else {
newColumns.add(column);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
index ef7efcf..b9a7bf9 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -106,7 +106,7 @@ public class KuduRecordReader extends AbstractRecordReader {
if (!isStarQuery()) {
List<String> colNames = Lists.newArrayList();
for (SchemaPath p : this.getColumns()) {
- colNames.add(p.getAsUnescapedPath());
+ colNames.add(p.getRootSegmentPath());
}
builder.setProjectedColumnNames(colNames);
}
@@ -199,7 +199,7 @@ public class KuduRecordReader extends AbstractRecordReader {
majorType = Types.required(minorType);
}
MaterializedField field = MaterializedField.create(name, majorType);
- final Class<? extends ValueVector> clazz = (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(
+ final Class<? extends ValueVector> clazz = TypeHelper.getValueVectorClass(
minorType, majorType.getMode());
ValueVector vector = output.addField(field, clazz);
vector.allocateNew();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
index 2e40acf..9b98ccd 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -76,7 +76,7 @@ public class KuduRecordWriterImpl extends KuduRecordWriter {
if (!checkForTable(name)) {
List<ColumnSchema> columns = new ArrayList<>();
for (MaterializedField f : schema) {
- columns.add(new ColumnSchema.ColumnSchemaBuilder(f.getLastName(), getType(f.getType()))
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(f.getName(), getType(f.getType()))
.nullable(f.getType().getMode() == DataMode.OPTIONAL)
.key(i == 0).build());
i++;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java
index 379f449..70e9a1a 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -165,7 +165,7 @@ public class MongoFilterBuilder extends
SchemaPath field, Object fieldValue) throws ClassNotFoundException,
IOException {
// extract the field name
- String fieldName = field.getAsUnescapedPath();
+ String fieldName = field.getRootSegmentPath();
MongoCompareOp compareOp = null;
switch (functionName) {
case "equal":
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
index a2428a7..b88d4a1 100644
--- a/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/EventBasedRecordWriter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -82,11 +82,11 @@ public class EventBasedRecordWriter {
try {
int fieldId = 0;
for (VectorWrapper w : batch) {
- if (w.getField().getPath().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) {
+ if (w.getField().getName().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) {
continue;
}
FieldReader reader = w.getValueVector().getReader();
- FieldConverter converter = getConverter(recordWriter, fieldId++, w.getField().getLastName(), reader);
+ FieldConverter converter = getConverter(recordWriter, fieldId++, w.getField().getName(), reader);
fieldConverters.add(converter);
}
} catch(Exception e) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java b/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java
index 6d92a2f..70c699e 100644
--- a/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java
+++ b/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -64,7 +64,7 @@ public abstract class StringOutputRecordWriter extends AbstractRecordWriter {
BatchSchema schema = batch.getSchema();
List<String> columnNames = Lists.newArrayList();
for (int i=0; i < schema.getFieldCount(); i++) {
- columnNames.add(schema.getColumn(i).getLastName());
+ columnNames.add(schema.getColumn(i).getName());
}
startNewSchema(columnNames);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
index cdbd3b4..6c85921 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -100,16 +100,16 @@ public class DumpCat {
* Options as input to JCommander.
*/
static class Options {
- @Parameter(names = {"-f"}, description = "file containing dump", required=true)
+ @Parameter(names = {"-f"}, description = "file containing dump", required = true)
public String location = null;
- @Parameter(names = {"-batch"}, description = "id of batch to show", required=false, validateWith = BatchNumValidator.class)
+ @Parameter(names = {"-batch"}, description = "id of batch to show", required = false, validateWith = BatchNumValidator.class)
public int batch = -1;
- @Parameter(names = {"-include-headers"}, description = "whether include header of batch", required=false)
+ @Parameter(names = {"-include-headers"}, description = "whether include header of batch", required = false)
public boolean include_headers = false;
- @Parameter(names = {"-h", "-help", "--help"}, description = "show usage", help=true)
+ @Parameter(names = {"-h", "-help", "--help"}, description = "show usage", help = true)
public boolean help = false;
}
@@ -138,7 +138,7 @@ public class DumpCat {
@Override
public String toString() {
- String avgRecSizeStr = null;
+ String avgRecSizeStr;
if (this.rows > 0) {
avgRecSizeStr = String.format("Average Record Size : %d ", this.dataSize/this.rows);
} else {
@@ -175,7 +175,7 @@ public class DumpCat {
while (input.available() > 0) {
final VectorAccessibleSerializable vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
vcSerializable.readFromStream(input);
- final VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
+ final VectorContainer vectorContainer = vcSerializable.get();
aggBatchMetaInfo.add(getBatchMetaInfo(vcSerializable));
@@ -224,7 +224,7 @@ public class DumpCat {
vcSerializable.readFromStream(input);
if (batchNum != targetBatchNum) {
- final VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
+ final VectorContainer vectorContainer = vcSerializable.get();
vectorContainer.zeroVectors();
}
}
@@ -237,13 +237,13 @@ public class DumpCat {
if (vcSerializable != null) {
showSingleBatch(vcSerializable, showHeader);
- final VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
+ final VectorContainer vectorContainer = vcSerializable.get();
vectorContainer.zeroVectors();
}
}
private void showSingleBatch (VectorAccessibleSerializable vcSerializable, boolean showHeader) {
- final VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
+ final VectorContainer vectorContainer = vcSerializable.get();
/* show the header of the batch */
if (showHeader) {
@@ -253,7 +253,7 @@ public class DumpCat {
for (final VectorWrapper w : vectorContainer) {
final MaterializedField field = w.getValueVector().getField();
System.out.println (String.format("name : %s, minor_type : %s, data_mode : %s",
- field.getPath(),
+ field.getName(),
field.getType().getMinorType().toString(),
field.isNullable() ? "nullable":"non-nullable"
));
@@ -268,8 +268,8 @@ public class DumpCat {
private BatchMetaInfo getBatchMetaInfo(VectorAccessibleSerializable vcSerializable) {
final VectorAccessible vectorContainer = vcSerializable.get();
- int rows = 0;
- int selectedRows = 0;
+ int rows;
+ int selectedRows;
int totalDataSize = 0;
rows = vectorContainer.getRecordCount();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 4218069..803bd48 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -359,19 +359,19 @@ public class ScanBatch implements CloseableRecordBatch {
public <T extends ValueVector> T addField(MaterializedField field,
Class<T> clazz) throws SchemaChangeException {
// Check if the field exists.
- ValueVector v = fieldVectorMap.get(field.getPath());
+ ValueVector v = fieldVectorMap.get(field.getName());
if (v == null || v.getClass() != clazz) {
// Field does not exist--add it to the map and the output container.
v = TypeHelper.getNewVector(field, allocator, callBack);
if (!clazz.isAssignableFrom(v.getClass())) {
throw new SchemaChangeException(
- String.format(
- "The class that was provided, %s, does not correspond to the "
- + "expected vector type of %s.",
- clazz.getSimpleName(), v.getClass().getSimpleName()));
+ String.format(
+ "The class that was provided, %s, does not correspond to the "
+ + "expected vector type of %s.",
+ clazz.getSimpleName(), v.getClass().getSimpleName()));
}
- final ValueVector old = fieldVectorMap.put(field.getPath(), v);
+ final ValueVector old = fieldVectorMap.put(field.getName(), v);
if (old != null) {
old.clear();
container.remove(old);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
index ff159cd..b82dfc8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
@@ -71,7 +71,7 @@ public abstract class PriorityQueueTemplate implements PriorityQueue {
BatchSchema schema = container.getSchema();
VectorContainer newContainer = new VectorContainer();
for (MaterializedField field : schema) {
- int[] ids = container.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds();
+ int[] ids = container.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds();
newContainer.add(container.getValueAccessorById(field.getValueClass(), ids).getValueVectors());
}
newContainer.buildSchema(BatchSchema.SelectionVectorMode.FOUR_BYTE);
[2/4] drill git commit: DRILL-4264: Allow field names to include dots
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
index 0394029..2d3c13c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -163,7 +163,7 @@ public class FieldIdUtil {
}
public static TypedFieldId getFieldId(ValueVector vector, int id, SchemaPath expectedPath, boolean hyper) {
- if (!expectedPath.getRootSegment().getNameSegment().getPath().equalsIgnoreCase(vector.getField().getPath())) {
+ if (!expectedPath.getRootSegment().getPath().equalsIgnoreCase(vector.getField().getName())) {
return null;
}
PathSegment seg = expectedPath.getRootSegment();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java
index 7a5863a..72c094a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -222,7 +222,7 @@ public class MapUtility {
}
} catch (ClassCastException e) {
final MaterializedField field = fieldReader.getField();
- throw new DrillRuntimeException(String.format(TYPE_MISMATCH_ERROR, field.getPath(), field.getType()));
+ throw new DrillRuntimeException(String.format(TYPE_MISMATCH_ERROR, field.getName(), field.getType()));
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index 4d49c7b..f7f41fc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -626,6 +626,11 @@ public class BaseTestQuery extends ExecTest {
copyMetaDataCacheToTempWithReplacements(srcFileOnClassPath, destFolderInTmp, metaFileName, null);
}
+ protected static void copyMetaDataCacheToTempReplacingInternalPaths(Path srcFileOnClassPath, String destFolderInTmp,
+ String metaFileName) throws IOException {
+ copyMetaDataCacheToTempReplacingInternalPaths(srcFileOnClassPath.toUri().getPath(), destFolderInTmp, metaFileName);
+ }
+
/**
* Old metadata cache files include full paths to the files that have been scanned.
* <p>
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
index 64aeef8..2bc78d4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
@@ -14,7 +14,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- ******************************************************************************/
+ */
package org.apache.drill;
import static org.junit.Assert.assertEquals;
@@ -33,7 +33,6 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-import com.google.common.base.Preconditions;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
@@ -239,7 +238,7 @@ public class DrillTestWrapper {
logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
totalRecords += loader.getRecordCount();
for (VectorWrapper<?> w : loader) {
- String field = SchemaPath.getSimplePath(w.getField().getPath()).toExpr();
+ String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr();
if (!combinedVectors.containsKey(field)) {
MaterializedField mf = w.getField();
ValueVector[] vvList = (ValueVector[]) Array.newInstance(mf.getValueClass(), 1);
@@ -350,7 +349,7 @@ public class DrillTestWrapper {
if (schema == null) {
schema = loader.getSchema();
for (MaterializedField mf : schema) {
- combinedVectors.put(SchemaPath.getSimplePath(mf.getPath()).toExpr(), new ArrayList<Object>());
+ combinedVectors.put(SchemaPath.getSimplePath(mf.getName()).toExpr(), new ArrayList<>());
}
} else {
// TODO - actually handle schema changes, this is just to get access to the SelectionVectorMode
@@ -361,7 +360,7 @@ public class DrillTestWrapper {
logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
totalRecords += loader.getRecordCount();
for (VectorWrapper<?> w : loader) {
- String field = SchemaPath.getSimplePath(w.getField().getPath()).toExpr();
+ String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr();
ValueVector[] vectors;
if (w.isHyper()) {
vectors = w.getValueVectors();
@@ -428,18 +427,18 @@ public class DrillTestWrapper {
final BatchSchema schema = loader.getSchema();
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = testBuilder.getExpectedSchema();
- if(schema.getFieldCount() != expectedSchema.size()) {
+ if (schema.getFieldCount() != expectedSchema.size()) {
throw new Exception("Expected and actual numbers of columns do not match.");
}
- for(int i = 0; i < schema.getFieldCount(); ++i) {
- final String actualSchemaPath = schema.getColumn(i).getPath();
+ for (int i = 0; i < schema.getFieldCount(); ++i) {
+ final String actualSchemaPath = schema.getColumn(i).getName();
final TypeProtos.MajorType actualMajorType = schema.getColumn(i).getType();
- final String expectedSchemaPath = expectedSchema.get(i).getLeft().getAsUnescapedPath();
+ final String expectedSchemaPath = expectedSchema.get(i).getLeft().getRootSegmentPath();
final TypeProtos.MajorType expectedMajorType = expectedSchema.get(i).getValue();
- if(!actualSchemaPath.equals(expectedSchemaPath)
+ if (!actualSchemaPath.equals(expectedSchemaPath)
|| !actualMajorType.equals(expectedMajorType)) {
throw new Exception(String.format("Schema path or type mismatch for column #%d:\n" +
"Expected schema path: %s\nActual schema path: %s\nExpected type: %s\nActual type: %s",
@@ -448,8 +447,8 @@ public class DrillTestWrapper {
}
}
- } finally {
- if(batch != null) {
+ } finally {
+ if (batch != null) {
batch.release();
}
loader.clear();
@@ -609,7 +608,7 @@ public class DrillTestWrapper {
private Map<SchemaPath, TypeProtos.MajorType> getTypeMapFromBatch(QueryDataBatch batch) {
Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
for (int i = 0; i < batch.getHeader().getDef().getFieldCount(); i++) {
- typeMap.put(SchemaPath.getSimplePath(MaterializedField.create(batch.getHeader().getDef().getField(i)).getPath()),
+ typeMap.put(SchemaPath.getSimplePath(MaterializedField.create(batch.getHeader().getDef().getField(i)).getName()),
batch.getHeader().getDef().getField(i).getMajorType());
}
return typeMap;
@@ -646,9 +645,9 @@ public class DrillTestWrapper {
if (obj instanceof Text) {
obj = obj.toString();
}
- record.put(SchemaPath.getSimplePath(w.getField().getPath()).toExpr(), obj);
+ record.put(SchemaPath.getSimplePath(w.getField().getName()).toExpr(), obj);
}
- record.put(SchemaPath.getSimplePath(w.getField().getPath()).toExpr(), obj);
+ record.put(SchemaPath.getSimplePath(w.getField().getName()).toExpr(), obj);
}
materializedRecords.add(record);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
index e422a77..eb11532 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -349,7 +349,7 @@ public class PlanTestBase extends BaseTestQuery {
}
if (!silent) {
- System.out.println(vw.getValueVector().getField().getPath());
+ System.out.println(vw.getValueVector().getField().getName());
}
final ValueVector vv = vw.getValueVector();
for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
index ee350ce..6965ab5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
@@ -28,9 +28,12 @@ import org.apache.drill.exec.work.foreman.SqlUnsupportedException;
import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException;
import org.junit.Test;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
import java.util.List;
-public class TestUnionAll extends BaseTestQuery{
+public class TestUnionAll extends BaseTestQuery {
private static final String sliceTargetSmall = "alter session set `planner.slice_target` = 1";
private static final String sliceTargetDefault = "alter session reset `planner.slice_target`";
@@ -1189,4 +1192,41 @@ public class TestUnionAll extends BaseTestQuery{
.go();
}
+ @Test // DRILL-4264
+ public void testFieldWithDots() throws Exception {
+ File directory = new File(BaseTestQuery.getTempDir("json/input"));
+ try {
+ directory.mkdirs();
+ String fileName = "table.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
+ writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
+ }
+
+ String query = String.format("select * from (" +
+ "(select t.m.`a.b` as a,\n" +
+ "t.m.a.b as b,\n" +
+ "t.m['a.b'] as c,\n" +
+ "t.rk.q as d,\n" +
+ "t.`rk.q` as e\n" +
+ "from dfs_test.`%1$s/%2$s` t)\n" +
+ "union all\n" +
+ "(select t.m.`a.b` as a,\n" +
+ "t.m.a.b as b,\n" +
+ "t.m['a.b'] as c,\n" +
+ "t.rk.q as d,\n" +
+ "t.`rk.q` as e\n" +
+ "from dfs_test.`%1$s/%2$s` t))",
+ directory.toPath().toString(), fileName);
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("a", "b", "c", "d", "e")
+ .baselineValues("1", "2", "1", null, "a")
+ .baselineValues("1", "2", "1", null, "a")
+ .go();
+
+ } finally {
+ org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
index c2ab18a..cf7fd90 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -93,4 +93,23 @@ public class TestSchemaPathMaterialization extends BaseTestQuery {
.go();
}
+ @Test //DRILL-4264
+ public void testFieldNameWithDot() throws Exception {
+ final String tableName = "dfs_test.tmp.table_with_dot_field";
+ try {
+ test("create table %s as select o_custkey as `x.y.z` from cp.`tpch/orders.parquet`", tableName);
+
+ final String query = "select * from %s t where `x.y.z`=1091";
+
+ testBuilder()
+ .sqlQuery(query, tableName)
+ .unOrdered()
+ .baselineColumns("`x.y.z`")
+ .baselineValues(1091)
+ .baselineValues(1091)
+ .go();
+ } finally {
+ test("drop table if exists %s", tableName);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
index 36ee1b9..cfb1c5f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -30,6 +30,9 @@ import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.junit.Ignore;
import org.junit.Test;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
import java.util.List;
import java.util.Map;
@@ -570,4 +573,32 @@ public class TestAggregateFunctions extends BaseTestQuery {
}
}
+ @Test // DRILL-4264
+ public void testCountOnFieldWithDots() throws Exception {
+ File directory = new File(BaseTestQuery.getTempDir("json/input"));
+ try {
+ directory.mkdirs();
+ String fileName = "table.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
+ writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
+ }
+
+ String query = String.format("select count(t.m.`a.b`) as a,\n" +
+ "count(t.m.a.b) as b,\n" +
+ "count(t.m['a.b']) as c,\n" +
+ "count(t.rk.q) as d,\n" +
+ "count(t.`rk.q`) as e\n" +
+ "from dfs_test.`%s/%s` t",
+ directory.toPath().toString(), fileName);
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("a", "b", "c", "d", "e")
+ .baselineValues(1L, 1L, 1L, 0L, 1L)
+ .go();
+
+ } finally {
+ org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index e016b04..9701c7d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -142,7 +142,7 @@ public class TestOptiqPlans extends ExecTest {
System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
loader.load(b.getHeader().getDef(), b.getData());
for (final VectorWrapper<?> vw : loader) {
- System.out.println(vw.getValueVector().getField().getPath());
+ System.out.println(vw.getValueVector().getField().getName());
final ValueVector vv = vw.getValueVector();
for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
final Object o = vv.getAccessor().getObject(i);
@@ -171,7 +171,7 @@ public class TestOptiqPlans extends ExecTest {
System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
loader.load(b.getHeader().getDef(), b.getData());
for (final VectorWrapper<?> vw : loader) {
- System.out.println(vw.getValueVector().getField().getPath());
+ System.out.println(vw.getValueVector().getField().getName());
final ValueVector vv = vw.getValueVector();
for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
final Object o = vv.getAccessor().getObject(i);
@@ -200,7 +200,7 @@ public class TestOptiqPlans extends ExecTest {
System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
loader.load(b.getHeader().getDef(), b.getData());
for (final VectorWrapper<?> vw : loader) {
- System.out.println(vw.getValueVector().getField().getPath());
+ System.out.println(vw.getValueVector().getField().getName());
final ValueVector vv = vw.getValueVector();
for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
final Object o = vv.getAccessor().getObject(i);
@@ -239,7 +239,7 @@ public class TestOptiqPlans extends ExecTest {
System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
loader.load(b.getHeader().getDef(), b.getData());
for (final VectorWrapper vw : loader) {
- System.out.println(vw.getValueVector().getField().getPath());
+ System.out.println(vw.getValueVector().getField().getName());
final ValueVector vv = vw.getValueVector();
for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
final Object o = vv.getAccessor().getObject(i);
@@ -278,7 +278,7 @@ public class TestOptiqPlans extends ExecTest {
System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
loader.load(b.getHeader().getDef(), b.getData());
for (final VectorWrapper vw : loader) {
- System.out.println(vw.getValueVector().getField().getPath());
+ System.out.println(vw.getValueVector().getField().getName());
final ValueVector vv = vw.getValueVector();
for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
final Object o = vv.getAccessor().getObject(i);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index 182e19e..3b8ab3f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -46,7 +46,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
public void runNoExchangeFragment() throws Exception {
try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
final Drillbit bit = new Drillbit(CONFIG, serviceSet);
- final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
+ final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
// run query.
bit.run();
@@ -72,7 +72,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
} else {
System.out.print("\t");
}
- System.out.print(value.getField().getPath());
+ System.out.print(value.getField().getName());
System.out.print("[");
System.out.print(value.getField().getType().getMinorType());
System.out.print("]");
@@ -147,7 +147,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
} else {
System.out.print("\t");
}
- System.out.print(v.getField().getPath());
+ System.out.print(v.getField().getName());
System.out.print("[");
System.out.print(v.getField().getType().getMinorType());
System.out.print("]");
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java
index a70a3f8..16df3ac 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -22,9 +22,12 @@ package org.apache.drill.exec.physical.impl.join;
import org.apache.drill.BaseTestQuery;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+
public class TestHashJoinAdvanced extends BaseTestQuery {
// Have to disable merge join, if this testcase is to test "HASH-JOIN".
@@ -88,6 +91,7 @@ public class TestHashJoinAdvanced extends BaseTestQuery {
.build()
.run();
}
+
@Test
public void testJoinWithDifferentTypesInCondition() throws Exception {
String query = "select t1.full_name from cp.`employee.json` t1, cp.`department.json` t2 " +
@@ -112,7 +116,7 @@ public class TestHashJoinAdvanced extends BaseTestQuery {
.optionSettingQueriesForTestQuery("alter session set `planner.enable_hashjoin` = true")
.unOrdered()
.baselineColumns("bigint_col")
- .baselineValues(1l)
+ .baselineValues(1L)
.go();
query = "select count(*) col1 from " +
@@ -123,7 +127,38 @@ public class TestHashJoinAdvanced extends BaseTestQuery {
.sqlQuery(query)
.unOrdered()
.baselineColumns("col1")
- .baselineValues(4l)
+ .baselineValues(4L)
.go();
}
+
+ @Test //DRILL-2197 Left Join with complex type in projection
+ public void testJoinWithMapAndDotField() throws Exception {
+ File directory = new File(BaseTestQuery.getTempDir("json/input"));
+ try {
+ directory.mkdirs();
+ String fileName = "table.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
+ writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
+ }
+
+ String query = String.format("select t1.m.`a.b` as a,\n" +
+ "t2.m.a.b as b,\n" +
+ "t1.m['a.b'] as c,\n" +
+ "t2.rk.q as d,\n" +
+ "t1.`rk.q` as e\n" +
+ "from dfs_test.`%1$s/%2$s` t1,\n" +
+ "dfs_test.`%1$s/%2$s` t2\n" +
+ "where t1.m.`a.b`=t2.m.`a.b` and t1.m.a.b=t2.m.a.b",
+ directory.toPath().toString(), fileName);
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("a", "b", "c", "d", "e")
+ .baselineValues("1", "2", "1", null, "a")
+ .go();
+
+ } finally {
+ org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
index 53c0a67..8ba442d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -74,7 +74,7 @@ public class TestMergeJoin extends PopUnitTestBase {
while (exec.next()) {
totalRecordCount += exec.getRecordCount();
for (final ValueVector v : exec) {
- System.out.print("[" + v.getField().getPath() + "] ");
+ System.out.print("[" + v.getField().getName() + "] ");
}
System.out.println("\n");
for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) {
@@ -131,7 +131,7 @@ public class TestMergeJoin extends PopUnitTestBase {
for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) {
final List<Object> row = Lists.newArrayList();
for (final ValueVector v : exec) {
- row.add(v.getField().getPath() + ":" + v.getAccessor().getObject(valueIdx));
+ row.add(v.getField().getName() + ":" + v.getAccessor().getObject(valueIdx));
}
for (final Object cell : row) {
if (cell == null) {
@@ -182,7 +182,7 @@ public class TestMergeJoin extends PopUnitTestBase {
for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) {
final List<Object> row = Lists.newArrayList();
for (final ValueVector v : exec) {
- row.add(v.getField().getPath() + ":" + v.getAccessor().getObject(valueIdx));
+ row.add(v.getField().getName() + ":" + v.getAccessor().getObject(valueIdx));
}
for (final Object cell : row) {
if (cell == null) {
@@ -232,7 +232,7 @@ public class TestMergeJoin extends PopUnitTestBase {
for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) {
final List<Object> row = Lists.newArrayList();
for (final ValueVector v : exec) {
- row.add(v.getField().getPath() + ":" + v.getAccessor().getObject(valueIdx));
+ row.add(v.getField().getName() + ":" + v.getAccessor().getObject(valueIdx));
}
for (final Object cell : row) {
if (cell == null) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
index e4a96bd..6fb7f72 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -49,7 +49,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
final Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
- final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
+ final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
bit1.run();
bit2.run();
client.connect();
@@ -78,7 +78,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
final Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
- final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
+ final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
bit1.run();
bit2.run();
@@ -102,9 +102,9 @@ public class TestMergingReceiver extends PopUnitTestBase {
final ValueVector.Accessor va = vv.getAccessor();
final MaterializedField materializedField = vv.getField();
final int numValues = va.getValueCount();
- for(int valueIdx = 0; valueIdx < numValues; ++valueIdx) {
- if (materializedField.getPath().equals("blue")) {
- final long longValue = ((Long) va.getObject(valueIdx)).longValue();
+ for (int valueIdx = 0; valueIdx < numValues; ++valueIdx) {
+ if (materializedField.getName().equals("blue")) {
+ final long longValue = (Long) va.getObject(valueIdx);
// check that order is ascending
if (lastBlueValue != null) {
assertTrue(longValue >= lastBlueValue);
@@ -127,7 +127,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
final Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
- final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
+ final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
bit1.run();
bit2.run();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java
index 302d0e5..4f0fcbf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java
@@ -23,14 +23,12 @@ import com.google.common.collect.Lists;
import mockit.NonStrictExpectations;
import org.apache.drill.DrillTestWrapper;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.impl.BatchCreator;
import org.apache.drill.exec.physical.impl.ScanBatch;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.rpc.NamedThreadFactory;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
@@ -54,7 +52,6 @@ import java.util.concurrent.Executors;
import static org.apache.drill.exec.physical.base.AbstractBase.INIT_ALLOCATION;
import static org.apache.drill.exec.physical.base.AbstractBase.MAX_ALLOCATION;
-import static org.apache.drill.exec.physical.unit.TestMiniPlan.fs;
/**
* A MiniPlanUnitTestBase extends PhysicalOpUnitTestBase, to construct MiniPlan (aka plan fragment).
@@ -112,7 +109,7 @@ public class MiniPlanUnitTestBase extends PhysicalOpUnitTestBase {
"Must supply the same number of baseline values as columns in expected schema.");
for (MaterializedField field : expectedSchema) {
- ret.put(SchemaPath.getSimplePath(field.getPath()).toExpr(), baselineValues[i]);
+ ret.put(SchemaPath.getSimplePath(field.getName()).toExpr(), baselineValues[i]);
i++;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java
index 3380a52..0ea552f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,9 +17,9 @@
*/
package org.apache.drill.exec.record;
-import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
+
import static org.junit.Assert.assertTrue;
import org.junit.Before;
@@ -71,7 +71,7 @@ public class TestMaterializedField {
final MaterializedField clone = field.withPathAndType(path, type);
- final boolean isPathEqual = path.equals(clone.getPath());
+ final boolean isPathEqual = path.equals(clone.getName());
assertTrue("Cloned path does not match the original", isPathEqual);
final boolean isTypeEqual = type.equals(clone.getType());
@@ -83,5 +83,4 @@ public class TestMaterializedField {
}
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
index 8ac7c45..0dc4149 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -24,8 +24,6 @@ import io.netty.buffer.DrillBuf;
import java.util.List;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
@@ -68,14 +66,14 @@ public class TestLoad extends ExecTest {
final RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
final ByteBuf[] byteBufs = writableBatch.getBuffers();
int bytes = 0;
- for (int i = 0; i < byteBufs.length; i++) {
- bytes += byteBufs[i].writerIndex();
+ for (ByteBuf buf : byteBufs) {
+ bytes += buf.writerIndex();
}
final DrillBuf byteBuf = allocator.buffer(bytes);
int index = 0;
- for (int i = 0; i < byteBufs.length; i++) {
- byteBufs[i].readBytes(byteBuf, index, byteBufs[i].writerIndex());
- index += byteBufs[i].writerIndex();
+ for (ByteBuf buf : byteBufs) {
+ buf.readBytes(byteBuf, index, buf.writerIndex());
+ index += buf.writerIndex();
}
byteBuf.writerIndex(bytes);
@@ -88,7 +86,7 @@ public class TestLoad extends ExecTest {
} else {
System.out.print("\t");
}
- System.out.print(v.getField().getPath());
+ System.out.print(v.getField().getName());
System.out.print("[");
System.out.print(v.getField().getType().getMinorType());
System.out.print("]");
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
index 0c5ff49..4e553de 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -66,7 +66,7 @@ public class TestOutputMutator implements OutputMutator, Iterable<VectorWrapper<
List<ValueVector> vectors = Lists.newArrayList();
for (VectorWrapper w : container) {
ValueVector vector = w.getValueVector();
- if (vector.getField().getPath().equals(schemaPath)) {
+ if (vector.getField().getName().equals(schemaPath.getRootSegmentPath())) {
vectors.add(newVector);
} else {
vectors.add(w.getValueVector());
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
index 8b9cd28..bd3a9e8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,7 +14,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- ******************************************************************************/
+ */
package org.apache.drill.exec.store.parquet;
import static org.junit.Assert.assertEquals;
@@ -126,12 +126,12 @@ public class ParquetResultListener implements UserResultsListener {
for (final VectorWrapper vw : batchLoader) {
final ValueVector vv = vw.getValueVector();
- currentField = props.fields.get(vv.getField().getPath());
- if (!valuesChecked.containsKey(vv.getField().getPath())) {
- valuesChecked.put(vv.getField().getPath(), 0);
+ currentField = props.fields.get(vv.getField().getName());
+ if (!valuesChecked.containsKey(vv.getField().getName())) {
+ valuesChecked.put(vv.getField().getName(), 0);
columnValCounter = 0;
} else {
- columnValCounter = valuesChecked.get(vv.getField().getPath());
+ columnValCounter = valuesChecked.get(vv.getField().getName());
}
printColumnMajor(vv);
@@ -145,9 +145,9 @@ public class ParquetResultListener implements UserResultsListener {
columnValCounter += vv.getAccessor().getValueCount();
}
- valuesChecked.remove(vv.getField().getPath());
+ valuesChecked.remove(vv.getField().getName());
assertEquals("Mismatched value count for vectors in the same batch.", valueCount, vv.getAccessor().getValueCount());
- valuesChecked.put(vv.getField().getPath(), columnValCounter);
+ valuesChecked.put(vv.getField().getName(), columnValCounter);
}
if (ParquetRecordReaderTest.VERBOSE_DEBUG){
@@ -184,7 +184,7 @@ public class ParquetResultListener implements UserResultsListener {
public void printColumnMajor(ValueVector vv) {
if (ParquetRecordReaderTest.VERBOSE_DEBUG){
- System.out.println("\n" + vv.getField().getPath());
+ System.out.println("\n" + vv.getField().getName());
}
for (int j = 0; j < vv.getAccessor().getValueCount(); j++) {
if (ParquetRecordReaderTest.VERBOSE_DEBUG){
@@ -211,7 +211,7 @@ public class ParquetResultListener implements UserResultsListener {
System.out.println();
for (VectorWrapper vw : batchLoader) {
ValueVector v = vw.getValueVector();
- System.out.print(Strings.padStart(v.getField().getPath(), 20, ' ') + " ");
+ System.out.print(Strings.padStart(v.getField().getName(), 20, ' ') + " ");
}
System.out.println();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
index 6397ef7..301374f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,7 +18,6 @@
package org.apache.drill.exec.store.parquet;
import org.apache.drill.BaseTestQuery;
-import org.apache.drill.exec.proto.UserBitShared;
import org.junit.Test;
public class TestParquetComplex extends BaseTestQuery {
@@ -180,8 +179,11 @@ public class TestParquetComplex extends BaseTestQuery {
@Test //DRILL-3533
public void notxistsField() throws Exception {
- String query = String.format("select t.`marketing_info`.notexists as notexists, t.`marketing_info`.camp_id as id from %s t", DATAFILE);
- String[] columns = {"notexists", "id"};
+ String query = String.format("select t.`marketing_info`.notexists as notexists1,\n" +
+ "t.`marketing_info`.camp_id as id,\n" +
+ "t.`marketing_info.camp_id` as notexists2\n" +
+ "from %s t", DATAFILE);
+ String[] columns = {"notexists1", "id", "notexists2"};
testBuilder()
.sqlQuery(query)
.unOrdered()
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
index b6f1408..7578476 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import com.google.common.io.Resources;
import mockit.Mock;
import mockit.MockUp;
import mockit.integration.junit4.JMockit;
@@ -465,36 +466,43 @@ public class TestParquetMetadataCache extends PlanTestBase {
}
@Test
- public void testMetadataCacheAbsolutePaths() throws Exception {
- final String absolutePathsMetadata = "absolute_paths_metadata";
- try {
- test("use dfs_test.tmp");
- // creating two inner directories to leverage METADATA_DIRECTORIES_FILENAME metadata file as well
- final String absolutePathsMetadataT1 = absolutePathsMetadata + "/t1";
- final String absolutePathsMetadataT2 = absolutePathsMetadata + "/t2";
- test("create table `%s` as select * from cp.`tpch/nation.parquet`", absolutePathsMetadataT1);
- test("create table `%s` as select * from cp.`tpch/nation.parquet`", absolutePathsMetadataT2);
- copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" +
- "metadata_directories_with_absolute_paths.requires_replace.txt", absolutePathsMetadata, Metadata.METADATA_DIRECTORIES_FILENAME);
- copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" +
- "metadata_table_with_absolute_paths.requires_replace.txt", absolutePathsMetadata, Metadata.METADATA_FILENAME);
- copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" +
- "metadata_table_with_absolute_paths_t1.requires_replace.txt", absolutePathsMetadataT1, Metadata.METADATA_FILENAME);
- copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" +
- "metadata_table_with_absolute_paths_t2.requires_replace.txt", absolutePathsMetadataT2, Metadata.METADATA_FILENAME);
- String query = String.format("select * from %s", absolutePathsMetadata);
- int expectedRowCount = 50;
- int expectedNumFiles = 1; // point to selectionRoot since no pruning is done in this query
- int actualRowCount = testSql(query);
- assertEquals("An incorrect result was obtained while querying a table with metadata cache files",
- expectedRowCount, actualRowCount);
- String numFilesPattern = "numFiles=" + expectedNumFiles;
- String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), absolutePathsMetadata);
- PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
- new String[] {"Filter"});
- } finally {
- test("drop table if exists %s", absolutePathsMetadata);
+ public void testOldMetadataVersions() throws Exception {
+ final String tablePath = "absolute_paths_metadata";
+ String rootMetadataPath = new Path("parquet", "metadata_files_with_old_versions").toUri().getPath();
+ // gets folders with different metadata cache versions
+ String[] metadataPaths = new File(Resources.getResource(rootMetadataPath).getFile()).list();
+ for (String metadataPath : metadataPaths) {
+ try {
+ test("use dfs_test.tmp");
+ // creating two inner directories to leverage METADATA_DIRECTORIES_FILENAME metadata file as well
+ final String absolutePathsMetadataT1 = new Path(tablePath, "t1").toUri().getPath();
+ final String absolutePathsMetadataT2 = new Path(tablePath, "t2").toUri().getPath();
+ String createQuery = "create table `%s` as select * from cp.`tpch/nation.parquet`";
+ test(createQuery, absolutePathsMetadataT1);
+ test(createQuery, absolutePathsMetadataT2);
+ Path relativePath = new Path(rootMetadataPath, metadataPath);
+ copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_directories.requires_replace.txt"),
+ tablePath, Metadata.METADATA_DIRECTORIES_FILENAME);
+ copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table.requires_replace.txt"),
+ tablePath, Metadata.METADATA_FILENAME);
+ copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table_t1.requires_replace.txt"),
+ absolutePathsMetadataT1, Metadata.METADATA_FILENAME);
+ copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table_t2.requires_replace.txt"),
+ absolutePathsMetadataT2, Metadata.METADATA_FILENAME);
+ String query = String.format("select * from %s", tablePath);
+ int expectedRowCount = 50;
+ int expectedNumFiles = 1; // point to selectionRoot since no pruning is done in this query
+ int actualRowCount = testSql(query);
+ assertEquals("An incorrect result was obtained while querying a table with metadata cache files",
+ expectedRowCount, actualRowCount);
+ String numFilesPattern = "numFiles=" + expectedNumFiles;
+ String usedMetaPattern = "usedMetadataFile=true";
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tablePath);
+ PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
+ new String[]{"Filter"});
+ } finally {
+ test("drop table if exists %s", tablePath);
+ }
}
}
@@ -681,6 +689,31 @@ public class TestParquetMetadataCache extends PlanTestBase {
}
}
+ @Test // DRILL-4264
+ public void testMetadataCacheFieldWithDots() throws Exception {
+ final String tableWithDots = "dfs_test.tmp.`complex_table`";
+ try {
+ test("create table %s as\n" +
+ "select cast(1 as int) as `column.with.dots`, t.`column`.`with.dots`\n" +
+ "from cp.`store/parquet/complex/complex.parquet` t limit 1", tableWithDots);
+
+ String query = String.format("select * from %s", tableWithDots);
+ int expectedRowCount = 1;
+
+ int actualRowCount = testSql(query);
+ assertEquals("Row count does not match the expected value", expectedRowCount, actualRowCount);
+ PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=false"}, null);
+
+ test("refresh table metadata %s", tableWithDots);
+
+ actualRowCount = testSql(query);
+ assertEquals("Row count does not match the expected value", expectedRowCount, actualRowCount);
+ PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=true"}, null);
+ } finally {
+ test(String.format("drop table if exists %s", tableWithDots));
+ }
+ }
+
/**
* Helper method for checking the metadata file existence
*
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
index 8714b30..e023ecb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -54,7 +54,7 @@ public class TestParquetPhysicalPlan extends ExecTest {
RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
DrillConfig config = DrillConfig.create();
- try (Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) {
+ try (Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
bit1.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
@@ -65,7 +65,7 @@ public class TestParquetPhysicalPlan extends ExecTest {
count += b.getHeader().getRowCount();
loader.load(b.getHeader().getDef(), b.getData());
for (VectorWrapper vw : loader) {
- System.out.print(vw.getValueVector().getField().getPath() + ": ");
+ System.out.print(vw.getValueVector().getField().getName() + ": ");
ValueVector vv = vw.getValueVector();
for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
Object o = vv.getAccessor().getObject(i);
@@ -124,7 +124,7 @@ public class TestParquetPhysicalPlan extends ExecTest {
public void testParseParquetPhysicalPlanRemote() throws Exception {
DrillConfig config = DrillConfig.create();
- try(DrillClient client = new DrillClient(config);) {
+ try (DrillClient client = new DrillClient(config)) {
client.connect();
ParquetResultsListener listener = new ParquetResultsListener();
Stopwatch watch = Stopwatch.createStarted();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index 7c0b345..9064c5c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -740,4 +740,33 @@ public class TestJsonReader extends BaseTestQuery {
org.apache.commons.io.FileUtils.deleteQuietly(directory);
}
}
+
+ @Test // DRILL-4264
+ public void testFieldWithDots() throws Exception {
+ File directory = new File(BaseTestQuery.getTempDir("json/input"));
+ try {
+ directory.mkdirs();
+ String fileName = "table.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
+ writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
+ }
+
+ String query = String.format("select t.m.`a.b` as a,\n" +
+ "t.m.a.b as b,\n" +
+ "t.m['a.b'] as c,\n" +
+ "t.rk.q as d,\n" +
+ "t.`rk.q` as e\n" +
+ "from dfs_test.`%s/%s` t",
+ directory.toPath().toString(), fileName);
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("a", "b", "c", "d", "e")
+ .baselineValues("1", "2", "1", null, "a")
+ .go();
+
+ } finally {
+ org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java
index 601abb1..42a7e63 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java
@@ -73,7 +73,7 @@ public class RowSetPrinter {
if (i > 0) {
out.print(", ");
}
- out.print(schema.column(i).getLastName());
+ out.print(schema.column(i).getName());
}
out.println();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
index 03417ff..af35cdf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
@@ -50,46 +50,46 @@ public class RowSetTest extends SubOperatorTest {
* Test a simple physical schema with no maps.
*/
- @Test
- public void testSchema() {
- BatchSchema batchSchema = new SchemaBuilder()
- .add("c", MinorType.INT)
- .add("a", MinorType.INT, DataMode.REPEATED)
- .addNullable("b", MinorType.VARCHAR)
- .build();
-
- assertEquals("c", batchSchema.getColumn(0).getName());
- assertEquals("a", batchSchema.getColumn(1).getName());
- assertEquals("b", batchSchema.getColumn(2).getName());
-
- RowSetSchema schema = new RowSetSchema(batchSchema);
- TupleSchema access = schema.hierarchicalAccess();
- assertEquals(3, access.count());
-
- crossCheck(access, 0, "c", MinorType.INT);
- assertEquals(DataMode.REQUIRED, access.column(0).getDataMode());
- assertEquals(DataMode.REQUIRED, access.column(0).getType().getMode());
- assertTrue(! access.column(0).isNullable());
-
- crossCheck(access, 1, "a", MinorType.INT);
- assertEquals(DataMode.REPEATED, access.column(1).getDataMode());
- assertEquals(DataMode.REPEATED, access.column(1).getType().getMode());
- assertTrue(! access.column(1).isNullable());
-
- crossCheck(access, 2, "b", MinorType.VARCHAR);
- assertEquals(MinorType.VARCHAR, access.column(2).getType().getMinorType());
- assertEquals(DataMode.OPTIONAL, access.column(2).getDataMode());
- assertEquals(DataMode.OPTIONAL, access.column(2).getType().getMode());
- assertTrue(access.column(2).isNullable());
-
- // No maps: physical schema is the same as access schema.
-
- PhysicalSchema physical = schema.physical();
- assertEquals(3, physical.count());
- assertEquals("c", physical.column(0).field().getName());
- assertEquals("a", physical.column(1).field().getName());
- assertEquals("b", physical.column(2).field().getName());
- }
+// @Test
+// public void testSchema() {
+// BatchSchema batchSchema = new SchemaBuilder()
+// .add("c", MinorType.INT)
+// .add("a", MinorType.INT, DataMode.REPEATED)
+// .addNullable("b", MinorType.VARCHAR)
+// .build();
+//
+// assertEquals("c", batchSchema.getColumn(0).getName());
+// assertEquals("a", batchSchema.getColumn(1).getName());
+// assertEquals("b", batchSchema.getColumn(2).getName());
+//
+// RowSetSchema schema = new RowSetSchema(batchSchema);
+// TupleSchema access = schema.hierarchicalAccess();
+// assertEquals(3, access.count());
+//
+// crossCheck(access, 0, "c", MinorType.INT);
+// assertEquals(DataMode.REQUIRED, access.column(0).getDataMode());
+// assertEquals(DataMode.REQUIRED, access.column(0).getType().getMode());
+// assertTrue(! access.column(0).isNullable());
+//
+// crossCheck(access, 1, "a", MinorType.INT);
+// assertEquals(DataMode.REPEATED, access.column(1).getDataMode());
+// assertEquals(DataMode.REPEATED, access.column(1).getType().getMode());
+// assertTrue(! access.column(1).isNullable());
+//
+// crossCheck(access, 2, "b", MinorType.VARCHAR);
+// assertEquals(MinorType.VARCHAR, access.column(2).getType().getMinorType());
+// assertEquals(DataMode.OPTIONAL, access.column(2).getDataMode());
+// assertEquals(DataMode.OPTIONAL, access.column(2).getType().getMode());
+// assertTrue(access.column(2).isNullable());
+//
+// // No maps: physical schema is the same as access schema.
+//
+// PhysicalSchema physical = schema.physical();
+// assertEquals(3, physical.count());
+// assertEquals("c", physical.column(0).field().getName());
+// assertEquals("a", physical.column(1).field().getName());
+// assertEquals("b", physical.column(2).field().getName());
+// }
/**
* Validate that the actual column metadata is as expected by
@@ -102,89 +102,89 @@ public class RowSetTest extends SubOperatorTest {
* @param type expected type
*/
- public void crossCheck(TupleSchema schema, int index, String fullName, MinorType type) {
- String name = null;
- for (String part : Splitter.on(".").split(fullName)) {
- name = part;
- }
- assertEquals(name, schema.column(index).getName());
- assertEquals(index, schema.columnIndex(fullName));
- assertSame(schema.column(index), schema.column(fullName));
- assertEquals(type, schema.column(index).getType().getMinorType());
- }
+// public void crossCheck(TupleSchema schema, int index, String fullName, MinorType type) {
+// String name = null;
+// for (String part : Splitter.on(".").split(fullName)) {
+// name = part;
+// }
+// assertEquals(name, schema.column(index).getName());
+// assertEquals(index, schema.columnIndex(fullName));
+// assertSame(schema.column(index), schema.column(fullName));
+// assertEquals(type, schema.column(index).getType().getMinorType());
+// }
/**
* Verify that a nested map schema works as expected.
*/
- @Test
- public void testMapSchema() {
- BatchSchema batchSchema = new SchemaBuilder()
- .add("c", MinorType.INT)
- .addMap("a")
- .addNullable("b", MinorType.VARCHAR)
- .add("d", MinorType.INT)
- .addMap("e")
- .add("f", MinorType.VARCHAR)
- .buildMap()
- .add("g", MinorType.INT)
- .buildMap()
- .add("h", MinorType.BIGINT)
- .build();
-
- RowSetSchema schema = new RowSetSchema(batchSchema);
-
- // Access schema: flattened with maps removed
-
- FlattenedSchema access = schema.flatAccess();
- assertEquals(6, access.count());
- crossCheck(access, 0, "c", MinorType.INT);
- crossCheck(access, 1, "a.b", MinorType.VARCHAR);
- crossCheck(access, 2, "a.d", MinorType.INT);
- crossCheck(access, 3, "a.e.f", MinorType.VARCHAR);
- crossCheck(access, 4, "a.g", MinorType.INT);
- crossCheck(access, 5, "h", MinorType.BIGINT);
-
- // Should have two maps.
-
- assertEquals(2, access.mapCount());
- assertEquals("a", access.map(0).getName());
- assertEquals("e", access.map(1).getName());
- assertEquals(0, access.mapIndex("a"));
- assertEquals(1, access.mapIndex("a.e"));
-
- // Verify physical schema: should mirror the schema created above.
-
- PhysicalSchema physical = schema.physical();
- assertEquals(3, physical.count());
- assertEquals("c", physical.column(0).field().getName());
- assertEquals("c", physical.column(0).fullName());
- assertFalse(physical.column(0).isMap());
- assertNull(physical.column(0).mapSchema());
-
- assertEquals("a", physical.column(1).field().getName());
- assertEquals("a", physical.column(1).fullName());
- assertTrue(physical.column(1).isMap());
- assertNotNull(physical.column(1).mapSchema());
-
- assertEquals("h", physical.column(2).field().getName());
- assertEquals("h", physical.column(2).fullName());
- assertFalse(physical.column(2).isMap());
- assertNull(physical.column(2).mapSchema());
-
- PhysicalSchema aSchema = physical.column(1).mapSchema();
- assertEquals(4, aSchema.count());
- assertEquals("b", aSchema.column(0).field().getName());
- assertEquals("a.b", aSchema.column(0).fullName());
- assertEquals("d", aSchema.column(1).field().getName());
- assertEquals("e", aSchema.column(2).field().getName());
- assertEquals("g", aSchema.column(3).field().getName());
-
- PhysicalSchema eSchema = aSchema.column(2).mapSchema();
- assertEquals(1, eSchema.count());
- assertEquals("f", eSchema.column(0).field().getName());
- assertEquals("a.e.f", eSchema.column(0).fullName());
- }
+// @Test
+// public void testMapSchema() {
+// BatchSchema batchSchema = new SchemaBuilder()
+// .add("c", MinorType.INT)
+// .addMap("a")
+// .addNullable("b", MinorType.VARCHAR)
+// .add("d", MinorType.INT)
+// .addMap("e")
+// .add("f", MinorType.VARCHAR)
+// .buildMap()
+// .add("g", MinorType.INT)
+// .buildMap()
+// .add("h", MinorType.BIGINT)
+// .build();
+//
+// RowSetSchema schema = new RowSetSchema(batchSchema);
+//
+// // Access schema: flattened with maps removed
+//
+// FlattenedSchema access = schema.flatAccess();
+// assertEquals(6, access.count());
+// crossCheck(access, 0, "c", MinorType.INT);
+// crossCheck(access, 1, "a.b", MinorType.VARCHAR);
+// crossCheck(access, 2, "a.d", MinorType.INT);
+// crossCheck(access, 3, "a.e.f", MinorType.VARCHAR);
+// crossCheck(access, 4, "a.g", MinorType.INT);
+// crossCheck(access, 5, "h", MinorType.BIGINT);
+//
+// // Should have two maps.
+//
+// assertEquals(2, access.mapCount());
+// assertEquals("a", access.map(0).getName());
+// assertEquals("e", access.map(1).getName());
+// assertEquals(0, access.mapIndex("a"));
+// assertEquals(1, access.mapIndex("a.e"));
+//
+// // Verify physical schema: should mirror the schema created above.
+//
+// PhysicalSchema physical = schema.physical();
+// assertEquals(3, physical.count());
+// assertEquals("c", physical.column(0).field().getName());
+// assertEquals("c", physical.column(0).fullName());
+// assertFalse(physical.column(0).isMap());
+// assertNull(physical.column(0).mapSchema());
+//
+// assertEquals("a", physical.column(1).field().getName());
+// assertEquals("a", physical.column(1).fullName());
+// assertTrue(physical.column(1).isMap());
+// assertNotNull(physical.column(1).mapSchema());
+//
+// assertEquals("h", physical.column(2).field().getName());
+// assertEquals("h", physical.column(2).fullName());
+// assertFalse(physical.column(2).isMap());
+// assertNull(physical.column(2).mapSchema());
+//
+// PhysicalSchema aSchema = physical.column(1).mapSchema();
+// assertEquals(4, aSchema.count());
+// assertEquals("b", aSchema.column(0).field().getName());
+// assertEquals("a.b", aSchema.column(0).fullName());
+// assertEquals("d", aSchema.column(1).field().getName());
+// assertEquals("e", aSchema.column(2).field().getName());
+// assertEquals("g", aSchema.column(3).field().getName());
+//
+// PhysicalSchema eSchema = aSchema.column(2).mapSchema();
+// assertEquals(1, eSchema.count());
+// assertEquals("f", eSchema.column(0).field().getName());
+// assertEquals("a.e.f", eSchema.column(0).fullName());
+// }
/**
* Verify that simple scalar (non-repeated) column readers
@@ -348,33 +348,33 @@ public class RowSetTest extends SubOperatorTest {
* Map fields are flattened into a logical schema.
*/
- @Test
- public void testMap() {
- BatchSchema batchSchema = new SchemaBuilder()
- .add("a", MinorType.INT)
- .addMap("b")
- .add("c", MinorType.INT)
- .add("d", MinorType.INT)
- .buildMap()
- .build();
- SingleRowSet rs = fixture.rowSetBuilder(batchSchema)
- .add(10, 20, 30)
- .add(40, 50, 60)
- .build();
- RowSetReader reader = rs.reader();
- assertTrue(reader.next());
- assertEquals(10, reader.column(0).getInt());
- assertEquals(20, reader.column(1).getInt());
- assertEquals(30, reader.column(2).getInt());
- assertEquals(10, reader.column("a").getInt());
- assertEquals(30, reader.column("b.d").getInt());
- assertTrue(reader.next());
- assertEquals(40, reader.column(0).getInt());
- assertEquals(50, reader.column(1).getInt());
- assertEquals(60, reader.column(2).getInt());
- assertFalse(reader.next());
- rs.clear();
- }
+// @Test
+// public void testMap() {
+// BatchSchema batchSchema = new SchemaBuilder()
+// .add("a", MinorType.INT)
+// .addMap("b")
+// .add("c", MinorType.INT)
+// .add("d", MinorType.INT)
+// .buildMap()
+// .build();
+// SingleRowSet rs = fixture.rowSetBuilder(batchSchema)
+// .add(10, 20, 30)
+// .add(40, 50, 60)
+// .build();
+// RowSetReader reader = rs.reader();
+// assertTrue(reader.next());
+// assertEquals(10, reader.column(0).getInt());
+// assertEquals(20, reader.column(1).getInt());
+// assertEquals(30, reader.column(2).getInt());
+// assertEquals(10, reader.column("a").getInt());
+// assertEquals(30, reader.column("b.d").getInt());
+// assertTrue(reader.next());
+// assertEquals(40, reader.column(0).getInt());
+// assertEquals(50, reader.column(1).getInt());
+// assertEquals(60, reader.column(2).getInt());
+// assertFalse(reader.next());
+// rs.clear();
+// }
/**
* Test an array of ints (as an example fixed-width type)
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_directories.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_directories.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_directories.requires_replace.txt
new file mode 100644
index 0000000..8a9989d
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_directories.requires_replace.txt
@@ -0,0 +1,3 @@
+{
+ "directories" : [ "file:REPLACED_IN_TEST/absolute_paths_metadata/t1", "file:REPLACED_IN_TEST/absolute_paths_metadata/t2" ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table.requires_replace.txt
new file mode 100644
index 0000000..e3734f3
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table.requires_replace.txt
@@ -0,0 +1,108 @@
+{
+ "metadata_version" : "v3",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t1/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ }, {
+ "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t2/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ "file:REPLACED_IN_TEST/absolute_paths_metadata/t1", "file:REPLACED_IN_TEST/absolute_paths_metadata/t2" ],
+ "drillVersion" : "1.11.0"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t1.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t1.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t1.requires_replace.txt
new file mode 100644
index 0000000..62a8c80
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t1.requires_replace.txt
@@ -0,0 +1,76 @@
+{
+ "metadata_version" : "v3",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t1/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ ],
+ "drillVersion" : "1.11.0"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t2.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t2.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t2.requires_replace.txt
new file mode 100644
index 0000000..b70c8fa
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t2.requires_replace.txt
@@ -0,0 +1,76 @@
+{
+ "metadata_version" : "v3",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t2/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ ],
+ "drillVersion" : "1.11.0"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_directories.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_directories.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_directories.requires_replace.txt
new file mode 100644
index 0000000..ad40340
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_directories.requires_replace.txt
@@ -0,0 +1,3 @@
+{
+ "directories" : [ "t2", "t1" ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table.requires_replace.txt
new file mode 100644
index 0000000..6c84901
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table.requires_replace.txt
@@ -0,0 +1,108 @@
+{
+ "metadata_version" : "3.1",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "t2/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ }, {
+ "path" : "t1/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ "t2", "t1" ],
+ "drillVersion" : "1.12.0-SNAPSHOT"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t1.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t1.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t1.requires_replace.txt
new file mode 100644
index 0000000..b611d13
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t1.requires_replace.txt
@@ -0,0 +1,76 @@
+{
+ "metadata_version" : "3.1",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ ],
+ "drillVersion" : "1.12.0-SNAPSHOT"
+}
\ No newline at end of file
[3/4] drill git commit: DRILL-4264: Allow field names to include dots
Posted by am...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
index f7fd1d6..e5ba98f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
@@ -293,7 +293,8 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
continue;
}
keyExprs[i] = expr;
- final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
+ final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(),
+ expr.getMajorType());
@SuppressWarnings("resource")
final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
keyOutputIds[i] = container.add(vector);
@@ -309,7 +310,8 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
continue;
}
- final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
+ final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(),
+ expr.getMajorType());
@SuppressWarnings("resource")
ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
TypedFieldId id = container.add(vector);
@@ -425,17 +427,17 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
case FOUR_BYTE: {
JVar var = g.declareClassField("sv4_", g.getModel()._ref(SelectionVector4.class));
g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector4"));
- g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
+ g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));
return;
}
case NONE: {
- g.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));;
+ g.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));
return;
}
case TWO_BYTE: {
JVar var = g.declareClassField("sv2_", g.getModel()._ref(SelectionVector2.class));
g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector2"));
- g.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));;
+ g.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));
return;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
index 436480e..387dad1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -196,7 +196,8 @@ public class ChainedHashTable {
*/
for (NamedExpression ne : htConfig.getKeyExprsBuild()) {
LogicalExpression expr = keyExprsBuild[i];
- final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType());
+ final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(),
+ expr.getMajorType());
@SuppressWarnings("resource")
ValueVector vv = TypeHelper.getNewVector(outputField, allocator);
htKeyFieldIds[i] = htContainerOrig.add(vv);
@@ -318,8 +319,7 @@ public class ChainedHashTable {
* aggregate. For join we need to hash everything as double (both for distribution and for comparison) but
* for aggregation we can avoid the penalty of casting to double
*/
- LogicalExpression hashExpression = HashPrelUtil.getHashExpression(Arrays.asList(keyExprs),
- incomingProbe != null ? true : false);
+ LogicalExpression hashExpression = HashPrelUtil.getHashExpression(Arrays.asList(keyExprs), incomingProbe != null);
final LogicalExpression materializedExpr = ExpressionTreeMaterializer.materializeAndCheckErrors(hashExpression, batch, context.getFunctionRegistry());
HoldingContainer hash = cg.addExpr(materializedExpr);
cg.getEvalBlock()._return(hash.getValue());
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 8fd9441..e64e919 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -253,8 +253,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
}
private FieldReference getRef(NamedExpression e) {
- final FieldReference ref = e.getRef();
- return ref;
+ return e.getRef();
}
/**
@@ -333,8 +332,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
ClassifierResult result = new ClassifierResult();
- for (int i = 0; i < exprs.size(); i++) {
- final NamedExpression namedExpression = exprs.get(i);
+ for (NamedExpression namedExpression : exprs) {
result.clear();
String outputName = getRef(namedExpression).getRootSegment().getPath();
@@ -408,10 +406,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
List<NamedExpression> exprs = Lists.newArrayList();
for (MaterializedField field : incoming.getSchema()) {
- if (field.getPath().equals(popConfig.getColumn().getAsUnescapedPath())) {
+ String fieldName = field.getName();
+ if (fieldName.equals(popConfig.getColumn().getRootSegmentPath())) {
continue;
}
- exprs.add(new NamedExpression(SchemaPath.getSimplePath(field.getPath()), new FieldReference(field.getPath())));
+ exprs.add(new NamedExpression(SchemaPath.getSimplePath(fieldName), new FieldReference(fieldName)));
}
return exprs;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 72b8833..1f74ba1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -477,7 +477,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
outputType = inputType;
}
- final ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getPath(), outputType));
+ final ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getName(), outputType));
if (v instanceof AbstractContainerVector) {
vv.getValueVector().makeTransferPair(v);
v.clear();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index c351517..e599702 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -400,7 +400,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
} else {
outputType = inputType;
}
- MaterializedField newField = MaterializedField.create(w.getField().getPath(), outputType);
+ MaterializedField newField = MaterializedField.create(w.getField().getName(), outputType);
ValueVector v = container.addOrGet(newField);
if (v instanceof AbstractContainerVector) {
w.getValueVector().makeTransferPair(v);
@@ -417,7 +417,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
} else {
outputType = inputType;
}
- MaterializedField newField = MaterializedField.create(w.getField().getPath(), outputType);
+ MaterializedField newField = MaterializedField.create(w.getField().getName(), outputType);
ValueVector v = container.addOrGet(newField);
if (v instanceof AbstractContainerVector) {
w.getValueVector().makeTransferPair(v);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
index 8336e86..35cc710 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
@@ -317,7 +317,7 @@ public class NestedLoopJoinBatch extends AbstractRecordBatch<NestedLoopJoinPOP>
outputType = inputType;
}
- MaterializedField newField = MaterializedField.create(field.getPath(), outputType);
+ MaterializedField newField = MaterializedField.create(field.getName(), outputType);
container.addOrGet(newField);
JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer",
@@ -384,7 +384,7 @@ public class NestedLoopJoinBatch extends AbstractRecordBatch<NestedLoopJoinPOP>
} else {
outputType = inputType;
}
- MaterializedField newField = MaterializedField.create(vectorWrapper.getField().getPath(), outputType);
+ MaterializedField newField = MaterializedField.create(vectorWrapper.getField().getName(), outputType);
ValueVector valueVector = container.addOrGet(newField);
if (valueVector instanceof AbstractContainerVector) {
vectorWrapper.getValueVector().makeTransferPair(valueVector);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index fede487..63133d4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -431,11 +431,10 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
int i = 0;
for (Ordering od : orderings) {
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), incoming, collector, context.getFunctionRegistry());
- SchemaPath schemaPath = SchemaPath.getSimplePath("f" + i++);
TypeProtos.MajorType.Builder builder = TypeProtos.MajorType.newBuilder().mergeFrom(expr.getMajorType())
.clearMode().setMode(TypeProtos.DataMode.REQUIRED);
TypeProtos.MajorType newType = builder.build();
- MaterializedField outputField = MaterializedField.create(schemaPath.getAsUnescapedPath(), newType);
+ MaterializedField outputField = MaterializedField.create("f" + i++, newType);
if (collector.hasErrors()) {
throw new SchemaChangeException(String.format(
"Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
@@ -533,7 +532,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
// if this now that all the batches on the queue are processed, we begin processing the incoming batches. For the
// first one
// we need to generate a new schema, even if the outcome is IterOutcome.OK After that we can reuse the schema.
- if (this.startedUnsampledBatches == false) {
+ if (!this.startedUnsampledBatches) {
this.startedUnsampledBatches = true;
if (upstream == IterOutcome.OK) {
upstream = IterOutcome.OK_NEW_SCHEMA;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
index 589754f..3afa852 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -99,9 +99,9 @@ public class ProducerConsumerBatch extends AbstractRecordBatch<ProducerConsumer>
final MaterializedField field = schema.getColumn(i);
final MajorType type = field.getType();
final ValueVector vOut = container.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()),
- container.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds()).getValueVector();
+ container.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds()).getValueVector();
final ValueVector vIn = newContainer.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()),
- newContainer.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds()).getValueVector();
+ newContainer.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds()).getValueVector();
final TransferPair tp = vIn.makeTransferPair(vOut);
tp.transfer();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 6baf070..9a72fcb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -202,7 +202,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
this.recordCount = remainderIndex;
} else {
setValueCount(incomingRecordCount);
- for(final VectorWrapper<?> v: incoming) {
+ for (final VectorWrapper<?> v: incoming) {
v.clear();
}
this.recordCount = outputRecords;
@@ -331,8 +331,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final ClassifierResult result = new ClassifierResult();
final boolean classify = isClassificationNeeded(exprs);
- for (int i = 0; i < exprs.size(); i++) {
- final NamedExpression namedExpression = exprs.get(i);
+ for (NamedExpression namedExpression : exprs) {
result.clear();
if (classify && namedExpression.getExpr() instanceof SchemaPath) {
@@ -341,15 +340,15 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
if (result.isStar) {
// The value indicates which wildcard we are processing now
final Integer value = result.prefixMap.get(result.prefix);
- if (value != null && value.intValue() == 1) {
+ if (value != null && value == 1) {
int k = 0;
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- if (k > result.outputNames.size()-1) {
+ if (k > result.outputNames.size() - 1) {
assert false;
}
final String name = result.outputNames.get(k++); // get the renamed column names
- if (name == EMPTY_STRING) {
+ if (name.isEmpty()) {
continue;
}
@@ -362,16 +361,16 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final TransferPair tp = vvIn.makeTransferPair(vvOut);
transfers.add(tp);
}
- } else if (value != null && value.intValue() > 1) { // subsequent wildcards should do a copy of incoming valuevectors
+ } else if (value != null && value > 1) { // subsequent wildcards should do a copy of incoming valuevectors
int k = 0;
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final SchemaPath originalPath = SchemaPath.getSimplePath(vvIn.getField().getPath());
- if (k > result.outputNames.size()-1) {
+ final SchemaPath originalPath = SchemaPath.getSimplePath(vvIn.getField().getName());
+ if (k > result.outputNames.size() - 1) {
assert false;
}
final String name = result.outputNames.get(k++); // get the renamed column names
- if (name == EMPTY_STRING) {
+ if (name.isEmpty()) {
continue;
}
@@ -379,7 +378,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
continue;
}
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry() );
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry());
if (collector.hasErrors()) {
throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
}
@@ -387,7 +386,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final MaterializedField outputField = MaterializedField.create(name, expr.getMajorType());
final ValueVector vv = container.addOrGet(outputField, callBack);
allocationVectors.add(vv);
- final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
+ final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName()));
final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
}
@@ -406,14 +405,14 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
boolean isMatched = false;
for (int j = 0; j < result.outputNames.size(); j++) {
- if (!result.outputNames.get(j).equals(EMPTY_STRING)) {
+ if (!result.outputNames.get(j).isEmpty()) {
outputName = result.outputNames.get(j);
isMatched = true;
break;
}
}
- if(!isMatched) {
+ if (!isMatched) {
continue;
}
}
@@ -437,7 +436,8 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
Preconditions.checkNotNull(incoming);
final FieldReference ref = getRef(namedExpression);
- final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getAsUnescapedPath(), vectorRead.getMajorType()), callBack);
+ final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getLastSegment().getNameSegment().getPath(),
+ vectorRead.getMajorType()), callBack);
final TransferPair tp = vvIn.makeTransferPair(vvOut);
transfers.add(tp);
transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
@@ -463,7 +463,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
// need to do evaluation.
final ValueVector vector = container.addOrGet(outputField, callBack);
allocationVectors.add(vector);
- final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
+ final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName()));
final boolean useSetSafe = !(vector instanceof FixedWidthVector);
final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe);
final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND);
@@ -510,17 +510,19 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final List<NamedExpression> exprs = Lists.newArrayList();
for (final MaterializedField field : incoming.getSchema()) {
+ String fieldName = field.getName();
if (Types.isComplex(field.getType()) || Types.isRepeated(field.getType())) {
- final LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON", SchemaPath.getSimplePath(field.getPath()), ExpressionPosition.UNKNOWN);
+ final LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON",
+ SchemaPath.getSimplePath(fieldName), ExpressionPosition.UNKNOWN);
final String castFuncName = CastFunctions.getCastFunc(MinorType.VARCHAR);
final List<LogicalExpression> castArgs = Lists.newArrayList();
castArgs.add(convertToJson); //input_expr
// implicitly casting to varchar, since we don't know actual source length, cast to undefined length, which will preserve source length
castArgs.add(new ValueExpressions.LongExpression(Types.MAX_VARCHAR_LENGTH, null));
final FunctionCall castCall = new FunctionCall(castFuncName, castArgs, ExpressionPosition.UNKNOWN);
- exprs.add(new NamedExpression(castCall, new FieldReference(field.getPath())));
+ exprs.add(new NamedExpression(castCall, new FieldReference(fieldName)));
} else {
- exprs.add(new NamedExpression(SchemaPath.getSimplePath(field.getPath()), new FieldReference(field.getPath())));
+ exprs.add(new NamedExpression(SchemaPath.getSimplePath(fieldName), new FieldReference(fieldName)));
}
}
return exprs;
@@ -528,8 +530,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
private boolean isClassificationNeeded(final List<NamedExpression> exprs) {
boolean needed = false;
- for (int i = 0; i < exprs.size(); i++) {
- final NamedExpression ex = exprs.get(i);
+ for (NamedExpression ex : exprs) {
if (!(ex.getExpr() instanceof SchemaPath)) {
continue;
}
@@ -570,7 +571,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
* @param origName the original input name of the column
* @param result the data structure to keep track of the used names and decide what output name should be
* to ensure uniqueness
- * @Param allowDupsWithRename if the original name has been used, is renaming allowed to ensure output name unique
+ * @param allowDupsWithRename if the original name has been used, is renaming allowed to ensure output name unique
*/
private void addToResultMaps(final String origName, final ClassifierResult result, final boolean allowDupsWithRename) {
String name = origName;
@@ -610,7 +611,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
boolean exprIsFirstWildcard = false;
if (exprContainsStar) {
result.isStar = true;
- final Integer value = (Integer) result.prefixMap.get(exprPrefix);
+ final Integer value = result.prefixMap.get(exprPrefix);
if (value == null) {
final Integer n = 1;
result.prefixMap.put(exprPrefix, n);
@@ -634,9 +635,9 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
assert(components.length == 2);
final String prefix = components[0];
result.outputNames = Lists.newArrayList();
- for(final VectorWrapper<?> wrapper : incoming) {
+ for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String name = vvIn.getField().getPath();
+ final String name = vvIn.getField().getName();
// add the prefix to the incoming column name
final String newName = prefix + StarColumnHelper.PREFIX_DELIMITER + name;
@@ -656,7 +657,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String incomingName = vvIn.getField().getPath();
+ final String incomingName = vvIn.getField().getName();
// get the prefix of the name
final String[] nameComponents = incomingName.split(StarColumnHelper.PREFIX_DELIMITER, 2);
// if incoming valuevector does not have a prefix, ignore it since this expression is not referencing it
@@ -666,10 +667,9 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
}
final String namePrefix = nameComponents[0];
if (exprPrefix.equalsIgnoreCase(namePrefix)) {
- final String newName = incomingName;
- if (!result.outputMap.containsKey(newName)) {
- result.outputNames.set(k, newName);
- result.outputMap.put(newName, newName);
+ if (!result.outputMap.containsKey(incomingName)) {
+ result.outputNames.set(k, incomingName);
+ result.outputMap.put(incomingName, incomingName);
}
}
k++;
@@ -679,7 +679,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
if (exprContainsStar) {
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String incomingName = vvIn.getField().getPath();
+ final String incomingName = vvIn.getField().getName();
if (refContainsStar) {
addToResultMaps(incomingName, result, true); // allow dups since this is likely top-level project
} else {
@@ -698,11 +698,11 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
}
// input is wildcard and it is not the first wildcard
- else if(exprIsStar) {
+ else if (exprIsStar) {
result.outputNames = Lists.newArrayList();
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String incomingName = vvIn.getField().getPath();
+ final String incomingName = vvIn.getField().getName();
addToResultMaps(incomingName, result, true); // allow dups since this is likely top-level project
}
}
@@ -723,7 +723,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String name = vvIn.getField().getPath();
+ final String name = vvIn.getField().getName();
final String[] components = name.split(StarColumnHelper.PREFIX_DELIMITER, 2);
if (components.length <= 1) {
k++;
@@ -760,7 +760,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
result.outputNames = Lists.newArrayList();
for (final VectorWrapper<?> wrapper : incoming) {
final ValueVector vvIn = wrapper.getValueVector();
- final String incomingName = vvIn.getField().getPath();
+ final String incomingName = vvIn.getField().getName();
if (expr.getPath().equalsIgnoreCase(incomingName)) { // case insensitive matching of field name.
final String newName = ref.getPath();
addToResultMaps(newName, result, true);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
index e6a0dd4..5afe66b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
@@ -103,7 +103,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
try {
IterOutcome upstream = unionAllInput.nextBatch();
logger.debug("Upstream of Union-All: {}", upstream);
- switch(upstream) {
+ switch (upstream) {
case NONE:
case OUT_OF_MEMORY:
case STOP:
@@ -114,7 +114,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
case OK:
IterOutcome workOutcome = doWork();
- if(workOutcome != IterOutcome.OK) {
+ if (workOutcome != IterOutcome.OK) {
return workOutcome;
} else {
return upstream;
@@ -164,9 +164,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
transfers.clear();
// If both sides of Union-All are empty
- if(unionAllInput.isBothSideEmpty()) {
- for(int i = 0; i < outputFields.size(); ++i) {
- final String colName = outputFields.get(i).getPath();
+ if (unionAllInput.isBothSideEmpty()) {
+ for (MaterializedField materializedField : outputFields) {
+ final String colName = materializedField.getName();
final MajorType majorType = MajorType.newBuilder()
.setMinorType(MinorType.INT)
.setMode(DataMode.OPTIONAL)
@@ -186,12 +186,12 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// Uncomment out this line to debug the generated code.
// cg.getCodeGenerator().saveCodeForDebugging(true);
int index = 0;
- for(VectorWrapper<?> vw : current) {
+ for (VectorWrapper<?> vw : current) {
ValueVector vvIn = vw.getValueVector();
// get the original input column names
- SchemaPath inputPath = SchemaPath.getSimplePath(vvIn.getField().getPath());
+ SchemaPath inputPath = SchemaPath.getSimplePath(vvIn.getField().getName());
// get the renamed column names
- SchemaPath outputPath = SchemaPath.getSimplePath(outputFields.get(index).getPath());
+ SchemaPath outputPath = SchemaPath.getSimplePath(outputFields.get(index).getName());
final ErrorCollector collector = new ErrorCollectorImpl();
// According to input data names, Minortypes, Datamodes, choose to
@@ -202,19 +202,20 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// Transfer column
MajorType outputFieldType = outputFields.get(index).getType();
- MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), outputFieldType);
+ MaterializedField outputField = MaterializedField.create(outputPath.getLastSegment().getNameSegment().getPath(),
+ outputFieldType);
/*
todo: Fix if condition when DRILL-4824 is merged
If condition should be changed to:
- `if (outputFields.get(index).getPath().equals(inputPath.getAsUnescapedPath())) {`
+ `if (outputFields.get(index).getName().equals(inputPath.getRootSegmentPath())) {`
DRILL-5419 has changed condition to correct one but this caused regression (DRILL-5521).
Root cause is missing indication of child column in map types when it is null.
DRILL-4824 is re-working json reader implementation, including map types and will fix this problem.
Reverting condition to previous one to avoid regression till DRILL-4824 is merged.
Unit test - TestJsonReader.testKvgenWithUnionAll().
*/
- if (outputFields.get(index).getPath().equals(inputPath)) {
+ if (outputFields.get(index).getName().equals(inputPath)) {
ValueVector vvOut = container.addOrGet(outputField);
TransferPair tp = vvIn.makeTransferPair(vvOut);
transfers.add(tp);
@@ -227,7 +228,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
ValueVector vv = container.addOrGet(outputField, callBack);
allocationVectors.add(vv);
- TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
+ TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName()));
ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
cg.addExpr(write);
}
@@ -240,7 +241,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// If the inputs' DataMode is required and the outputs' DataMode is not required
// cast to the one with the least restriction
- if(vvIn.getField().getType().getMode() == DataMode.REQUIRED
+ if (vvIn.getField().getType().getMode() == DataMode.REQUIRED
&& outputFields.get(index).getType().getMode() != DataMode.REQUIRED) {
expr = ExpressionTreeMaterializer.convertToNullableType(expr, vvIn.getField().getType().getMinorType(), context.getFunctionRegistry(), collector);
if (collector.hasErrors()) {
@@ -250,17 +251,18 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// If two inputs' MinorTypes are different,
// Insert a cast before the Union operation
- if(vvIn.getField().getType().getMinorType() != outputFields.get(index).getType().getMinorType()) {
+ if (vvIn.getField().getType().getMinorType() != outputFields.get(index).getType().getMinorType()) {
expr = ExpressionTreeMaterializer.addCastExpression(expr, outputFields.get(index).getType(), context.getFunctionRegistry(), collector);
if (collector.hasErrors()) {
throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
}
}
- final MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), expr.getMajorType());
+ final MaterializedField outputField = MaterializedField.create(outputPath.getLastSegment().getNameSegment().getPath(),
+ expr.getMajorType());
ValueVector vector = container.addOrGet(outputField, callBack);
allocationVectors.add(vector);
- TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath()));
+ TypedFieldId fid = container.getValueVectorId(outputPath);
boolean useSetSafe = !(vector instanceof FixedWidthVector);
ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe);
@@ -272,12 +274,12 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
unionall = context.getImplementationClass(cg.getCodeGenerator());
unionall.setup(context, current, this, transfers);
- if(!schemaAvailable) {
+ if (!schemaAvailable) {
container.buildSchema(BatchSchema.SelectionVectorMode.NONE);
schemaAvailable = true;
}
- if(!doAlloc()) {
+ if (!doAlloc()) {
return IterOutcome.OUT_OF_MEMORY;
}
@@ -298,7 +300,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
// This method is used by inner class to clear the current record batch
private void clearCurrentRecordBatch() {
- for(VectorWrapper<?> v: current) {
+ for (VectorWrapper<?> v: current) {
v.clear();
}
}
@@ -333,9 +335,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
}
public IterOutcome nextBatch() throws SchemaChangeException {
- if(upstream == RecordBatch.IterOutcome.NOT_YET) {
+ if (upstream == RecordBatch.IterOutcome.NOT_YET) {
IterOutcome iterLeft = leftSide.nextBatch();
- switch(iterLeft) {
+ switch (iterLeft) {
case OK_NEW_SCHEMA:
/*
* If the first few record batches are all empty,
@@ -345,7 +347,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
* Thus, while-loop is necessary to skip those empty batches.
*/
whileLoop:
- while(leftSide.getRecordBatch().getRecordCount() == 0) {
+ while (leftSide.getRecordBatch().getRecordCount() == 0) {
iterLeft = leftSide.nextBatch();
switch(iterLeft) {
@@ -380,11 +382,11 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
}
IterOutcome iterRight = rightSide.nextBatch();
- switch(iterRight) {
+ switch (iterRight) {
case OK_NEW_SCHEMA:
// Unless there is no record batch on the left side of the inputs,
// always start processing from the left side.
- if(leftIsFinish) {
+ if (leftIsFinish) {
unionAllRecordBatch.setCurrentRecordBatch(rightSide.getRecordBatch());
} else {
unionAllRecordBatch.setCurrentRecordBatch(leftSide.getRecordBatch());
@@ -402,9 +404,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
* Thus, while-loop is necessary to skip those empty batches.
*/
whileLoop:
- while(rightSide.getRecordBatch().getRecordCount() == 0) {
+ while (rightSide.getRecordBatch().getRecordCount() == 0) {
iterRight = rightSide.nextBatch();
- switch(iterRight) {
+ switch (iterRight) {
case STOP:
case OUT_OF_MEMORY:
return iterRight;
@@ -425,7 +427,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
}
}
- if(leftIsFinish && rightIsFinish) {
+ if (leftIsFinish && rightIsFinish) {
setBothSideEmpty(true);
}
@@ -446,19 +448,19 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
upstream = IterOutcome.OK_NEW_SCHEMA;
return upstream;
} else {
- if(isBothSideEmpty()) {
+ if (isBothSideEmpty()) {
return IterOutcome.NONE;
}
unionAllRecordBatch.clearCurrentRecordBatch();
- if(leftIsFinish && rightIsFinish) {
+ if (leftIsFinish && rightIsFinish) {
upstream = IterOutcome.NONE;
return upstream;
- } else if(leftIsFinish) {
+ } else if (leftIsFinish) {
IterOutcome iterOutcome = rightSide.nextBatch();
- switch(iterOutcome) {
+ switch (iterOutcome) {
case NONE:
rightIsFinish = true;
// fall through
@@ -468,7 +470,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
return upstream;
case OK_NEW_SCHEMA:
- if(!rightSide.getRecordBatch().getSchema().equals(rightSchema)) {
+ if (!rightSide.getRecordBatch().getSchema().equals(rightSchema)) {
throw new SchemaChangeException("Schema change detected in the right input of Union-All. This is not currently supported");
}
iterOutcome = IterOutcome.OK;
@@ -481,9 +483,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
default:
throw new IllegalStateException(String.format("Unknown state %s.", upstream));
}
- } else if(rightIsFinish) {
+ } else if (rightIsFinish) {
IterOutcome iterOutcome = leftSide.nextBatch();
- switch(iterOutcome) {
+ switch (iterOutcome) {
case STOP:
case OUT_OF_MEMORY:
case NONE:
@@ -501,14 +503,14 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
} else {
IterOutcome iterOutcome = leftSide.nextBatch();
- switch(iterOutcome) {
+ switch (iterOutcome) {
case STOP:
case OUT_OF_MEMORY:
upstream = iterOutcome;
return upstream;
case OK_NEW_SCHEMA:
- if(!leftSide.getRecordBatch().getSchema().equals(leftSchema)) {
+ if (!leftSide.getRecordBatch().getSchema().equals(leftSchema)) {
throw new SchemaChangeException("Schema change detected in the left input of Union-All. This is not currently supported");
}
@@ -549,10 +551,10 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
* types are nullable integer
*/
private void inferOutputFields() {
- if(!leftIsFinish && !rightIsFinish) {
+ if (!leftIsFinish && !rightIsFinish) {
// Both sides are non-empty
inferOutputFieldsBothSide();
- } else if(!rightIsFinish) {
+ } else if (!rightIsFinish) {
// Left side is non-empty
// While use left side's column names as output column names,
// use right side's column types as output column types.
@@ -585,7 +587,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
if (hasSameTypeAndMode(leftField, rightField)) {
MajorType.Builder builder = MajorType.newBuilder().setMinorType(leftField.getType().getMinorType()).setMode(leftField.getDataMode());
builder = Types.calculateTypePrecisionAndScale(leftField.getType(), rightField.getType(), builder);
- outputFields.add(MaterializedField.create(leftField.getPath(), builder.build()));
+ outputFields.add(MaterializedField.create(leftField.getName(), builder.build()));
} else {
// If the output type is not the same,
// cast the column of one of the table to a data type which is the Least Restrictive
@@ -612,7 +614,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
dataModes.add(rightField.getType().getMode());
builder.setMode(TypeCastRules.getLeastRestrictiveDataMode(dataModes));
- outputFields.add(MaterializedField.create(leftField.getPath(), builder.build()));
+ outputFields.add(MaterializedField.create(leftField.getName(), builder.build()));
}
++index;
}
@@ -624,20 +626,19 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
outputFields = Lists.newArrayList();
final List<String> outputColumnNames = Lists.newArrayList();
- final Iterator<MaterializedField> iterForNames = schemaForNames.iterator();
- while(iterForNames.hasNext()) {
- outputColumnNames.add(iterForNames.next().getPath());
+ for (MaterializedField materializedField : schemaForNames) {
+ outputColumnNames.add(materializedField.getName());
}
final Iterator<MaterializedField> iterForTypes = schemaForTypes.iterator();
- for(int i = 0; iterForTypes.hasNext(); ++i) {
+ for (int i = 0; iterForTypes.hasNext(); ++i) {
MaterializedField field = iterForTypes.next();
outputFields.add(MaterializedField.create(outputColumnNames.get(i), field.getType()));
}
}
public List<MaterializedField> getOutputFields() {
- if(outputFields == null) {
+ if (outputFields == null) {
throw new NullPointerException("Output fields have not been inferred");
}
@@ -670,11 +671,11 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
}
public IterOutcome nextBatch() {
- if(upstream == IterOutcome.NONE) {
+ if (upstream == IterOutcome.NONE) {
throw new IllegalStateException(String.format("Unknown state %s.", upstream));
}
- if(upstream == IterOutcome.NOT_YET) {
+ if (upstream == IterOutcome.NOT_YET) {
upstream = unionAllRecordBatch.next(recordBatch);
return upstream;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index 39c662f..e0cfc7a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -678,7 +678,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
for (BatchGroup group : batchGroupList) {
vectors[i++] = group.getValueAccessorById(
field.getValueClass(),
- group.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds())
+ group.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds())
.getValueVector();
}
cont.add(vectors);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
index 6ec8862..2d82cc1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
@@ -310,7 +310,7 @@ public class PriorityQueueCopierWrapper extends BaseSortWrapper {
for (BatchGroup group : batchGroupList) {
vectors[i++] = group.getValueAccessorById(
field.getValueClass(),
- group.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds())
+ group.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds())
.getValueVector();
}
cont.add(vectors);
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
index 9a3ef96..837ddd8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -160,7 +160,7 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule {
metaContext = ((FormatSelection)selection).getSelection().getMetaContext();
}
- RexNode condition = null;
+ RexNode condition;
if (projectRel == null) {
condition = filterRel.getCondition();
} else {
@@ -244,7 +244,7 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule {
for (int partitionColumnIndex : BitSets.toIter(partitionColumnBitSet)) {
SchemaPath column = SchemaPath.getSimplePath(fieldNameMap.get(partitionColumnIndex));
MajorType type = descriptor.getVectorType(column, settings);
- MaterializedField field = MaterializedField.create(column.getAsUnescapedPath(), type);
+ MaterializedField field = MaterializedField.create(column.getLastSegment().getNameSegment().getPath(), type);
ValueVector v = TypeHelper.getNewVector(field, allocator);
v.allocateNew();
vectors[partitionColumnIndex] = v;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
index 3801cb5..9a8483b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -88,7 +88,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp
final Map<String, ValueVector> oldFields = Maps.newHashMap();
for(final VectorWrapper<?> wrapper : container) {
final ValueVector vector = wrapper.getValueVector();
- oldFields.put(vector.getField().getPath(), vector);
+ oldFields.put(vector.getField().getName(), vector);
}
final VectorContainer newVectors = new VectorContainer();
@@ -97,7 +97,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp
int bufOffset = 0;
for(final SerializedField field : fields) {
final MaterializedField fieldDef = MaterializedField.create(field);
- ValueVector vector = oldFields.remove(fieldDef.getPath());
+ ValueVector vector = oldFields.remove(fieldDef.getName());
if (vector == null) {
// Field did not exist previously--is schema change.
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java
index 28f5bf2..e1a1031 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -53,7 +53,7 @@ public class SchemaUtil {
for (BatchSchema s : schemas) {
for (MaterializedField field : s) {
- SchemaPath path = SchemaPath.getSimplePath(field.getPath());
+ SchemaPath path = SchemaPath.getSimplePath(field.getName());
Set<MinorType> currentTypes = typeSetMap.get(path);
if (currentTypes == null) {
currentTypes = Sets.newHashSet();
@@ -64,9 +64,7 @@ public class SchemaUtil {
throw new RuntimeException("Schema change not currently supported for schemas with complex types");
}
if (newType == MinorType.UNION) {
- for (MinorType subType : field.getType().getSubTypeList()) {
- currentTypes.add(subType);
- }
+ currentTypes.addAll(field.getType().getSubTypeList());
} else {
currentTypes.add(newType);
}
@@ -82,10 +80,11 @@ public class SchemaUtil {
for (MinorType t : types) {
builder.addSubType(t);
}
- MaterializedField field = MaterializedField.create(path.getAsUnescapedPath(), builder.build());
+ MaterializedField field = MaterializedField.create(path.getLastSegment().getNameSegment().getPath(), builder.build());
fields.add(field);
} else {
- MaterializedField field = MaterializedField.create(path.getAsUnescapedPath(), Types.optional(types.iterator().next()));
+ MaterializedField field = MaterializedField.create(path.getLastSegment().getNameSegment().getPath(),
+ Types.optional(types.iterator().next()));
fields.add(field);
}
}
@@ -162,12 +161,12 @@ public class SchemaUtil {
if (w.isHyper()) {
isHyper = true;
final ValueVector[] vvs = w.getValueVectors();
- vectorMap.put(vvs[0].getField().getPath(), vvs);
+ vectorMap.put(vvs[0].getField().getName(), vvs);
} else {
assert !isHyper;
@SuppressWarnings("resource")
final ValueVector v = w.getValueVector();
- vectorMap.put(v.getField().getPath(), v);
+ vectorMap.put(v.getField().getName(), v);
}
}
@@ -175,7 +174,7 @@ public class SchemaUtil {
for (MaterializedField field : toSchema) {
if (isHyper) {
- final ValueVector[] vvs = (ValueVector[]) vectorMap.remove(field.getPath());
+ final ValueVector[] vvs = (ValueVector[]) vectorMap.remove(field.getName());
final ValueVector[] vvsOut;
if (vvs == null) {
vvsOut = new ValueVector[1];
@@ -189,7 +188,7 @@ public class SchemaUtil {
c.add(vvsOut);
} else {
@SuppressWarnings("resource")
- final ValueVector v = (ValueVector) vectorMap.remove(field.getPath());
+ final ValueVector v = (ValueVector) vectorMap.remove(field.getName());
c.add(coerceVector(v, c, field, recordCount, allocator));
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index 99353ea..63cac7d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -136,7 +136,7 @@ public class VectorContainer implements VectorAccessible {
@SuppressWarnings({ "resource", "unchecked" })
public <T extends ValueVector> T addOrGet(final MaterializedField field, final SchemaChangeCallBack callBack) {
- final TypedFieldId id = getValueVectorId(SchemaPath.getSimplePath(field.getPath()));
+ final TypedFieldId id = getValueVectorId(SchemaPath.getSimplePath(field.getName()));
final ValueVector vector;
final Class<?> clazz = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getType().getMode());
if (id != null) {
@@ -206,12 +206,12 @@ public class VectorContainer implements VectorAccessible {
*/
public static VectorContainer canonicalize(VectorContainer original) {
VectorContainer vc = new VectorContainer();
- List<VectorWrapper<?>> canonicalWrappers = new ArrayList<VectorWrapper<?>>(original.wrappers);
+ List<VectorWrapper<?>> canonicalWrappers = new ArrayList<>(original.wrappers);
// Sort list of VectorWrapper alphabetically based on SchemaPath.
Collections.sort(canonicalWrappers, new Comparator<VectorWrapper<?>>() {
@Override
public int compare(VectorWrapper<?> v1, VectorWrapper<?> v2) {
- return v1.getField().getPath().compareTo(v2.getField().getPath());
+ return v1.getField().getName().compareTo(v2.getField().getName());
}
});
@@ -255,9 +255,7 @@ public class VectorContainer implements VectorAccessible {
schema = null;
Class<?> clazz = hyperVector[0].getClass();
ValueVector[] c = (ValueVector[]) Array.newInstance(clazz, hyperVector.length);
- for (int i = 0; i < hyperVector.length; i++) {
- c[i] = hyperVector[i];
- }
+ System.arraycopy(hyperVector, 0, c, 0, hyperVector.length);
// todo: work with a merged schema.
wrappers.add(HyperVectorWrapper.create(hyperVector[0].getField(), c, releasable));
}
@@ -283,7 +281,7 @@ public class VectorContainer implements VectorAccessible {
for (VectorWrapper<?> w : wrappers){
if (!w.isHyper() && old == w.getValueVector()) {
w.clear();
- wrappers.set(i, new SimpleVectorWrapper<ValueVector>(newVector));
+ wrappers.set(i, new SimpleVectorWrapper<>(newVector));
return;
}
i++;
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
index 62c6efd..bcce9eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
@@ -99,7 +99,7 @@ public class WebUserConnection extends AbstractDisposableUserClientConnection im
// TODO: Clean: DRILL-2933: That load(...) no longer throws
// SchemaChangeException, so check/clean catch clause below.
for (int i = 0; i < loader.getSchema().getFieldCount(); ++i) {
- columns.add(loader.getSchema().getColumn(i).getPath());
+ columns.add(loader.getSchema().getColumn(i).getName());
}
for (int i = 0; i < rows; ++i) {
final Map<String, String> record = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java
index ccd622b..fa8121e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java
@@ -95,7 +95,7 @@ public class ColumnExplorer {
*/
public static boolean isPartitionColumn(OptionManager optionManager, SchemaPath column){
String partitionDesignator = optionManager.getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL).string_val;
- String path = column.getAsUnescapedPath();
+ String path = column.getRootSegmentPath();
return isPartitionColumn(partitionDesignator, path);
}
@@ -187,7 +187,7 @@ public class ColumnExplorer {
selectedImplicitColumns.putAll(allImplicitColumns);
} else {
for (SchemaPath column : columns) {
- String path = column.getAsUnescapedPath();
+ String path = column.getRootSegmentPath();
if (isPartitionColumn(partitionDesignator, path)) {
selectedPartitionColumns.add(Integer.parseInt(path.substring(partitionDesignator.length())));
} else if (allImplicitColumns.get(path) != null) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
index 22fb48c..6262dce 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -138,7 +138,7 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void,
public ExprNode visitCastExpression(CastExpression e, Void value) throws RuntimeException {
if (e.getInput() instanceof FieldReference) {
FieldReference fieldRef = (FieldReference) e.getInput();
- String field = fieldRef.getAsUnescapedPath().toUpperCase();
+ String field = fieldRef.getRootSegmentPath().toUpperCase();
if (field.equals(CATS_COL_CATALOG_NAME)
|| field.equals(SCHS_COL_SCHEMA_NAME)
|| field.equals(SHRD_COL_TABLE_NAME)
@@ -158,7 +158,7 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void,
@Override
public ExprNode visitSchemaPath(SchemaPath path, Void value) throws RuntimeException {
- String field = path.getAsUnescapedPath().toUpperCase();
+ String field = path.getRootSegmentPath().toUpperCase();
if (field.equals(CATS_COL_CATALOG_NAME)
|| field.equals(SCHS_COL_SCHEMA_NAME)
|| field.equals(SHRD_COL_TABLE_NAME)
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
index a33f46a..4a8c5f3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
@@ -80,10 +80,12 @@ import com.google.common.collect.Maps;
import javax.annotation.Nullable;
+import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.SUPPORTED_VERSIONS;
import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V1;
import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V2;
import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3;
import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3_1;
+import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3_2;
public class Metadata {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Metadata.class);
@@ -234,7 +236,8 @@ public class Metadata {
childFiles.add(file);
}
}
- ParquetTableMetadata_v3 parquetTableMetadata = new ParquetTableMetadata_v3(V3_1, DrillVersionInfo.getVersion());
+ ParquetTableMetadata_v3 parquetTableMetadata = new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(),
+ DrillVersionInfo.getVersion());
if (childFiles.size() > 0) {
List<ParquetFileMetadata_v3 > childFilesMetadata =
getParquetFileMetadata_v3(parquetTableMetadata, childFiles);
@@ -308,7 +311,8 @@ public class Metadata {
*/
private ParquetTableMetadata_v3 getParquetTableMetadata(List<FileStatus> fileStatuses)
throws IOException {
- ParquetTableMetadata_v3 tableMetadata = new ParquetTableMetadata_v3(V3_1, DrillVersionInfo.getVersion());
+ ParquetTableMetadata_v3 tableMetadata = new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(),
+ DrillVersionInfo.getVersion());
List<ParquetFileMetadata_v3> fileMetadataList = getParquetFileMetadata_v3(tableMetadata, fileStatuses);
tableMetadata.files = fileMetadataList;
tableMetadata.directories = new ArrayList<String>();
@@ -695,7 +699,8 @@ public class Metadata {
@JsonSubTypes.Type(value = ParquetTableMetadata_v1.class, name = V1),
@JsonSubTypes.Type(value = ParquetTableMetadata_v2.class, name = V2),
@JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3),
- @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_1)
+ @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_1),
+ @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_2)
})
public static abstract class ParquetTableMetadataBase {
@@ -1421,7 +1426,7 @@ public class Metadata {
}
- @JsonTypeName(V3_1)
+ @JsonTypeName(V3_2)
public static class ParquetTableMetadata_v3 extends ParquetTableMetadataBase {
@JsonProperty(value = "metadata_version", access = JsonProperty.Access.WRITE_ONLY) private String metadataVersion;
/*
@@ -1654,16 +1659,20 @@ public class Metadata {
}
private static class Key {
- private String[] name;
+ private SchemaPath name;
private int hashCode = 0;
public Key(String[] name) {
- this.name = name;
+ this.name = SchemaPath.getCompoundPath(name);
+ }
+
+ public Key(SchemaPath name) {
+ this.name = new SchemaPath(name);
}
@Override public int hashCode() {
if (hashCode == 0) {
- hashCode = Arrays.hashCode(name);
+ hashCode = name.hashCode();
}
return hashCode;
}
@@ -1676,20 +1685,11 @@ public class Metadata {
return false;
}
final Key other = (Key) obj;
- return Arrays.equals(this.name, other.name);
+ return this.name.equals(other.name);
}
@Override public String toString() {
- String s = null;
- for (String namePart : name) {
- if (s != null) {
- s += ".";
- s += namePart;
- } else {
- s = namePart;
- }
- }
- return s;
+ return name.toString();
}
public static class DeSerializer extends KeyDeserializer {
@@ -1701,6 +1701,10 @@ public class Metadata {
@Override
public Object deserializeKey(String key, com.fasterxml.jackson.databind.DeserializationContext ctxt)
throws IOException, com.fasterxml.jackson.core.JsonProcessingException {
+ // key string should contain '`' char if the field was serialized as SchemaPath object
+ if (key.contains("`")) {
+ return new Key(SchemaPath.parseFromString(key));
+ }
return new Key(key.split("\\."));
}
}
@@ -1884,8 +1888,8 @@ public class Metadata {
filesWithRelativePaths.add(new ParquetFileMetadata_v3(
relativize(baseDir, file.getPath()), file.length, file.rowGroups));
}
- return new ParquetTableMetadata_v3(V3_1, tableMetadataWithAbsolutePaths, filesWithRelativePaths,
- directoriesWithRelativePaths, DrillVersionInfo.getVersion());
+ return new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(), tableMetadataWithAbsolutePaths,
+ filesWithRelativePaths, directoriesWithRelativePaths, DrillVersionInfo.getVersion());
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java
index bc6fd70..5ceadcd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java
@@ -134,6 +134,13 @@ public class MetadataVersion implements Comparable<MetadataVersion> {
public static final String V3_1 = "3.1";
/**
+ * Version 3.2: An array with the components of the field name in
+ * {@link Metadata.ColumnTypeMetadata_v3.Key} class is replaced by the SchemaPath.<br>
+ * See DRILL-4264
+ */
+ public static final String V3_2 = "3.2";
+
+ /**
* All historical versions of the Drill metadata cache files. In case of introducing a new parquet metadata version
* please follow the {@link MetadataVersion#FORMAT}.
*/
@@ -141,7 +148,8 @@ public class MetadataVersion implements Comparable<MetadataVersion> {
new MetadataVersion(V1),
new MetadataVersion(V2),
new MetadataVersion(V3),
- new MetadataVersion(V3_1)
+ new MetadataVersion(V3_1),
+ new MetadataVersion(V3_2)
);
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
index 1d4d161..9b84e81 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java
@@ -199,7 +199,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
private void newSchema() throws IOException {
List<Type> types = Lists.newArrayList();
for (MaterializedField field : batchSchema) {
- if (field.getPath().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) {
+ if (field.getName().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) {
continue;
}
types.add(getType(field));
@@ -230,7 +230,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
private PrimitiveType getPrimitiveType(MaterializedField field) {
MinorType minorType = field.getType().getMinorType();
- String name = field.getLastName();
+ String name = field.getName();
PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
Repetition repetition = ParquetTypeHelper.getRepetitionForDataMode(field.getDataMode());
OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
@@ -248,7 +248,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter {
for (MaterializedField childField : field.getChildren()) {
types.add(getType(childField));
}
- return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getLastName(), types);
+ return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getName(), types);
case LIST:
throw new UnsupportedOperationException("Unsupported type " + minorType);
default:
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
index bbdf246..d49a416 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
@@ -59,12 +59,12 @@ public class ParquetColumnMetadata {
se = schemaElements.get(column.getPath()[0]);
type = ParquetToDrillTypeConverter.toMajorType(column.getType(), se.getType_length(),
getDataMode(column), se, options);
- field = MaterializedField.create(toFieldName(column.getPath()), type);
+ field = MaterializedField.create(toFieldName(column.getPath()).getLastSegment().getNameSegment().getPath(), type);
length = getDataTypeLength();
}
- private String toFieldName(String[] paths) {
- return SchemaPath.getCompoundPath(paths).getAsUnescapedPath();
+ private SchemaPath toFieldName(String[] paths) {
+ return SchemaPath.getCompoundPath(paths);
}
private TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
index ab4b1b8..9814b53 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
@@ -108,7 +108,6 @@ public class ParquetSchema {
* Build the schema for this read as a combination of the schema specified in
* the Parquet footer and the list of columns selected in the query.
*
- * @param footer Parquet metadata
* @param batchSize target size of the batch, in rows
* @throws Exception if anything goes wrong
*/
@@ -188,7 +187,7 @@ public class ParquetSchema {
/**
* Determine if a Parquet field is selected for the query. It is selected
* either if this is a star query (we want all columns), or the column
- * appers in the select list.
+ * appears in the select list.
*
* @param field the Parquet column expressed as as Drill field.
* @return true if the column is to be included in the scan, false
@@ -205,7 +204,7 @@ public class ParquetSchema {
int i = 0;
for (SchemaPath expr : selectedCols) {
- if ( field.getPath().equalsIgnoreCase(expr.getAsUnescapedPath())) {
+ if (field.getName().equalsIgnoreCase(expr.getRootSegmentPath())) {
columnsFound[i] = true;
return true;
}
@@ -243,8 +242,9 @@ public class ParquetSchema {
*/
private NullableIntVector createMissingColumn(SchemaPath col, OutputMutator output) throws SchemaChangeException {
- MaterializedField field = MaterializedField.create(col.getAsUnescapedPath(),
- Types.optional(TypeProtos.MinorType.INT));
+ // col.toExpr() is used here as field name since we don't want to see these fields in the existing maps
+ MaterializedField field = MaterializedField.create(col.toExpr(),
+ Types.optional(TypeProtos.MinorType.INT));
return (NullableIntVector) output.addField(field,
TypeHelper.getValueVectorClass(TypeProtos.MinorType.INT, DataMode.OPTIONAL));
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 68d3bbb..9e66f6d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -208,27 +208,28 @@ public class DrillParquetReader extends AbstractRecordReader {
try {
this.operatorContext = context;
schema = footer.getFileMetaData().getSchema();
- MessageType projection = null;
+ MessageType projection;
if (isStarQuery()) {
projection = schema;
} else {
- columnsNotFound=new ArrayList<SchemaPath>();
+ columnsNotFound = new ArrayList<>();
projection = getProjection(schema, getColumns(), columnsNotFound);
- if(projection == null){
+ if (projection == null) {
projection = schema;
}
- if(columnsNotFound!=null && columnsNotFound.size()>0) {
+ if (columnsNotFound != null && columnsNotFound.size() > 0) {
nullFilledVectors = new ArrayList<>();
- for(SchemaPath col: columnsNotFound){
+ for (SchemaPath col: columnsNotFound) {
+ // col.toExpr() is used here as field name since we don't want to see these fields in the existing maps
nullFilledVectors.add(
- (NullableIntVector)output.addField(MaterializedField.create(col.getAsUnescapedPath(),
+ (NullableIntVector) output.addField(MaterializedField.create(col.toExpr(),
org.apache.drill.common.types.Types.optional(TypeProtos.MinorType.INT)),
(Class<? extends ValueVector>) TypeHelper.getValueVectorClass(TypeProtos.MinorType.INT,
TypeProtos.DataMode.OPTIONAL)));
}
- if(columnsNotFound.size()==getColumns().size()){
- noColumnsFound=true;
+ if (columnsNotFound.size() == getColumns().size()) {
+ noColumnsFound = true;
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
index cb5183e..7e32a4d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -34,21 +34,18 @@ import com.google.common.collect.Lists;
public class BatchPrinter {
public static void printHyperBatch(VectorAccessible batch, SelectionVector4 sv4) {
List<String> columns = Lists.newArrayList();
- List<ValueVector> vectors = Lists.newArrayList();
- int numBatches = 0;
for (VectorWrapper vw : batch) {
- columns.add(vw.getValueVectors()[0].getField().getPath());
- numBatches = vw.getValueVectors().length;
+ columns.add(vw.getValueVectors()[0].getField().getName());
}
int width = columns.size();
for (int j = 0; j < sv4.getCount(); j++) {
- if (j%50 == 0) {
+ if (j % 50 == 0) {
System.out.println(StringUtils.repeat("-", width * 17 + 1));
for (String column : columns) {
System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
}
System.out.printf("|\n");
- System.out.println(StringUtils.repeat("-", width*17 + 1));
+ System.out.println(StringUtils.repeat("-", width * 17 + 1));
}
for (VectorWrapper vw : batch) {
Object o = vw.getValueVectors()[sv4.get(j) >>> 16].getAccessor().getObject(sv4.get(j) & 65535);
@@ -60,7 +57,7 @@ public class BatchPrinter {
} else {
value = o.toString();
}
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
+ System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14));
}
System.out.printf("|\n");
}
@@ -71,32 +68,31 @@ public class BatchPrinter {
List<String> columns = Lists.newArrayList();
List<ValueVector> vectors = Lists.newArrayList();
for (VectorWrapper vw : batch) {
- columns.add(vw.getValueVector().getField().getPath());
+ columns.add(vw.getValueVector().getField().getName());
vectors.add(vw.getValueVector());
}
int width = columns.size();
int rows = vectors.get(0).getMetadata().getValueCount();
for (int row = 0; row < rows; row++) {
- if (row%50 == 0) {
+ if (row % 50 == 0) {
System.out.println(StringUtils.repeat("-", width * 17 + 1));
for (String column : columns) {
System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
}
System.out.printf("|\n");
- System.out.println(StringUtils.repeat("-", width*17 + 1));
+ System.out.println(StringUtils.repeat("-", width * 17 + 1));
}
for (ValueVector vv : vectors) {
Object o = vv.getAccessor().getObject(row);
String value;
if (o == null) {
value = "null";
- } else
- if (o instanceof byte[]) {
+ } else if (o instanceof byte[]) {
value = new String((byte[]) o);
} else {
value = o.toString();
}
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
+ System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14));
}
System.out.printf("|\n");
}
@@ -106,19 +102,19 @@ public class BatchPrinter {
List<String> columns = Lists.newArrayList();
List<ValueVector> vectors = Lists.newArrayList();
for (VectorWrapper vw : batch) {
- columns.add(vw.getValueVector().getField().getPath());
+ columns.add(vw.getValueVector().getField().getName());
vectors.add(vw.getValueVector());
}
int width = columns.size();
int rows = vectors.get(0).getMetadata().getValueCount();
for (int i = 0; i < rows; i++) {
- if (i%50 == 0) {
+ if (i % 50 == 0) {
System.out.println(StringUtils.repeat("-", width * 17 + 1));
for (String column : columns) {
System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
}
System.out.printf("|\n");
- System.out.println(StringUtils.repeat("-", width*17 + 1));
+ System.out.println(StringUtils.repeat("-", width * 17 + 1));
}
int row = sv2.getIndex(i);
for (ValueVector vv : vectors) {
@@ -126,13 +122,12 @@ public class BatchPrinter {
String value;
if (o == null) {
value = "null";
- } else
- if (o instanceof byte[]) {
+ } else if (o instanceof byte[]) {
value = new String((byte[]) o);
} else {
value = o.toString();
}
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
+ System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14));
}
System.out.printf("|\n");
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index b150179..d836bfc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -40,12 +40,12 @@ public class VectorUtil {
System.out.println(rows + " row(s):");
List<String> columns = Lists.newArrayList();
for (VectorWrapper<?> vw : va) {
- columns.add(vw.getValueVector().getField().getPath());
+ columns.add(vw.getValueVector().getField().getName());
}
int width = columns.size();
for (String column : columns) {
- System.out.printf("%s%s",column, column == columns.get(width - 1) ? "\n" : delimiter);
+ System.out.printf("%s%s",column, column.equals(columns.get(width - 1)) ? "\n" : delimiter);
}
for (int row = 0; row < rows; row++) {
int columnCounter = 0;
@@ -54,8 +54,8 @@ public class VectorUtil {
Object o ;
try{
o = vw.getValueVector().getAccessor().getObject(row);
- }catch(Exception e){
- throw new RuntimeException("failure while trying to read column " + vw.getField().getPath());
+ } catch (Exception e) {
+ throw new RuntimeException("failure while trying to read column " + vw.getField().getName());
}
if (o == null) {
//null value
@@ -83,7 +83,7 @@ public class VectorUtil {
if (includeHeader) {
List<String> columns = Lists.newArrayList();
for (VectorWrapper<?> vw : va) {
- columns.add(vw.getValueVector().getField().getPath());
+ columns.add(vw.getValueVector().getField().getName());
}
formattedResults.append(Joiner.on(delimiter).join(columns));
@@ -134,7 +134,7 @@ public class VectorUtil {
width += columnWidth + 2;
formats.add("| %-" + columnWidth + "s");
MaterializedField field = vw.getValueVector().getField();
- columns.add(field.getPath() + "<" + field.getType().getMinorType() + "(" + field.getType().getMode() + ")" + ">");
+ columns.add(field.getName() + "<" + field.getType().getMinorType() + "(" + field.getType().getMode() + ")" + ">");
columnIndex++;
}