You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/03/04 09:07:28 UTC
[01/38] DRILL-357: Hive Storage Engine phase 2 - hive record reader
Repository: incubator-drill
Updated Branches:
refs/heads/master b5ab447de -> 4871fd0de
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
new file mode 100644
index 0000000..1e47684
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.google.common.collect.Lists;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.InputSplit;
+
+import java.io.IOException;
+import java.util.List;
+
+public class HiveTextRecordReader extends HiveRecordReader {
+
+ public final byte delimiter;
+ public final List<Integer> columnIds;
+ private final int numCols;
+
+ public HiveTextRecordReader(Table table, Partition partition, InputSplit inputSplit, List<FieldReference> columns, FragmentContext context) throws ExecutionSetupException {
+ super(table, partition, inputSplit, columns, context);
+ String d = table.getSd().getSerdeInfo().getParameters().get("field.delim");
+ if (d != null) {
+ delimiter = d.getBytes()[0];
+ } else {
+ delimiter = (byte) 1;
+ }
+ assert delimiter > 0;
+ List<Integer> ids = Lists.newArrayList();
+ for (int i = 0; i < tableColumns.size(); i++) {
+ if (columnNames.contains(tableColumns.get(i))) {
+ ids.add(i);
+ }
+ }
+ columnIds = ids;
+ numCols = tableColumns.size();
+ }
+
+ public boolean setValue(PrimitiveObjectInspector.PrimitiveCategory pCat, ValueVector vv, int index, byte[] bytes, int start) {
+ switch(pCat) {
+ case BINARY:
+ throw new UnsupportedOperationException();
+ case BOOLEAN:
+ throw new UnsupportedOperationException();
+ case BYTE:
+ throw new UnsupportedOperationException();
+ case DECIMAL:
+ throw new UnsupportedOperationException();
+ case DOUBLE:
+ throw new UnsupportedOperationException();
+ case FLOAT:
+ throw new UnsupportedOperationException();
+ case INT: {
+ int value = 0;
+ byte b;
+ for (int i = start; (b = bytes[i]) != delimiter; i++) {
+ value = (value * 10) + b - 48;
+ }
+ ((IntVector) vv).getMutator().set(index, value); // No need to use setSafe for fixed length vectors
+ return true;
+ }
+ case LONG: {
+ long value = 0;
+ byte b;
+ for (int i = start; (b = bytes[i]) != delimiter; i++) {
+ value = (value * 10) + b - 48;
+ }
+ ((BigIntVector) vv).getMutator().set(index, value); // No need to use setSafe for fixed length vectors
+ return true;
+ }
+ case SHORT:
+ throw new UnsupportedOperationException();
+ case STRING: {
+ int end = start;
+ for (int i = start; i < bytes.length; i++) {
+ if (bytes[i] == delimiter) {
+ end = i;
+ break;
+ }
+ end = bytes.length;
+ }
+ return ((VarCharVector) vv).getMutator().setSafe(index, bytes, start, end - start);
+ }
+ case TIMESTAMP:
+ throw new UnsupportedOperationException();
+
+ default:
+ throw new UnsupportedOperationException("Could not determine type");
+ }
+ }
+
+
+ @Override
+ public int next() {
+ for (ValueVector vv : vectors) {
+ VectorAllocator.getAllocator(vv, 50).alloc(TARGET_RECORD_COUNT);
+ }
+ try {
+ int recordCount = 0;
+ if (redoRecord != null) {
+ int length = ((Text) value).getLength();
+ byte[] bytes = ((Text) value).getBytes();
+ int[] delimPositions = new int[numCols];
+ delimPositions[0] = -1;
+ int p = 0;
+ for (int i = 0; i < length; i++) {
+ if (bytes[i] == delimiter) {
+ delimPositions[p++] = i;
+ }
+ }
+ for (int id : columnIds) {
+ boolean success = setValue(primitiveCategories.get(id), vectors.get(id), recordCount, bytes, delimPositions[id]);
+ if (!success) {
+ throw new DrillRuntimeException(String.format("Failed to write value for column %s", columnNames.get(id)));
+ }
+
+ }
+ redoRecord = null;
+ }
+ while (recordCount < TARGET_RECORD_COUNT && reader.next(key, value)) {
+ int length = ((Text) value).getLength();
+ byte[] bytes = ((Text) value).getBytes();
+ int[] delimPositions = new int[numCols + 1];
+ delimPositions[0] = -1;
+ int p = 1;
+ for (int i = 0; i < length; i++) {
+ if (bytes[i] == delimiter) {
+ delimPositions[p++] = i;
+ }
+ }
+ for (int i = 0; i < columnIds.size(); i++) {
+ int id = columnIds.get(i);
+ boolean success = setValue(primitiveCategories.get(i), vectors.get(i), recordCount, bytes, delimPositions[id] + 1);
+ if (!success) {
+ redoRecord = value;
+ if (partition != null) populatePartitionVectors(recordCount);
+ return recordCount;
+ }
+ }
+ recordCount++;
+ }
+ if (partition != null) populatePartitionVectors(recordCount);
+ return recordCount;
+ } catch (IOException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
index 209961d..86be49e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
@@ -25,7 +25,6 @@ import org.apache.drill.exec.store.ClassPathFileSystem;
import org.apache.drill.exec.store.SchemaProvider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import com.beust.jcommander.internal.Lists;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
new file mode 100644
index 0000000..aa68752
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import com.beust.jcommander.internal.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.vector.ValueVector;
+
+import java.util.List;
+
+/**
+ * This is a tool for printing the content of record batches to screen. Used for debugging.
+ */
+public class BatchPrinter {
+ public static void printHyperBatch(VectorAccessible batch) {
+ List<String> columns = Lists.newArrayList();
+ List<ValueVector> vectors = Lists.newArrayList();
+ int numBatches = 0;
+ for (VectorWrapper vw : batch) {
+ columns.add(vw.getValueVectors()[0].getField().getName());
+ numBatches = vw.getValueVectors().length;
+ }
+ int width = columns.size();
+ for (int i = 0; i < numBatches; i++) {
+ int rows = batch.iterator().next().getValueVectors()[i].getMetadata().getValueCount();
+ for (int j = 0; j < rows; j++) {
+ for (VectorWrapper vw : batch) {
+ Object o = vw.getValueVectors()[i].getAccessor().getObject(j);
+ if (o instanceof byte[]) {
+ String value = new String((byte[]) o);
+ System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
+ } else {
+ String value = o.toString();
+ System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
+ }
+ }
+ System.out.printf("|\n");
+ }
+ }
+ System.out.printf("|\n");
+ }
+ public static void printBatch(VectorAccessible batch) {
+ List<String> columns = Lists.newArrayList();
+ List<ValueVector> vectors = Lists.newArrayList();
+ for (VectorWrapper vw : batch) {
+ columns.add(vw.getValueVector().getField().getName());
+ vectors.add(vw.getValueVector());
+ }
+ int width = columns.size();
+ int rows = vectors.get(0).getMetadata().getValueCount();
+ for (int row = 0; row < rows; row++) {
+ if (row%50 == 0) {
+ System.out.println(StringUtils.repeat("-", width * 17 + 1));
+ for (String column : columns) {
+ System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
+ }
+ System.out.printf("|\n");
+ System.out.println(StringUtils.repeat("-", width*17 + 1));
+ }
+ for (ValueVector vv : vectors) {
+ Object o = vv.getAccessor().getObject(row);
+ if (o instanceof byte[]) {
+ String value = new String((byte[]) o);
+ System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
+ } else {
+ String value = o.toString();
+ System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
+ }
+ }
+ System.out.printf("|\n");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index c801163..4dd08c1 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -82,5 +82,6 @@ drill.exec: {
delete: false,
size: 100000000
}
- }
+ },
+ cache.hazel.subnets: ["*.*.*.*"]
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
new file mode 100644
index 0000000..71e6283
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec;
+
+import org.apache.drill.exec.client.QuerySubmitter;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: sphillips
+ * Date: 1/24/14
+ * Time: 3:46 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class TestPlan {
+
+ String location = "/Users/sphillips/hive-lineitem-orderkey";
+ String type = "physical";
+ String zkQuorum = null;
+ boolean local = true;
+ int bits = 1;
+
+
+ @Test
+ @Ignore
+ public void testSubmitPlan() throws Exception {
+ QuerySubmitter submitter = new QuerySubmitter();
+ submitter.submitQuery(location, type, zkQuorum, local, bits);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
new file mode 100644
index 0000000..c6edc20
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.pop.PopUnitTestBase;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: sphillips
+ * Date: 1/23/14
+ * Time: 5:22 AM
+ * To change this template use File | Settings | File Templates.
+ */
+public class TestHiveScan extends PopUnitTestBase {
+ @Ignore
+ @Test
+ public void twoBitTwoExchangeTwoEntryRun() throws Exception {
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
+ Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
+
+ bit1.run();
+ bit2.run();
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
+ Files.toString(FileUtils.getResourceAsFile("/hive/test.json"),
+ Charsets.UTF_8));
+ int count = 0;
+ for(QueryResultBatch b : results) {
+ if (b.getHeader().getRowCount() != 0)
+ count += b.getHeader().getRowCount();
+ }
+ assertEquals(100, count);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/test/java/org/apache/drill/exec/util/BatchPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/BatchPrinter.java
deleted file mode 100644
index aa68752..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/BatchPrinter.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import com.beust.jcommander.internal.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.record.VectorAccessible;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.record.selection.SelectionVector4;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.vector.ValueVector;
-
-import java.util.List;
-
-/**
- * This is a tool for printing the content of record batches to screen. Used for debugging.
- */
-public class BatchPrinter {
- public static void printHyperBatch(VectorAccessible batch) {
- List<String> columns = Lists.newArrayList();
- List<ValueVector> vectors = Lists.newArrayList();
- int numBatches = 0;
- for (VectorWrapper vw : batch) {
- columns.add(vw.getValueVectors()[0].getField().getName());
- numBatches = vw.getValueVectors().length;
- }
- int width = columns.size();
- for (int i = 0; i < numBatches; i++) {
- int rows = batch.iterator().next().getValueVectors()[i].getMetadata().getValueCount();
- for (int j = 0; j < rows; j++) {
- for (VectorWrapper vw : batch) {
- Object o = vw.getValueVectors()[i].getAccessor().getObject(j);
- if (o instanceof byte[]) {
- String value = new String((byte[]) o);
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
- } else {
- String value = o.toString();
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
- }
- }
- System.out.printf("|\n");
- }
- }
- System.out.printf("|\n");
- }
- public static void printBatch(VectorAccessible batch) {
- List<String> columns = Lists.newArrayList();
- List<ValueVector> vectors = Lists.newArrayList();
- for (VectorWrapper vw : batch) {
- columns.add(vw.getValueVector().getField().getName());
- vectors.add(vw.getValueVector());
- }
- int width = columns.size();
- int rows = vectors.get(0).getMetadata().getValueCount();
- for (int row = 0; row < rows; row++) {
- if (row%50 == 0) {
- System.out.println(StringUtils.repeat("-", width * 17 + 1));
- for (String column : columns) {
- System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
- }
- System.out.printf("|\n");
- System.out.println(StringUtils.repeat("-", width*17 + 1));
- }
- for (ValueVector vv : vectors) {
- Object o = vv.getAccessor().getObject(row);
- if (o instanceof byte[]) {
- String value = new String((byte[]) o);
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
- } else {
- String value = o.toString();
- System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
- }
- }
- System.out.printf("|\n");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/test/resources/hive/test.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/hive/test.json b/exec/java-exec/src/test/resources/hive/test.json
new file mode 100644
index 0000000..a039d9e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/hive/test.json
@@ -0,0 +1,75 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"hive-scan",
+ storageengine: { type: "hive"},
+ hive-table: {
+ "tableName" : "nation",
+ "dbName" : "default",
+ "owner" : "root",
+ "createTime" : 1386876893,
+ "lastAccessTime" : 0,
+ "retention" : 0,
+ "sd" : {
+ "cols" : [ {
+ "name" : "n_nationkey",
+ "type" : "bigint",
+ "comment" : null
+ }, {
+ "name" : "n_name",
+ "type" : "string",
+ "comment" : null
+ }, {
+ "name" : "n_regionkey",
+ "type" : "bigint",
+ "comment" : null
+ }, {
+ "name" : "n_comment",
+ "type" : "string",
+ "comment" : null
+ } ],
+ "location" : "maprfs:/user/hive/warehouse/nation",
+ "inputFormat" : "org.apache.hadoop.mapred.TextInputFormat",
+ "outputFormat" : "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+ "compressed" : false,
+ "numBuckets" : 0,
+ "serDeInfo" : {
+ "name" : null,
+ "serializationLib" : "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
+ "parameters" : {
+ "serialization.format" : "|",
+ "field.delim" : "|"
+ }
+ },
+ "sortCols" : [ ],
+ "parameters" : { }
+ },
+ "partitionKeys" : [ ],
+ "parameters" : {
+ "numPartitions" : "0",
+ "numFiles" : "1",
+ "transient_lastDdlTime" : "1386877487",
+ "totalSize" : "2224",
+ "numRows" : "0",
+ "rawDataSize" : "0"
+ },
+ "viewOriginalText" : null,
+ "viewExpandedText" : null,
+ "tableType" : "MANAGED_TABLE"
+ }
+ },
+ {
+ @id: 2,
+ child: 1,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2121adc..f8ddbf2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -356,6 +356,52 @@
<version>${dep.slf4j.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>0.12.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-hbase-handler</artifactId>
+ <version>0.12.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>com.googlecode.jmockit</groupId>
@@ -594,6 +640,11 @@
<version>1.0.3-mapr-3.0.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.13-mapr-1401-m7-3.0.2</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>1.0.3-mapr-3.0.0</version>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
index fefe7bc..4712302 100644
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
@@ -41,7 +41,7 @@ import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.store.SchemaProvider;
import org.apache.drill.exec.store.SchemaProviderRegistry;
-import org.apache.drill.exec.store.hive.HiveSchemaProvider;
+import org.apache.drill.exec.store.hive.HiveStorageEngine.HiveSchemaProvider;
import org.apache.drill.exec.store.json.JsonSchemaProvider;
import org.apache.drill.exec.store.parquet.ParquetSchemaProvider;
import org.apache.drill.sql.client.full.FileSystemSchema;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
index d444348..1b58805 100644
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
+++ b/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import jline.internal.Preconditions;
+import com.google.common.base.Preconditions;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.exception.SchemaChangeException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
index c12057a..c02a478 100644
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
+++ b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
@@ -31,12 +31,15 @@ import net.hydromatic.optiq.impl.java.JavaTypeFactory;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.exec.store.hive.HiveReadEntry;
+import org.apache.drill.exec.store.hive.HiveStorageEngine;
import org.apache.drill.exec.store.hive.HiveStorageEngineConfig;
import org.apache.drill.jdbc.DrillTable;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -53,11 +56,11 @@ public class HiveDatabaseSchema implements Schema{
private final String name;
private final Expression expression;
private final QueryProvider queryProvider;
- private final SchemaProvider schemaProvider;
+ private final HiveStorageEngine.HiveSchemaProvider schemaProvider;
private final DrillClient client;
private final HiveStorageEngineConfig config;
- public HiveDatabaseSchema(DrillClient client, HiveStorageEngineConfig config, SchemaProvider schemaProvider,
+ public HiveDatabaseSchema(DrillClient client, HiveStorageEngineConfig config, HiveStorageEngine.HiveSchemaProvider schemaProvider,
JavaTypeFactory typeFactory, HiveSchema parentSchema, String name,
Expression expression, QueryProvider queryProvider) {
super();
@@ -121,7 +124,54 @@ public class HiveDatabaseSchema implements Schema{
return Collections.EMPTY_LIST;
}
- static Map<PrimitiveObjectInspector.PrimitiveCategory, SqlTypeName> mapPrimHive2Sql = new HashMap<>();
+ private RelDataType getRelDataTypeFromHiveTypeString(String type) {
+ switch(type) {
+ case "boolean":
+ return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+
+ case "tinyint":
+ return typeFactory.createSqlType(SqlTypeName.TINYINT);
+
+ case "smallint":
+ return typeFactory.createSqlType(SqlTypeName.SMALLINT);
+
+ case "int":
+ return typeFactory.createSqlType(SqlTypeName.INTEGER);
+
+ case "bigint":
+ return typeFactory.createSqlType(SqlTypeName.BIGINT);
+
+ case "float":
+ return typeFactory.createSqlType(SqlTypeName.FLOAT);
+
+ case "double":
+ return typeFactory.createSqlType(SqlTypeName.DOUBLE);
+
+ case "date":
+ return typeFactory.createSqlType(SqlTypeName.DATE);
+
+ case "timestamp":
+ return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
+
+ case "binary":
+ return typeFactory.createSqlType(SqlTypeName.BINARY);
+
+ case "decimal":
+ return typeFactory.createSqlType(SqlTypeName.DECIMAL);
+
+ case "string":
+ case "varchar": {
+ return typeFactory.createTypeWithCharsetAndCollation(
+ typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
+ Charset.forName("ISO-8859-1"), /*unicode char set*/
+ SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
+ );
+ }
+
+ default:
+ throw new RuntimeException("Unknown or unsupported hive type: " + type);
+ }
+ }
private RelDataType getRelDataTypeFromHivePrimitiveType(PrimitiveObjectInspector poi) {
switch(poi.getPrimitiveCategory()) {
@@ -161,7 +211,7 @@ public class HiveDatabaseSchema implements Schema{
case VARCHAR: {
return typeFactory.createTypeWithCharsetAndCollation(
typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
- Charset.forName("UTF-16"), /*unicode char set*/
+ Charset.forName("ISO-8859-1"), /*unicode char set*/
SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
);
}
@@ -189,22 +239,20 @@ public class HiveDatabaseSchema implements Schema{
@SuppressWarnings("unchecked")
@Override
public <E> Table<E> getTable(String name, Class<E> elementType) {
- try {
- org.apache.hadoop.hive.ql.metadata.Table hiveTable =
- parentSchema.getHiveDb().getTable(getName(), name, false /*throwException*/);
-
- if (hiveTable == null) {
- logger.debug("Table name {} is invalid", name);
- return null;
- }
+ Object selection = schemaProvider.getSelectionBaseOnName(String.format("%s.%s",this.name, name));
+ if(selection == null) return null;
+ org.apache.hadoop.hive.metastore.api.Table t = ((HiveReadEntry) selection).getTable();
+ if (t == null) {
+ logger.debug("Table name {} is invalid", name);
+ return null;
+ }
+ org.apache.hadoop.hive.ql.metadata.Table hiveTable = new org.apache.hadoop.hive.ql.metadata.Table(t);
- Object selection = schemaProvider.getSelectionBaseOnName(name);
- if(selection == null) return null;
- final MethodCallExpression call = Expressions.call(getExpression(), //
- BuiltinMethod.DATA_CONTEXT_GET_TABLE.method, //
- Expressions.constant(name), //
- Expressions.constant(Object.class));
+ final MethodCallExpression call = Expressions.call(getExpression(), //
+ BuiltinMethod.DATA_CONTEXT_GET_TABLE.method, //
+ Expressions.constant(name), //
+ Expressions.constant(Object.class));
ArrayList<RelDataType> typeList = new ArrayList<>();
ArrayList<String> fieldNameList = new ArrayList<>();
@@ -215,21 +263,22 @@ public class HiveDatabaseSchema implements Schema{
typeList.add(getRelDataTypeFromHiveType(hiveField.getFieldObjectInspector()));
}
- final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList);
- return (Table<E>) new DrillTable(
- client,
- this,
- Object.class,
- call,
- rowType,
- name,
- null /*storageEngineName*/,
- selection,
- config /*storageEngineConfig*/);
- } catch (HiveException ex) {
- logger.error("getTable failed", ex);
- return null;
- }
+ for (FieldSchema field : hiveTable.getPartitionKeys()) {
+ fieldNameList.add(field.getName());
+ typeList.add(getRelDataTypeFromHiveTypeString(field.getType()));
+ }
+
+ final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList);
+ return (Table<E>) new DrillTable(
+ client,
+ this,
+ Object.class,
+ call,
+ rowType,
+ name,
+ null /*storageEngineName*/,
+ selection,
+ config /*storageEngineConfig*/);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
index ffd6a6e..007c6d9 100644
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
+++ b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
@@ -30,6 +30,7 @@ import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.store.SchemaProvider;
import org.apache.drill.exec.store.hive.HiveStorageEngineConfig;
+import org.apache.drill.exec.store.hive.HiveStorageEngine.HiveSchemaProvider;
import org.apache.drill.jdbc.DrillTable;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -47,7 +48,7 @@ public class HiveSchema implements Schema{
private final String name;
private final Expression expression;
private final QueryProvider queryProvider;
- private final SchemaProvider schemaProvider;
+ private final HiveSchemaProvider schemaProvider;
private final DrillClient client;
private final HiveStorageEngineConfig config;
private Hive hiveDb;
@@ -62,7 +63,7 @@ public class HiveSchema implements Schema{
this.name = name;
this.expression = expression;
this.queryProvider = queryProvider;
- this.schemaProvider = schemaProvider;
+ this.schemaProvider = (HiveSchemaProvider) schemaProvider;
this.config = (HiveStorageEngineConfig) config;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
index 4f9d210..5e5812c 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
@@ -54,6 +54,7 @@ public class FullEngineTest {
* @throws Exception
*/
@Test(timeout=100000) // derby initialization is slow
+ @Ignore
public void listHiveTables() throws Exception {
JdbcAssert.withFull("hive-derby")
.sql("select * from \"metadata\".\"TABLES\"")
[19/38] Integrate new SQL changes with Hive storage engine,
moving to automatic file detection. Rename storage engine to storage
plugin. Separate storage plugins from format plugins,
updating Parquet and JSON to format engines. Refactor distribution log
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetTestProperties.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetTestProperties.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetTestProperties.java
new file mode 100644
index 0000000..7c68a16
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetTestProperties.java
@@ -0,0 +1,37 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import java.util.HashMap;
+
+public class ParquetTestProperties {
+
+ int numberRowGroups;
+ int recordsPerRowGroup;
+ int bytesPerPage = 1024 * 1024 * 1;
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+
+ public ParquetTestProperties(int numberRowGroups, int recordsPerRowGroup, int bytesPerPage,
+ HashMap<String, FieldInfo> fields){
+ this.numberRowGroups = numberRowGroups;
+ this.recordsPerRowGroup = recordsPerRowGroup;
+ this.bytesPerPage = bytesPerPage;
+ this.fields = fields;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
index d2c95fe..689b168 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
@@ -20,7 +20,6 @@ package org.apache.drill.exec.store.parquet;
import static parquet.column.Encoding.PLAIN;
import java.util.HashMap;
-import java.util.Map;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.store.ByteArrayUtil;
@@ -58,84 +57,60 @@ public class TestFileGenerator {
static final Object[] binVals = { varLen1, varLen2, varLen3 };
static final Object[] bin2Vals = { varLen3, varLen2, varLen1 };
- static class FieldInfo {
-
- String parquetType;
- String name;
- int bitLength;
- int numberOfPages;
- Object[] values;
- TypeProtos.MinorType type;
-
- FieldInfo(int recordsPerRowGroup, String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type) {
- this.parquetType = parquetType;
- this.name = name;
- this.bitLength = bitLength;
- this.numberOfPages = Math.max(1, (int) Math.ceil( ((long) recordsPerRowGroup) * bitLength / 8.0 / bytesPerPage));
- this.values = values;
- // generator is designed to use 3 values
- assert values.length == 3;
- this.type = type;
- }
+ static void populateFieldInfoMap(ParquetTestProperties props){
+ props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("f", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
+ props.fields.put("d", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
+ props.fields.put("b", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
}
- private static class WrapAroundCounter {
-
- int maxVal;
- int val;
-
- public WrapAroundCounter(int maxVal) {
- this.maxVal = maxVal;
- }
-
- public int increment() {
- val++;
- if (val > maxVal) {
- val = 0;
- }
- return val;
- }
-
- public void reset() {
- val = 0;
- }
-
+ static void populatePigTPCHCustomerFields(ParquetTestProperties props){
+ // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
+ props.fields.put("C_CUSTKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("C_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("C_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
+ props.fields.put("C_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
+ props.fields.put("C_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ props.fields.put("C_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("C_MKTSEGMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("C_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
}
- public static HashMap<String, FieldInfo> getFieldMap(int recordsPerRowGroup) {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- fields.put("integer", new FieldInfo(recordsPerRowGroup, "int32", "integer", 32, intVals, TypeProtos.MinorType.INT));
- fields.put("bigInt", new FieldInfo(recordsPerRowGroup, "int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT));
- fields.put("f", new FieldInfo(recordsPerRowGroup, "float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4));
- fields.put("d", new FieldInfo(recordsPerRowGroup, "double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8));
- fields.put("b", new FieldInfo(recordsPerRowGroup, "boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT));
- fields.put("bin", new FieldInfo(recordsPerRowGroup, "binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY));
- fields.put("bin2", new FieldInfo(recordsPerRowGroup, "binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY));
- return fields;
+ static void populatePigTPCHSupplierFields(ParquetTestProperties props){
+ // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
+ props.fields.put("S_SUPPKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("S_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("S_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
+ props.fields.put("S_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
+ props.fields.put("S_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ props.fields.put("S_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("S_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
}
- public static void generateParquetFile(String filename, int numberRowGroups, int recordsPerRowGroup) throws Exception {
- final Map<String, FieldInfo> fields = getFieldMap(recordsPerRowGroup);
+ public static void generateParquetFile(String filename, ParquetTestProperties props) throws Exception {
int currentBooleanByte = 0;
WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7);
-
+
Configuration configuration = new Configuration();
- configuration.set(ParquetSchemaProvider.HADOOP_DEFAULT_NAME, "file:///");
- // "message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
+ configuration.set("fs.default.name", "file:///");
+ //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
FileSystem fs = FileSystem.get(configuration);
Path path = new Path(filename);
- if (fs.exists(path))
- fs.delete(path, false);
+ if (fs.exists(path)) fs.delete(path, false);
+
String messageSchema = "message m {";
- for (FieldInfo fieldInfo : fields.values()) {
+ for (FieldInfo fieldInfo : props.fields.values()) {
messageSchema += " required " + fieldInfo.parquetType + " " + fieldInfo.name + ";";
}
// remove the last semicolon, java really needs a join method for strings...
// TODO - nvm apparently it requires a semicolon after every field decl, might want to file a bug
- // messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
+ //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
messageSchema += "}";
MessageType schema = MessageTypeParser.parseMessageType(messageSchema);
@@ -145,25 +120,25 @@ public class TestFileGenerator {
w.start();
HashMap<String, Integer> columnValuesWritten = new HashMap();
int valsWritten;
- for (int k = 0; k < numberRowGroups; k++) {
+ for (int k = 0; k < props.numberRowGroups; k++){
w.startBlock(1);
currentBooleanByte = 0;
booleanBitCounter.reset();
- for (FieldInfo fieldInfo : fields.values()) {
+ for (FieldInfo fieldInfo : props.fields.values()) {
- if (!columnValuesWritten.containsKey(fieldInfo.name)) {
+ if ( ! columnValuesWritten.containsKey(fieldInfo.name)){
columnValuesWritten.put((String) fieldInfo.name, 0);
valsWritten = 0;
} else {
valsWritten = columnValuesWritten.get(fieldInfo.name);
}
- String[] path1 = { (String) fieldInfo.name };
+ String[] path1 = {(String) fieldInfo.name};
ColumnDescriptor c1 = schema.getColumnDescription(path1);
- w.startColumn(c1, recordsPerRowGroup, codec);
- int valsPerPage = (int) Math.ceil(recordsPerRowGroup / (float) fieldInfo.numberOfPages);
+ w.startColumn(c1, props.recordsPerRowGroup, codec);
+ int valsPerPage = (int) Math.ceil(props.recordsPerRowGroup / (float) fieldInfo.numberOfPages);
byte[] bytes;
// for variable length binary fields
int bytesNeededToEncodeLength = 4;
@@ -171,21 +146,18 @@ public class TestFileGenerator {
bytes = new byte[(int) Math.ceil(valsPerPage * (int) fieldInfo.bitLength / 8.0)];
} else {
// the twelve at the end is to account for storing a 4 byte length with each value
- int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length
- + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength;
+ int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength;
// used for the case where there is a number of values in this row group that is not divisible by 3
int leftOverBytes = 0;
- if (valsPerPage % 3 > 0)
- leftOverBytes += ((byte[]) fieldInfo.values[1]).length + 4;
- if (valsPerPage % 3 > 1)
- leftOverBytes += ((byte[]) fieldInfo.values[2]).length + 4;
+ if ( valsPerPage % 3 > 0 ) leftOverBytes += ((byte[])fieldInfo.values[1]).length + bytesNeededToEncodeLength;
+ if ( valsPerPage % 3 > 1 ) leftOverBytes += ((byte[])fieldInfo.values[2]).length + bytesNeededToEncodeLength;
bytes = new byte[valsPerPage / 3 * totalValLength + leftOverBytes];
}
int bytesPerPage = (int) (valsPerPage * ((int) fieldInfo.bitLength / 8.0));
int bytesWritten = 0;
for (int z = 0; z < (int) fieldInfo.numberOfPages; z++, bytesWritten = 0) {
for (int i = 0; i < valsPerPage; i++) {
- // System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : ""));
+ //System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : ""));
if (fieldInfo.values[0] instanceof Boolean) {
bytes[currentBooleanByte] |= bitFields[booleanBitCounter.val]
@@ -195,25 +167,23 @@ public class TestFileGenerator {
currentBooleanByte++;
}
valsWritten++;
- if (currentBooleanByte > bytesPerPage)
- break;
+ if (currentBooleanByte > bytesPerPage) break;
} else {
- if (fieldInfo.values[valsWritten % 3] instanceof byte[]) {
- System.arraycopy(ByteArrayUtil.toByta(((byte[]) fieldInfo.values[valsWritten % 3]).length), 0, bytes,
- bytesWritten, bytesNeededToEncodeLength);
- System.arraycopy(fieldInfo.values[valsWritten % 3], 0, bytes, bytesWritten + bytesNeededToEncodeLength,
- ((byte[]) fieldInfo.values[valsWritten % 3]).length);
- bytesWritten += ((byte[]) fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength;
- } else {
- System.arraycopy(ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]), 0, bytes, i
- * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8);
+ if (fieldInfo.values[valsWritten % 3] instanceof byte[]){
+ System.arraycopy(ByteArrayUtil.toByta(((byte[])fieldInfo.values[valsWritten % 3]).length),
+ 0, bytes, bytesWritten, bytesNeededToEncodeLength);
+ System.arraycopy(fieldInfo.values[valsWritten % 3],
+ 0, bytes, bytesWritten + bytesNeededToEncodeLength, ((byte[])fieldInfo.values[valsWritten % 3]).length);
+ bytesWritten += ((byte[])fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength;
+ } else{
+ System.arraycopy( ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]),
+ 0, bytes, i * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8);
}
valsWritten++;
}
}
- w.writeDataPage((int) (recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length,
- BytesInput.from(bytes), PLAIN, PLAIN, PLAIN);
+ w.writeDataPage((int) (props.recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length, BytesInput.from(bytes), PLAIN, PLAIN, PLAIN);
currentBooleanByte = 0;
}
w.endColumn();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
new file mode 100644
index 0000000..48c0773
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.vector.ValueVector;
+import org.junit.AfterClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
+import com.google.common.io.Resources;
+
+public class TestParquetPhysicalPlan {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetPhysicalPlan.class);
+
+ public String fileName = "parquet/parquet_scan_filter_union_screen_physical.json";
+
+ @Test
+ @Ignore
+ public void testParseParquetPhysicalPlan() throws Exception {
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ DrillConfig config = DrillConfig.create();
+
+ try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+ bit1.run();
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
+ RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+ int count = 0;
+ for (QueryResultBatch b : results) {
+ System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+ count += b.getHeader().getRowCount();
+ loader.load(b.getHeader().getDef(), b.getData());
+ for (VectorWrapper vw : loader) {
+ System.out.print(vw.getValueVector().getField().getName() + ": ");
+ ValueVector vv = vw.getValueVector();
+ for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
+ Object o = vv.getAccessor().getObject(i);
+ if (o instanceof byte[]) {
+ System.out.print(" [" + new String((byte[]) o) + "]");
+ } else {
+ System.out.print(" [" + vv.getAccessor().getObject(i) + "]");
+ }
+// break;
+ }
+ System.out.println();
+ }
+ }
+ client.close();
+ System.out.println(String.format("Got %d total results", count));
+ }
+ }
+
+ private class ParquetResultsListener implements UserResultsListener {
+ AtomicInteger count = new AtomicInteger();
+ private CountDownLatch latch = new CountDownLatch(1);
+ @Override
+ public void submissionFailed(RpcException ex) {
+ logger.error("submission failed", ex);
+ latch.countDown();
+ }
+
+ @Override
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
+ int rows = result.getHeader().getRowCount();
+ System.out.println(String.format("Result batch arrived. Number of records: %d", rows));
+ count.addAndGet(rows);
+ if (result.getHeader().getIsLastChunk()) latch.countDown();
+ result.release();
+ }
+
+ public int await() throws Exception {
+ latch.await();
+ return count.get();
+ }
+
+ @Override
+ public void queryIdArrived(QueryId queryId) {
+ }
+ }
+ @Test
+ @Ignore
+ public void testParseParquetPhysicalPlanRemote() throws Exception {
+ DrillConfig config = DrillConfig.create();
+
+ try(DrillClient client = new DrillClient(config);){
+ client.connect();
+ ParquetResultsListener listener = new ParquetResultsListener();
+ Stopwatch watch = new Stopwatch();
+ watch.start();
+ client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8), listener);
+ System.out.println(String.format("Got %d total records in %d seconds", listener.await(), watch.elapsed(TimeUnit.SECONDS)));
+ client.close();
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception{
+ // pause to get logger to catch up.
+ Thread.sleep(1000);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/WrapAroundCounter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/WrapAroundCounter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/WrapAroundCounter.java
new file mode 100644
index 0000000..eef8560
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/WrapAroundCounter.java
@@ -0,0 +1,40 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+public class WrapAroundCounter {
+
+ int maxVal;
+ int val;
+
+ public WrapAroundCounter(int maxVal) {
+ this.maxVal = maxVal;
+ }
+
+ public int increment() {
+ val++;
+ if (val > maxVal) {
+ val = 0;
+ }
+ return val;
+ }
+
+ public void reset() {
+ val = 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json b/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json
index 5c7b8e3..5f00ce7 100644
--- a/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json
+++ b/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json
@@ -9,25 +9,21 @@
graph:[
{
@id:1,
- pop:"json-sub-scan",
- readEntries:[
- {path: "#{LEFT_FILE}"}
- ],
- engineConfig:{
- "type":"json",
- "dfsName" : "file:///"
- }
+ pop:"fs-sub-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files:[
+ { path: "#{LEFT_FILE}" }
+ ]
},
{
@id:2,
- pop:"json-sub-scan",
- readEntries:[
- {path: "#{RIGHT_FILE}"}
- ],
- engineConfig:{
- "type":"json",
- "dfsName" : "file:///"
- }
+ pop:"fs-sub-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files:[
+ { path: "#{RIGHT_FILE}" }
+ ]
},
{
@id: 3,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/join/merge_multi_batch.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/join/merge_multi_batch.json b/exec/java-exec/src/test/resources/join/merge_multi_batch.json
index 7abd19f..2859951 100644
--- a/exec/java-exec/src/test/resources/join/merge_multi_batch.json
+++ b/exec/java-exec/src/test/resources/join/merge_multi_batch.json
@@ -9,27 +9,21 @@
graph:[
{
@id:1,
- pop:"json-sub-scan",
- readEntries:[
- {path: "#{LEFT_FILE}"}
- ],
- engineConfig:{
- "type":"json",
- "dfsName" : "file:///"
- }
-
+ pop:"fs-sub-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files:[
+ { path: "#{LEFT_FILE}" }
+ ]
},
{
@id:2,
- pop:"json-sub-scan",
- readEntries:[
- {path: "#{RIGHT_FILE}"}
- ],
- engineConfig:{
- "type":"json",
- "dfsName" : "file:///"
- }
-
+ pop:"fs-sub-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files:[
+ { path: "#{RIGHT_FILE}" }
+ ]
},
{
@id: 3,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/join/merge_single_batch.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/join/merge_single_batch.json b/exec/java-exec/src/test/resources/join/merge_single_batch.json
index 1f7e0b5..2859951 100644
--- a/exec/java-exec/src/test/resources/join/merge_single_batch.json
+++ b/exec/java-exec/src/test/resources/join/merge_single_batch.json
@@ -9,25 +9,21 @@
graph:[
{
@id:1,
- pop:"json-sub-scan",
- readEntries:[
- {path: "#{LEFT_FILE}"}
- ],
- engineConfig:{
- "type":"json",
- "dfsName" : "file:///"
- }
+ pop:"fs-sub-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files:[
+ { path: "#{LEFT_FILE}" }
+ ]
},
{
@id:2,
- pop:"json-sub-scan",
- readEntries:[
- {path: "#{RIGHT_FILE}"}
- ],
- engineConfig:{
- "type":"json",
- "dfsName" : "file:///"
- }
+ pop:"fs-sub-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files:[
+ { path: "#{RIGHT_FILE}" }
+ ]
},
{
@id: 3,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/logical_json_scan.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/logical_json_scan.json b/exec/java-exec/src/test/resources/logical_json_scan.json
index f9a8ffa..b1cc4ce 100644
--- a/exec/java-exec/src/test/resources/logical_json_scan.json
+++ b/exec/java-exec/src/test/resources/logical_json_scan.json
@@ -9,8 +9,8 @@
},
storage : {
jsonl : {
- type : "json",
- dfsName : "file:///"
+ type : "file",
+ connection : "file:///"
}
},
query : [ {
@@ -18,9 +18,7 @@
@id : 1,
memo : "initial_scan",
storageengine : "jsonl",
- selection : [ {
- path : "/src/drill/sandbox/prototype/exec/ref/src/test/resources/employees.json"
- } ],
+ files : [ path : "/src/drill/sandbox/prototype/exec/ref/src/test/resources/employees.json" ],
ref : "_MAP"
}, {
op : "store",
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json
index 29cab68..50281f4 100644
--- a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json
+++ b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json
@@ -8,10 +8,10 @@
}
},
storage:{
- "parquet" :
+ "dfs" :
{
- "type":"parquet",
- "dfsName" : "file:///"
+ "type":"named",
+ "name":"dfs"
}
},
query:[
@@ -19,15 +19,14 @@
@id:"1",
op:"scan",
memo:"initial_scan",
- storageengine:"parquet",
- selection: [
- {
- path: "/tmp/parquet_test_file_many_types"
- },
- {
- path: "/tmp/parquet_test_file_many_types"
- }
- ]
+ storageengine:"dfs",
+ selection: {
+ format: {type: "parquet"},
+ files: [
+ "/tmp/parquet_test_file_many_types",
+ "/tmp/parquet_test_file_many_types"
+ ]
+ }
},
{
@id:"2",
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json
index af76e01..1bd52b4 100644
--- a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json
+++ b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json
@@ -8,10 +8,10 @@
}
},
storage:{
- "parquet" :
+ "fs" :
{
- "type":"parquet",
- "dfsName" : "file:///"
+ "type":"file",
+ "connection" : "file:///"
}
},
query:[
@@ -19,10 +19,11 @@
@id:"1",
op:"scan",
memo:"initial_scan",
- storageengine:"parquet",
- selection: [
- &REPLACED_IN_PARQUET_TEST&
- ]
+ storageengine:"fs",
+ selection: {
+ format: {type: "parquet"},
+ files: [&REPLACED_IN_PARQUET_TEST&]
+ }
},
{
@id:"2",
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json b/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json
index 5efecaf..a49b914 100644
--- a/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json
+++ b/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json
@@ -14,9 +14,9 @@
path : "/tmp/parquet_test_file_many_types"
}
],
- storageengine:{
- "type":"parquet",
- "dfsName" : "file:///"
+ storage:{
+ "type":"file",
+ "connection" : "file:///"
}
},
{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/parquet_scan_screen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_scan_screen.json b/exec/java-exec/src/test/resources/parquet_scan_screen.json
index f18c738..3b3f86a 100644
--- a/exec/java-exec/src/test/resources/parquet_scan_screen.json
+++ b/exec/java-exec/src/test/resources/parquet_scan_screen.json
@@ -8,10 +8,10 @@
}
},
storage:{
- "parquet" :
+ "dfs" :
{
- "type":"parquet",
- "dfsName" : "file:///"
+ "type":"file",
+ "connection" : "file:///"
}
},
query:[
@@ -19,12 +19,11 @@
@id:"1",
op:"scan",
memo:"initial_scan",
- storageengine:"parquet",
- selection: [
- {
- path: "/tmp/test.parquet"
- }
- ]
+ storageengine:"dfs",
+ selection: {
+ format: {type: "parquet"},
+ files: ["/tmp/test.parquet"]
+ }
},
{
@id:"2",
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json b/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
index af76e01..ab11a5f 100644
--- a/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
+++ b/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
@@ -8,10 +8,10 @@
}
},
storage:{
- "parquet" :
+ "fs" :
{
- "type":"parquet",
- "dfsName" : "file:///"
+ "type":"file",
+ "connection" : "file:///"
}
},
query:[
@@ -19,10 +19,11 @@
@id:"1",
op:"scan",
memo:"initial_scan",
- storageengine:"parquet",
- selection: [
- &REPLACED_IN_PARQUET_TEST&
- ]
+ storageengine:"fs",
+ selection: {
+ format: {type: "parquet"}
+ files: [ &REPLACED_IN_PARQUET_TEST& ]
+ }
},
{
@id:"2",
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/parquet_selective_column_read.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_selective_column_read.json b/exec/java-exec/src/test/resources/parquet_selective_column_read.json
index 61e582d..515affc 100644
--- a/exec/java-exec/src/test/resources/parquet_selective_column_read.json
+++ b/exec/java-exec/src/test/resources/parquet_selective_column_read.json
@@ -13,12 +13,11 @@
entries : [ {
path : "/tmp/test.parquet"
} ],
- storageengine : {
- type : "parquet",
- dfsName : "file:///"
+ storage : {
+ type : "file",
+ connection : "file:///"
},
- columns: [ "integer", "bigInt", "bin", "bin2"],
- fragmentPointer : 0
+ columns: [ "integer", "bigInt", "bin", "bin2"]
}, {
pop : "screen",
@id : 2,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/physical_json_scan_test1.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/physical_json_scan_test1.json b/exec/java-exec/src/test/resources/physical_json_scan_test1.json
index 93bd966..5293d3b 100644
--- a/exec/java-exec/src/test/resources/physical_json_scan_test1.json
+++ b/exec/java-exec/src/test/resources/physical_json_scan_test1.json
@@ -9,16 +9,10 @@
graph:[
{
@id:1,
- pop:"json-scan",
- entries: [
- {
- path : "#{TEST_FILE}"
- }
- ],
- storageengine: {
- "type": "json",
- "dfsName": "file:///"
- }
+ pop:"fs-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files: [ "#{TEST_FILE}" ]
},
{
@id: 2,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/sender/broadcast_exchange.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/sender/broadcast_exchange.json b/exec/java-exec/src/test/resources/sender/broadcast_exchange.json
index 64f1606..408e35b 100644
--- a/exec/java-exec/src/test/resources/sender/broadcast_exchange.json
+++ b/exec/java-exec/src/test/resources/sender/broadcast_exchange.json
@@ -9,29 +9,17 @@
graph:[
{
@id:1,
- pop:"json-scan",
- entries: [
- {
- path : "#{LEFT_FILE}"
- }
- ],
- storageengine: {
- "type": "json",
- "dfsName": "file:///"
- }
+ pop:"fs-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files:["#{LEFT_FILE}"]
},
{
- @id:2,
- pop:"json-scan",
- entries: [
- {
- path : "#{RIGHT_FILE}"
- }
- ],
- storageengine: {
- "type": "json",
- "dfsName": "file:///"
- }
+ @id:2,
+ pop:"fs-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "file:///"},
+ files:["#{RIGHT_FILE}"]
},
{
@id: 3,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/storage-engines.json b/exec/java-exec/src/test/resources/storage-engines.json
new file mode 100644
index 0000000..7d3dc38
--- /dev/null
+++ b/exec/java-exec/src/test/resources/storage-engines.json
@@ -0,0 +1,20 @@
+{
+ "storage":{
+ dfs: {
+ type: "file",
+ connection: "file:///"
+ },
+ hive :
+ {
+ type:"hive",
+ config :
+ {
+ "hive.metastore.uris" : "",
+ "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true",
+ "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
+ "fs.default.name" : "file:///",
+ "hive.metastore.sasl.enabled" : "false"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 235d737..1bb58c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -107,6 +107,7 @@
<excludeSubprojects>false</excludeSubprojects>
<excludes>
+ <exclude>**/*.log</exclude>
<exclude>**/*.md</exclude>
<exclude>sandbox/**</exclude>
<exclude>**/*.json</exclude>
@@ -623,7 +624,7 @@
<dependency>
<groupId>net.hydromatic</groupId>
<artifactId>optiq-core</artifactId>
- <version>0.4.18-SNAPSHOT</version>
+ <version>0.4.18</version>
<exclusions>
<exclusion>
<groupId>org.jgrapht</groupId>
@@ -645,7 +646,7 @@
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
- <version>2.9.1</version>
+ <version>2.11.0</version>
</dependency>
<dependency>
<groupId>xalan</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/sqlparser/pom.xml
----------------------------------------------------------------------
diff --git a/sqlparser/pom.xml b/sqlparser/pom.xml
index c1b114f..90e044f 100644
--- a/sqlparser/pom.xml
+++ b/sqlparser/pom.xml
@@ -31,30 +31,10 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.drill</groupId>
- <artifactId>storage-hive</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </dependency>
- <dependency>
- <groupId>xalan</groupId>
- <artifactId>xalan</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.drill.exec</groupId>
@@ -72,7 +52,7 @@
<dependency>
<groupId>pentaho</groupId>
<artifactId>mondrian-data-foodmart-json</artifactId>
- <version>0.2</version>
+ <version>0.3.2</version>
</dependency>
<dependency>
@@ -89,12 +69,10 @@
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
- <version>2.9.1</version>
</dependency>
<dependency>
<groupId>xalan</groupId>
<artifactId>xalan</artifactId>
- <version>2.7.1</version>
</dependency>
</dependencies>
@@ -109,6 +87,7 @@
<exclude>**/.buildpath</exclude>
<exclude>**/*.json</exclude>
<exclude>**/donuts-output-data.txt</exclude>
+ <exclude>**/derby.log</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
deleted file mode 100644
index c02a478..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.nio.charset.Charset;
-import java.util.*;
-
-import net.hydromatic.linq4j.QueryProvider;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.linq4j.expressions.Expressions;
-import net.hydromatic.linq4j.expressions.MethodCallExpression;
-import net.hydromatic.optiq.BuiltinMethod;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.Table;
-import net.hydromatic.optiq.impl.java.JavaTypeFactory;
-
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.drill.exec.store.hive.HiveReadEntry;
-import org.apache.drill.exec.store.hive.HiveStorageEngine;
-import org.apache.drill.exec.store.hive.HiveStorageEngineConfig;
-import org.apache.drill.jdbc.DrillTable;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.sql.SqlCollation;
-import org.eigenbase.sql.type.SqlTypeName;
-
-public class HiveDatabaseSchema implements Schema{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveDatabaseSchema.class);
-
- private final JavaTypeFactory typeFactory;
- private final HiveSchema parentSchema;
- private final String name;
- private final Expression expression;
- private final QueryProvider queryProvider;
- private final HiveStorageEngine.HiveSchemaProvider schemaProvider;
- private final DrillClient client;
- private final HiveStorageEngineConfig config;
-
- public HiveDatabaseSchema(DrillClient client, HiveStorageEngineConfig config, HiveStorageEngine.HiveSchemaProvider schemaProvider,
- JavaTypeFactory typeFactory, HiveSchema parentSchema, String name,
- Expression expression, QueryProvider queryProvider) {
- super();
- this.client = client;
- this.typeFactory = typeFactory;
- this.parentSchema = parentSchema;
- this.name = name;
- this.expression = expression;
- this.queryProvider = queryProvider;
- this.schemaProvider = schemaProvider;
- this.config = config;
- }
-
- @Override
- public Schema getSubSchema(String name) {
- return null;
- }
-
- @Override
- public JavaTypeFactory getTypeFactory() {
- return typeFactory;
- }
-
- @Override
- public Schema getParentSchema() {
- return parentSchema;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Expression getExpression() {
- return expression;
- }
-
- @Override
- public QueryProvider getQueryProvider() {
- return queryProvider;
- }
-
- // TODO: Need to integrates UDFs?
- @Override
- public Collection<TableFunctionInSchema> getTableFunctions(String name) {
- return Collections.EMPTY_LIST;
- }
-
- // TODO: Need to integrates UDFs?
- @Override
- public Multimap<String, TableFunctionInSchema> getTableFunctions() {
- return ArrayListMultimap.create();
- }
-
- @Override
- /**
- * No more sub schemas within a database schema
- */
- public Collection<String> getSubSchemaNames() {
- return Collections.EMPTY_LIST;
- }
-
- private RelDataType getRelDataTypeFromHiveTypeString(String type) {
- switch(type) {
- case "boolean":
- return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
-
- case "tinyint":
- return typeFactory.createSqlType(SqlTypeName.TINYINT);
-
- case "smallint":
- return typeFactory.createSqlType(SqlTypeName.SMALLINT);
-
- case "int":
- return typeFactory.createSqlType(SqlTypeName.INTEGER);
-
- case "bigint":
- return typeFactory.createSqlType(SqlTypeName.BIGINT);
-
- case "float":
- return typeFactory.createSqlType(SqlTypeName.FLOAT);
-
- case "double":
- return typeFactory.createSqlType(SqlTypeName.DOUBLE);
-
- case "date":
- return typeFactory.createSqlType(SqlTypeName.DATE);
-
- case "timestamp":
- return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
-
- case "binary":
- return typeFactory.createSqlType(SqlTypeName.BINARY);
-
- case "decimal":
- return typeFactory.createSqlType(SqlTypeName.DECIMAL);
-
- case "string":
- case "varchar": {
- return typeFactory.createTypeWithCharsetAndCollation(
- typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
- Charset.forName("ISO-8859-1"), /*unicode char set*/
- SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
- );
- }
-
- default:
- throw new RuntimeException("Unknown or unsupported hive type: " + type);
- }
- }
-
- private RelDataType getRelDataTypeFromHivePrimitiveType(PrimitiveObjectInspector poi) {
- switch(poi.getPrimitiveCategory()) {
- case BOOLEAN:
- return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
- case BYTE:
- return typeFactory.createSqlType(SqlTypeName.TINYINT);
-
- case SHORT:
- return typeFactory.createSqlType(SqlTypeName.SMALLINT);
-
- case INT:
- return typeFactory.createSqlType(SqlTypeName.INTEGER);
-
- case LONG:
- return typeFactory.createSqlType(SqlTypeName.BIGINT);
-
- case FLOAT:
- return typeFactory.createSqlType(SqlTypeName.FLOAT);
-
- case DOUBLE:
- return typeFactory.createSqlType(SqlTypeName.DOUBLE);
-
- case DATE:
- return typeFactory.createSqlType(SqlTypeName.DATE);
-
- case TIMESTAMP:
- return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
-
- case BINARY:
- return typeFactory.createSqlType(SqlTypeName.BINARY);
-
- case DECIMAL:
- return typeFactory.createSqlType(SqlTypeName.DECIMAL);
-
- case STRING:
- case VARCHAR: {
- return typeFactory.createTypeWithCharsetAndCollation(
- typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
- Charset.forName("ISO-8859-1"), /*unicode char set*/
- SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
- );
- }
-
- case UNKNOWN:
- case VOID:
- default:
- throw new RuntimeException("Unknown or unsupported hive type");
- }
- }
-
- private RelDataType getRelDataTypeFromHiveType(ObjectInspector oi) {
- switch(oi.getCategory()) {
- case PRIMITIVE:
- return getRelDataTypeFromHivePrimitiveType(((PrimitiveObjectInspector) oi));
- case LIST:
- case MAP:
- case STRUCT:
- case UNION:
- default:
- throw new RuntimeException("Unknown or unsupported hive type");
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <E> Table<E> getTable(String name, Class<E> elementType) {
- Object selection = schemaProvider.getSelectionBaseOnName(String.format("%s.%s",this.name, name));
- if(selection == null) return null;
- org.apache.hadoop.hive.metastore.api.Table t = ((HiveReadEntry) selection).getTable();
- if (t == null) {
- logger.debug("Table name {} is invalid", name);
- return null;
- }
- org.apache.hadoop.hive.ql.metadata.Table hiveTable = new org.apache.hadoop.hive.ql.metadata.Table(t);
-
-
- final MethodCallExpression call = Expressions.call(getExpression(), //
- BuiltinMethod.DATA_CONTEXT_GET_TABLE.method, //
- Expressions.constant(name), //
- Expressions.constant(Object.class));
-
- ArrayList<RelDataType> typeList = new ArrayList<>();
- ArrayList<String> fieldNameList = new ArrayList<>();
-
- ArrayList<StructField> hiveFields = hiveTable.getFields();
- for(StructField hiveField : hiveFields) {
- fieldNameList.add(hiveField.getFieldName());
- typeList.add(getRelDataTypeFromHiveType(hiveField.getFieldObjectInspector()));
- }
-
- for (FieldSchema field : hiveTable.getPartitionKeys()) {
- fieldNameList.add(field.getName());
- typeList.add(getRelDataTypeFromHiveTypeString(field.getType()));
- }
-
- final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList);
- return (Table<E>) new DrillTable(
- client,
- this,
- Object.class,
- call,
- rowType,
- name,
- null /*storageEngineName*/,
- selection,
- config /*storageEngineConfig*/);
- }
-
- @Override
- public Map<String, TableInSchema> getTables() {
- Map<String, TableInSchema> tables = Maps.newHashMap();
-
- try {
- List<String> dbTables = parentSchema.getHiveDb().getAllTables(name);
- for(String table : dbTables) {
- TableInfo tableInfo = new TableInfo(this, table);
- tables.put(tableInfo.name, tableInfo);
- }
- } catch (HiveException ex) {
- throw new RuntimeException("Failed to get tables from HiveMetaStore", ex);
- }
-
- return tables;
- }
-
- private class TableInfo extends TableInSchema{
-
- public TableInfo(HiveDatabaseSchema schema, String tableName) {
- super(schema, schema.getName() + "." + tableName, TableType.TABLE);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <E> Table<E> getTable(Class<E> elementType) {
- if( !elementType.isAssignableFrom(DrillTable.class)) throw new UnsupportedOperationException();
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
deleted file mode 100644
index 007c6d9..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.util.*;
-import java.util.concurrent.ConcurrentMap;
-
-import net.hydromatic.linq4j.QueryProvider;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.Table;
-import net.hydromatic.optiq.impl.java.JavaTypeFactory;
-
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.drill.exec.store.hive.HiveStorageEngineConfig;
-import org.apache.drill.exec.store.hive.HiveStorageEngine.HiveSchemaProvider;
-
-import org.apache.drill.jdbc.DrillTable;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-
-public class HiveSchema implements Schema{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveSchema.class);
-
- private final JavaTypeFactory typeFactory;
- private final Schema parentSchema;
- private final String name;
- private final Expression expression;
- private final QueryProvider queryProvider;
- private final HiveSchemaProvider schemaProvider;
- private final DrillClient client;
- private final HiveStorageEngineConfig config;
- private Hive hiveDb;
-
- public HiveSchema(DrillClient client, StorageEngineConfig config, SchemaProvider schemaProvider,
- JavaTypeFactory typeFactory, Schema parentSchema, String name,
- Expression expression, QueryProvider queryProvider) {
- super();
- this.client = client;
- this.typeFactory = typeFactory;
- this.parentSchema = parentSchema;
- this.name = name;
- this.expression = expression;
- this.queryProvider = queryProvider;
- this.schemaProvider = (HiveSchemaProvider) schemaProvider;
- this.config = (HiveStorageEngineConfig) config;
- }
-
- public Hive getHiveDb() {
- try {
- if (hiveDb == null) {
- this.hiveDb = Hive.get(this.config.getHiveConf());
- }
- } catch(HiveException ex) {
- throw new RuntimeException("Failed to create Hive MetaStore Client", ex);
- }
-
- return hiveDb;
- }
-
- /**
- * fetch the database details from
- */
- @Override
- public Schema getSubSchema(String name) {
- return new HiveDatabaseSchema(client, config, schemaProvider, typeFactory, this,
- name, expression, queryProvider);
- }
-
- @Override
- public JavaTypeFactory getTypeFactory() {
- return typeFactory;
- }
-
- @Override
- public Schema getParentSchema() {
- return parentSchema;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Expression getExpression() {
- return expression;
- }
-
- @Override
- public QueryProvider getQueryProvider() {
- return queryProvider;
- }
-
- @Override
- public Collection<TableFunctionInSchema> getTableFunctions(String name) {
- return Collections.EMPTY_LIST;
- }
-
- @Override
- public Multimap<String, TableFunctionInSchema> getTableFunctions() {
- return ArrayListMultimap.create();
- }
-
- /**
- * return all databases in metastore
- */
- @Override
- public Collection<String> getSubSchemaNames() {
- try {
- return getHiveDb().getAllDatabases();
- } catch(HiveException ex) {
- throw new RuntimeException("Failed to get databases from Hive MetaStore", ex);
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <E> Table<E> getTable(String name, Class<E> elementType) {
- return null;
- }
-
- /**
- * Fetch the databases and tables in each database
- * @return
- */
- @Override
- public Map<String, TableInSchema> getTables() {
- Map<String, TableInSchema> allTables = Maps.newHashMap();
-
- Collection<String> dbs = getSubSchemaNames();
- for(String db : dbs) {
- // create a sub schema for each database and query the tables within it
- HiveDatabaseSchema dbSchema = new HiveDatabaseSchema(client, config, schemaProvider,
- typeFactory, this, db, expression, queryProvider);
- allTables.putAll(dbSchema.getTables());
- }
-
- return allTables;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
index aef4266..8810b6b 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
@@ -25,7 +25,7 @@ import java.util.Map;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties;
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.logical.data.CollapsingAggregate;
import org.apache.drill.common.logical.data.Filter;
import org.apache.drill.common.logical.data.Join;
@@ -178,8 +178,8 @@ public class JdbcTest {
Assert.assertEquals("na", planProperties.generator.info);
Assert.assertEquals(1, planProperties.version);
Assert.assertEquals(PlanProperties.PlanType.APACHE_DRILL_LOGICAL, planProperties.type);
- Map<String, StorageEngineConfig> seConfigs = plan.getStorageEngines();
- StorageEngineConfig config = seConfigs.get("donuts-json");
+ Map<String, StoragePluginConfig> seConfigs = plan.getStorageEngines();
+ StoragePluginConfig config = seConfigs.get("donuts-json");
// Assert.assertTrue(config != null && config instanceof ClasspathRSE.ClasspathRSEConfig);
config = seConfigs.get("queue");
// Assert.assertTrue(config != null && config instanceof QueueRSE.QueueRSEConfig);
@@ -234,8 +234,8 @@ public class JdbcTest {
Assert.assertEquals("na", planProperties.generator.info);
Assert.assertEquals(1, planProperties.version);
Assert.assertEquals(PlanProperties.PlanType.APACHE_DRILL_LOGICAL, planProperties.type);
- Map<String, StorageEngineConfig> seConfigs = plan.getStorageEngines();
- StorageEngineConfig config = seConfigs.get("donuts-json");
+ Map<String, StoragePluginConfig> seConfigs = plan.getStorageEngines();
+ StoragePluginConfig config = seConfigs.get("donuts-json");
// Assert.assertTrue(config != null && config instanceof ClasspathRSE.ClasspathRSEConfig);
config = seConfigs.get("queue");
// Assert.assertTrue(config != null && config instanceof QueueRSE.QueueRSEConfig);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index e6944cb..dc80f33 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.jdbc.test;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -26,6 +27,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.drill.common.util.TestTools;
import org.apache.drill.jdbc.Driver;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
@@ -38,34 +40,59 @@ public class TestJdbcQuery {
static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
// Set a timeout unless we're debugging.
- @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule();
+ @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(20000);
+ private static final String WORKING_PATH;
static{
Driver.load();
+ WORKING_PATH = Paths.get("").toAbsolutePath().toString();
}
+ @Test
+ public void testHiveRead() throws Exception{
+ testQuery("select * from hive.kv_text");
+ }
+
+ @Test
+ @Ignore // something not working here.
+ public void testHiveReadWithDb() throws Exception{
+ testQuery("select * from hive.`default`.kv_text");
+ }
+
+ @Test
+ public void testJsonQuery() throws Exception{
+ testQuery("select * from cp.`employee.json`");
+ }
+
+
@Test
public void testCast() throws Exception{
- testQuery("select R_REGIONKEY, cast(R_NAME as varchar(15)) as region, cast(R_COMMENT as varchar(255)) as comment from parquet.`/Users/jnadeau/region.parquet`");
+ testQuery(String.format("select R_REGIONKEY, cast(R_NAME as varchar(15)) as region, cast(R_COMMENT as varchar(255)) as comment from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH));
}
+
+ @Test
+ public void testWorkspace() throws Exception{
+ testQuery(String.format("select * from dfs.home.`%s/../sample-data/region.parquet`", WORKING_PATH));
+ }
+
@Test
public void testWildcard() throws Exception{
- testQuery("select * from parquet.`/Users/jnadeau/region.parquet`");
+ testQuery(String.format("select * from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH));
}
@Test
public void testLogicalExplain() throws Exception{
- testQuery("EXPLAIN PLAN WITHOUT IMPLEMENTATION FOR select * from parquet.`/Users/jnadeau/region.parquet`");
+ testQuery(String.format("EXPLAIN PLAN WITHOUT IMPLEMENTATION FOR select * from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH));
}
@Test
public void testPhysicalExplain() throws Exception{
- testQuery("EXPLAIN PLAN FOR select * from parquet.`/Users/jnadeau/region.parquet`");
+ testQuery(String.format("EXPLAIN PLAN FOR select * from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH));
}
@Test
public void checkUnknownColumn() throws Exception{
- testQuery("select unknownColumn from parquet.`/Users/jnadeau/region.parquet`");
+ testQuery(String.format("SELECT unknownColumn FROM dfs.`%s/../sample-data/region.parquet`", WORKING_PATH));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/sqlparser/src/test/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/resources/storage-engines.json b/sqlparser/src/test/resources/storage-engines.json
index da2f6c1..fca6b68 100644
--- a/sqlparser/src/test/resources/storage-engines.json
+++ b/sqlparser/src/test/resources/storage-engines.json
@@ -1,34 +1,20 @@
{
"storage":{
- "parquet-local" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- },
- "parquet-cp" :
- {
- "type":"parquet",
- "dfsName" : "classpath:///"
- },
- "jsonl" :
- {
- "type":"json",
- "dfsName" : "file:///"
- },
- "json-cp" :
- {
- "type":"json",
- "dfsName" : "classpath:///"
- },
- "parquet" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- },
- "hive-derby" :
- {
- "type":"hive",
- "config" :
+ dfs: {
+ type: "file",
+ connection: "file:///",
+ workspaces: {
+ default: "/",
+ home: "/"
+ }
+ },
+ cp: {
+ type: "file",
+ connection: "classpath:///"
+ },
+ hive : {
+ type:"hive",
+ config :
{
"hive.metastore.uris" : "",
"javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../sample-data/drill_hive_db;create=true",
@@ -38,5 +24,4 @@
}
}
}
-}
-
+}
\ No newline at end of file
[11/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/SimpleEvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/SimpleEvaluationVisitor.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/SimpleEvaluationVisitor.java
deleted file mode 100644
index 1b04880..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/SimpleEvaluationVisitor.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
-import org.apache.drill.common.expression.visitors.AggregateChecker;
-import org.apache.drill.common.expression.visitors.ConstantChecker;
-import org.apache.drill.common.expression.visitors.SimpleExprVisitor;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.eval.fn.FunctionArguments;
-import org.apache.drill.exec.ref.eval.fn.FunctionEvaluatorRegistry;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues.BooleanScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.DoubleScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.IntegerScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.LongScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.StringScalar;
-
-public class SimpleEvaluationVisitor extends SimpleExprVisitor<BasicEvaluator>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleEvaluationVisitor.class);
-
- private RecordPointer record;
- private List<AggregatingEvaluator> aggregators = new ArrayList<AggregatingEvaluator>();
-
- public SimpleEvaluationVisitor(RecordPointer record) {
- super();
- this.record = record;
- }
-
- public List<AggregatingEvaluator> getAggregators() {
- return aggregators;
- }
-
- @Override
- public BasicEvaluator visitFunctionCall(FunctionCall call) {
- List<BasicEvaluator> evals = new ArrayList<BasicEvaluator>();
- boolean includesAggregates = false;
- boolean onlyConstants = true;
- for(LogicalExpression e : call){
- if(AggregateChecker.isAggregating(e)) includesAggregates = true;
- if(!ConstantChecker.onlyIncludesConstants(e)) onlyConstants = false;
- evals.add(e.accept(this, null));
- }
- FunctionArguments args = new FunctionArguments(onlyConstants, includesAggregates, evals, call);
-
- if(call.getDefinition().isAggregating()){
- if(args.includesAggregates()) throw new SetupException(String.format("The call for %s contains one or more arguments that also contain aggregating functions. An aggregating function cannot contain aggregating expressions.", call.getDefinition()));
- BasicEvaluator e = FunctionEvaluatorRegistry.getEvaluator(call.getDefinition().getName(), args, record);
- if(!(e instanceof AggregatingEvaluator ) ){
- throw new SetupException(String.format("Function %s is an aggregating function. However, the provided evaluator (%s) is not an aggregating evaluator.", call.getDefinition(), e.getClass()));
- }
-
- aggregators.add( (AggregatingEvaluator) e);
- return e;
- }else{
- BasicEvaluator e = FunctionEvaluatorRegistry.getEvaluator(call.getDefinition().getName(), args, record);
- return e;
- }
- }
-
- @Override
- public BasicEvaluator visitIfExpression(IfExpression ifExpr) {
- return new IfEvaluator(ifExpr, this, record);
- }
-
- @Override
- public BasicEvaluator visitSchemaPath(SchemaPath path) {
- return new FieldEvaluator(path, record);
- }
-
- @Override
- public BasicEvaluator visitLongConstant(LongExpression longExpr) {
- return new LongScalar(longExpr.getLong());
- }
-
- @Override
- public BasicEvaluator visitDoubleConstant(DoubleExpression dExpr) {
- return new DoubleScalar(dExpr.getDouble());
- }
-
- @Override
- public BasicEvaluator visitBooleanConstant(BooleanExpression e) {
- return new BooleanScalar(e.getBoolean());
- }
-
- @Override
- public BasicEvaluator visitQuotedStringConstant(QuotedString e) {
- return new StringScalar(e.value);
- }
-
-
- @Override
- public BasicEvaluator visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
- throw new UnsupportedOperationException();
- }
-
- public static void main(String[] args) throws Exception {
- String expr = "if( a == 1) then 4 else 2 end";
- ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
- CommonTokenStream tokens = new CommonTokenStream(lexer);
- ExprParser parser = new ExprParser(tokens);
- LogicalExpression e = parser.parse().e;
- RecordPointer r = new UnbackedRecord();
- r.addField(new SchemaPath("a", ExpressionPosition.UNKNOWN), new IntegerScalar(3));
- SimpleEvaluationVisitor builder = new SimpleEvaluationVisitor(r);
- BasicEvaluator eval = e.accept(builder, null);
- DataValue v = eval.eval();
- System.out.println(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/ComparisonEvaluators.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/ComparisonEvaluators.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/ComparisonEvaluators.java
deleted file mode 100644
index 1b1b3af..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/ComparisonEvaluators.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn;
-
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.BaseBasicEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.exceptions.RecordException;
-import org.apache.drill.exec.ref.values.ComparableValue;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues.BooleanScalar;
-
-public class ComparisonEvaluators {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComparisonEvaluators.class);
-
- @FunctionEvaluator("equal")
- public static class EqualEvaluator extends BaseBasicEvaluator{
- private final BasicEvaluator left;
- private final BasicEvaluator right;
-
- public EqualEvaluator(RecordPointer record, FunctionArguments args){
- super(args.isOnlyConstants(), record);
- left = args.getEvaluator(0);
- right = args.getEvaluator(1);
- }
-
- @Override
- public BooleanScalar eval() {
- return new BooleanScalar(left.eval().equals(right.eval()));
- }
-
- }
-
- public static boolean isComparable(DataValue a, DataValue b) {
- return a instanceof ComparableValue && b instanceof ComparableValue && ((ComparableValue) a).supportsCompare(b);
- }
-
- private abstract static class ComparisonEvaluator extends BaseBasicEvaluator{
- private final BasicEvaluator left;
- private final BasicEvaluator right;
-
- public ComparisonEvaluator(RecordPointer record, FunctionArguments args){
- super(args.isOnlyConstants(), record);
- left = args.getEvaluator(0);
- right = args.getEvaluator(1);
- }
-
- public abstract boolean valid(int i);
-
- @Override
- public BooleanScalar eval() {
- DataValue a = left.eval();
- DataValue b = right.eval();
-
- if(isComparable(a, b)){
- int i = ((ComparableValue)a).compareTo(b);
- return new BooleanScalar(valid( i));
- }else{
- throw new RecordException(String.format("Values cannot be compared. A %s cannot be compared to a %s.", a, b), null);
- }
- }
- }
-
- @FunctionEvaluator("less than")
- public static class LessThan extends ComparisonEvaluator{
-
- public LessThan(RecordPointer record, FunctionArguments args) {
- super(record, args);
- }
-
- @Override
- public boolean valid(int i) {
- return i == -1;
- }
-
- }
-
- @FunctionEvaluator("greater than")
- public static class GreaterThan extends ComparisonEvaluator{
-
- public GreaterThan(RecordPointer record, FunctionArguments args) {
- super(record, args);
- }
-
- @Override
- public boolean valid(int i) {
- return i == 1;
- }
-
- }
-
- @FunctionEvaluator("greater than or equal to")
- public static class GreaterOrEqualTo extends ComparisonEvaluator{
-
- public GreaterOrEqualTo(RecordPointer record, FunctionArguments args) {
- super(record, args);
- }
-
- @Override
- public boolean valid(int i) {
- return i >= 0;
- }
-
- }
-
- @FunctionEvaluator("less than or equal to")
- public static class LessThanOrEqualTo extends ComparisonEvaluator{
-
- public LessThanOrEqualTo(RecordPointer record, FunctionArguments args) {
- super(record, args);
- }
-
- @Override
- public boolean valid(int i) {
- return i <= 0;
- }
-
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionArguments.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionArguments.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionArguments.java
deleted file mode 100644
index 47c0405..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionArguments.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-
-import com.carrotsearch.hppc.ObjectIntOpenHashMap;
-
-public class FunctionArguments {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionArguments.class);
-
- private final boolean onlyConstants;
- private final boolean includesAggregates;
- private final List<BasicEvaluator> ev;
- private ObjectIntOpenHashMap<String> nameToPosition = new ObjectIntOpenHashMap<String>();
-
- public FunctionArguments(boolean onlyConstants, boolean includesAggregates, List<BasicEvaluator> evaluators, FunctionCall call){
- this.onlyConstants = onlyConstants;
- this.includesAggregates = includesAggregates;
- String[] names = call.getDefinition().getArgumentNames();
- for(int i =0; i < names.length; i++){
- if(names[i] != null){
- nameToPosition.put(names[i], i);
- }
- }
- ev = evaluators;
- }
-
-
- public boolean includesAggregates() {
- return includesAggregates;
- }
-
-
- public boolean isOnlyConstants(){
- return onlyConstants;
- }
-
- public BasicEvaluator getEvaluator(String name){
- if(!nameToPosition.containsKey(name)) throw new RuntimeException("Unknown Item provided.");
- int i = nameToPosition.lget();
- return getEvaluator(i);
- }
-
- public BasicEvaluator getEvaluator(int argIndex){
- BasicEvaluator eval = ev.get(argIndex);
- if(eval == null) throw new RuntimeException("Unknown Item provided.");
- return eval;
-
- }
-
- /**
- * Return this single argument evaluator in this set of evaluators. If there isn't exactly one, throw a SetupException
- * @return The single evaluator.
- */
- public BasicEvaluator getOnlyEvaluator() throws SetupException{
- if(ev.size() != 1) throw new SetupException(String.format("Looking for a single argument. Received %d arguments.", ev.size()));
- return ev.get(0);
- }
-
- public int size(){
- return ev.size();
- }
-
- public BasicEvaluator[] getArgsAsArray(){
- return ev.toArray(new BasicEvaluator[size()]);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluator.java
deleted file mode 100644
index 03cee68..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-
-@Retention(RetentionPolicy.RUNTIME)
-public @interface FunctionEvaluator {
- String value();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluatorRegistry.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluatorRegistry.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluatorRegistry.java
deleted file mode 100644
index 363304a..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/FunctionEvaluatorRegistry.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.reflections.Reflections;
-import org.reflections.scanners.ResourcesScanner;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.scanners.TypeAnnotationsScanner;
-import org.reflections.util.ClasspathHelper;
-import org.reflections.util.ConfigurationBuilder;
-import org.reflections.util.FilterBuilder;
-
-public class FunctionEvaluatorRegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionEvaluatorRegistry.class);
-
- static Map<String, Constructor<? extends BasicEvaluator>> map;
-
- static {
- final String scanPackage = "org.apache.drill.exec";
-
- String s = FilterBuilder.Include.prefix(scanPackage);
-
- Reflections r = new Reflections(new ConfigurationBuilder().filterInputsBy(new FilterBuilder().include(s))
- .setUrls(ClasspathHelper.forPackage(scanPackage))
- .setScanners(new SubTypesScanner(), new TypeAnnotationsScanner(), new ResourcesScanner()));
-
- Set<Class<? extends BasicEvaluator>> providerClasses = r.getSubTypesOf(BasicEvaluator.class);
- Map<String, Constructor<? extends BasicEvaluator>> funcs = new HashMap<String, Constructor<? extends BasicEvaluator>>();
- for (Class<? extends BasicEvaluator> c : providerClasses) {
- try {
- FunctionEvaluator annotation = c.getAnnotation(FunctionEvaluator.class);
- if (annotation == null) {
- // only basic evaluator marked with a function evaluator interface will be examinged.
- continue;
- }
-
- Constructor<? extends BasicEvaluator> constructor = c.getConstructor(RecordPointer.class, FunctionArguments.class);
-// if (!constructor.isAccessible()) {
-// logger.warn("Unable to register Basic Evaluator {} because is not acccessible.", c);
-// continue;
-// }
-
- funcs.put(annotation.value(), constructor);
- } catch (NoSuchMethodException e) {
- logger.warn(
- "Unable to register Basic Evaluator {} because it does not have a constructor that accepts arguments of [SimpleRecord, ArgumentEvaluators] as its arguments.",
- c);
- } catch (SecurityException e) {
- logger.warn("Unable to register Basic Evaluator {} because of security exception.", c, e);
- }
- }
-
- map = Collections.unmodifiableMap(funcs);
- }
-
- public static BasicEvaluator getEvaluator(String name, FunctionArguments args, RecordPointer record) {
- Constructor<? extends BasicEvaluator> c = map.get(name);
- if (c == null) throw new SetupException(String.format("Unable to find requested basic evaluator %s.", name));
- try {
- try {
- BasicEvaluator e = c.newInstance(record, args);
- return e;
- } catch (InvocationTargetException e) {
- Throwable ex = e.getCause();
- if (ex instanceof SetupException) {
- throw (SetupException) ex;
- } else {
- if(ex instanceof RuntimeException){
- throw (RuntimeException) ex;
- }else{
- throw new SetupException(String.format("Failure while attempting to create a new evaluator of type '%s'.", name));
- }
- }
- }
- } catch (RuntimeException | IllegalAccessException | InstantiationException ex) {
- throw new SetupException(String.format("Failure while attempting to create a new evaluator of type '%s'.", name),
- ex);
- }
-
- }
-
- public static void main(String[] args) {
- System.out.println("loaded.");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/MathEvaluators.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/MathEvaluators.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/MathEvaluators.java
deleted file mode 100644
index 347a4f3..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/MathEvaluators.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn;
-
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.BaseBasicEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.exceptions.RecordException;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.NumericValue;
-import org.apache.drill.exec.ref.values.NumericValue.NumericType;
-import org.apache.drill.exec.ref.values.ScalarValues;
-
-public class MathEvaluators {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MathEvaluators.class);
-
- @FunctionEvaluator("add")
- public static class AddEvaluator extends BaseBasicEvaluator{
- private final BasicEvaluator args[];
-
- public AddEvaluator(RecordPointer record, FunctionArguments args){
- super(args.isOnlyConstants(), record);
- this.args = args.getArgsAsArray();
- }
-
- @Override
- public NumericValue eval() {
- NumericValue[] values = new NumericValue[args.length];
- for(int i =0; i < values.length; i++){
- DataValue v = args[i].eval();
- if(Types.isNumericType(v.getDataType())){
- values[i] = v.getAsNumeric();
- }
- }
- return NumericValue.add(values);
- }
-
-
- }
-
- @FunctionEvaluator("multiply")
- public static class MultiplyE extends BaseBasicEvaluator{
- private final BasicEvaluator args[];
-
- public MultiplyE(RecordPointer record, FunctionArguments args){
- super(args.isOnlyConstants(), record);
- this.args = args.getArgsAsArray();
- }
-
- @Override
- public NumericValue eval() {
- long l = 1;
- double d = 1;
- boolean isFloating = false;
-
- for(int i =0; i < args.length; i++){
- final DataValue v = args[i].eval();
-// logger.debug("DataValue {}", v);
- if(Types.isNumericType(v.getDataType())){
- NumericValue n = v.getAsNumeric();
- NumericType nt = n.getNumericType();
-// logger.debug("Numeric Type: {}", nt);
- if(isFloating || nt == NumericType.FLOAT || nt == NumericType.DOUBLE){
- if(!isFloating){
- d = l;
- isFloating = true;
- }
- d *= n.getAsDouble();
- }else{
- l *= n.getAsLong();
- }
-
- }else{
- throw new RecordException(String.format("Unable to multiply a value of %s.", v), null);
- }
- }
-
- NumericValue out = null;
- if(isFloating){
- out = new ScalarValues.DoubleScalar(d);
- }else{
- out = new ScalarValues.LongScalar(l);
- }
-
- return out;
- }
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/StringEvaluators.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/StringEvaluators.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/StringEvaluators.java
deleted file mode 100644
index 5a3ee02..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/StringEvaluators.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.BaseBasicEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.ValueReader;
-import org.apache.drill.exec.ref.values.ScalarValues.BooleanScalar;
-
-public class StringEvaluators {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StringEvaluators.class);
-
- @FunctionEvaluator("regex_like")
- public static class RegexEvaluator extends BaseBasicEvaluator{
- private final Matcher matcher;
- private final BasicEvaluator eval;
-
- public RegexEvaluator(RecordPointer record, FunctionArguments args){
- super(args.isOnlyConstants(), record);
- matcher = Pattern.compile(ValueReader.getString(args.getEvaluator("pattern").eval())).matcher("");
- eval = args.getEvaluator("value");
- }
-
- @Override
- public BooleanScalar eval() {
- matcher.reset(ValueReader.getChars(eval.eval()));
- return new BooleanScalar(matcher.find());
- }
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/AggregatingWrapperEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/AggregatingWrapperEvaluator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/AggregatingWrapperEvaluator.java
deleted file mode 100644
index b56e5f4..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/AggregatingWrapperEvaluator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn.agg;
-
-import java.util.List;
-
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.DataValue;
-
-public class AggregatingWrapperEvaluator implements AggregatingEvaluator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AggregatingWrapperEvaluator.class);
-
- private List<AggregatingEvaluator> aggs;
- private BasicEvaluator topEvaluator;
-
- public AggregatingWrapperEvaluator(List<AggregatingEvaluator> aggs, BasicEvaluator topEvaluator) {
- super();
- this.aggs = aggs;
- this.topEvaluator = topEvaluator;
- }
-
- @Override
- public DataValue eval() {
- return topEvaluator.eval();
- }
-
- @Override
- public boolean isConstant() {
- return topEvaluator.isConstant();
- }
-
- @Override
- public void addRecord() {
- for(AggregatingEvaluator a : aggs){
- a.addRecord();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/CountAggregator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/CountAggregator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/CountAggregator.java
deleted file mode 100644
index 247807b..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/CountAggregator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn.agg;
-
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.eval.fn.FunctionArguments;
-import org.apache.drill.exec.ref.eval.fn.FunctionEvaluator;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues;
-
-@FunctionEvaluator("count")
-public class CountAggregator implements AggregatingEvaluator{
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CountAggregator.class);
-
- boolean isConstant;
- long l = 0;
- boolean checkForNull = false;
- BasicEvaluator child;
-
- public CountAggregator(RecordPointer inputRecord, FunctionArguments e) {
- isConstant = e.isOnlyConstants();
- if(!e.getOnlyEvaluator().isConstant()){
- checkForNull = true;
- child = e.getOnlyEvaluator();
- }
- }
-
-
- @Override
- public void addRecord() {
- if(checkForNull){
- if(child.eval() != DataValue.NULL_VALUE){
- l++;
- }
- }else{
- l++;
- }
- }
-
- @Override
- public DataValue eval() {
- DataValue v = new ScalarValues.LongScalar(l);
- l = 0;
- return v;
- }
-
-
- @Override
- public boolean isConstant() {
- return isConstant;
- }
-
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/SumAggregator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/SumAggregator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/SumAggregator.java
deleted file mode 100644
index f1e01c0..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/fn/agg/SumAggregator.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval.fn.agg;
-
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.eval.fn.FunctionArguments;
-import org.apache.drill.exec.ref.eval.fn.FunctionEvaluator;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.NumericValue;
-import org.apache.drill.exec.ref.values.ScalarValues;
-
-@FunctionEvaluator("sum")
-public class SumAggregator implements AggregatingEvaluator {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SumAggregator.class);
- private boolean constantsOnly;
- private long l = 0;
- private double d = 0;
-
- boolean integer = true;
- private BasicEvaluator child;
-
- public SumAggregator(RecordPointer inputRecord, FunctionArguments e) {
- this.child = e.getOnlyEvaluator();
- this.constantsOnly = e.isOnlyConstants();
- }
-
- @Override
- public void addRecord() {
- DataValue dv = child.eval();
- NumericValue v = dv.getAsNumeric();
- if (integer) {
-
- switch (v.getNumericType()) {
- case DOUBLE:
- case FLOAT:
- integer = false;
- d = l; // loss of precision
- d += v.getAsDouble();
- break;
- case INT:
- case LONG:
- l += v.getAsLong();
- return;
- default:
- throw new UnsupportedOperationException();
- }
- }else{
- switch (v.getNumericType()) {
- case DOUBLE:
- case FLOAT:
- case INT:
- case LONG:
- integer = false;
- d += v.getAsDouble();
- return;
- default:
- throw new UnsupportedOperationException();
- }
-
- }
- }
-
- @Override
- public DataValue eval() {
- DataValue v;
- if (integer) {
- v = new ScalarValues.LongScalar(l);
- } else {
- v = new ScalarValues.DoubleScalar(d);
- }
- reset();
- return v;
- }
-
- private void reset() {
- l = 0;
- d = 0;
- integer = true;
- }
-
- @Override
- public boolean isConstant() {
- return constantsOnly;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/MajorException.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/MajorException.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/MajorException.java
deleted file mode 100644
index 393c825..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/MajorException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.exceptions;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-
-public class MajorException extends DrillRuntimeException{
- @SuppressWarnings("unused") static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MajorException.class);
-
- public MajorException() {
- super();
- }
-
- public MajorException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public MajorException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public MajorException(String message) {
- super(message);
- }
-
- public MajorException(Throwable cause) {
- super(cause);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/RecordException.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/RecordException.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/RecordException.java
deleted file mode 100644
index 4f8ccd6..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/RecordException.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.exceptions;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.ref.RecordPointer;
-
-public class RecordException extends DrillRuntimeException{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RecordException.class);
-
- private RecordPointer problemRecord;
-
- public RecordException(String message, RecordPointer problemRecord) {
- super(message);
- this.problemRecord = problemRecord;
- }
-
- public RecordException(String message, RecordPointer problemRecord, Throwable cause) {
- super(message, cause);
- this.problemRecord = problemRecord;
- }
-
- public RecordPointer getProblemRecord(){
- return problemRecord;
- }
-
- public void setProblemRecord(RecordPointer problemRecord) {
- this.problemRecord = problemRecord;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/SetupException.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/SetupException.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/SetupException.java
deleted file mode 100644
index cf44c82..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/exceptions/SetupException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.exceptions;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-
-public class SetupException extends DrillRuntimeException{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SetupException.class);
-
- public SetupException() {
- super();
- }
-
- public SetupException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-
- public SetupException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public SetupException(String message) {
- super(message);
- }
-
- public SetupException(Throwable cause) {
- super(cause);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/AbstractBlockingOperator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/AbstractBlockingOperator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/AbstractBlockingOperator.java
deleted file mode 100644
index abbdc5c..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/AbstractBlockingOperator.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.logical.data.SingleInputOperator;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.RecordIterator.NextOutcome;
-
-
-public abstract class AbstractBlockingOperator<T extends SingleInputOperator> extends SingleInputROPBase<T> {
-
- public AbstractBlockingOperator(T config) {
- super(config);
- }
-
- private RecordIterator incoming;
- protected RecordPointer inputRecord;
- protected final ProxySimpleRecord outputRecord = new ProxySimpleRecord();
-
- @Override
- public void setInput(RecordIterator incoming) {
- this.incoming = incoming;
- inputRecord = incoming.getRecordPointer();
- }
-
- protected abstract void consumeRecord();
- protected abstract RecordIterator doWork();
-
- private RecordIterator consumeData(){
- while (incoming.next() != NextOutcome.NONE_LEFT) {
- consumeRecord();
- }
- return doWork();
- }
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return new BlockingIterator();
- }
-
- private class BlockingIterator implements RecordIterator{
- private RecordIterator iter;
-
- public BlockingIterator(){
- }
-
- @Override
- public NextOutcome next() {
- if(this.iter == null){
- this.iter = consumeData();
- }
- return this.iter.next();
- }
-
- @Override
- public ROP getParent() {
- return AbstractBlockingOperator.this;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return outputRecord;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BaseSinkROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BaseSinkROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BaseSinkROP.java
deleted file mode 100644
index 4b46f41..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BaseSinkROP.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.io.IOException;
-import java.util.concurrent.locks.ReadWriteLock;
-
-import org.apache.drill.common.logical.data.SinkOperator;
-import org.apache.drill.exec.ref.RunOutcome;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
-import org.apache.drill.exec.ref.RecordIterator.NextOutcome;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-
-public abstract class BaseSinkROP<T extends SinkOperator> extends SingleInputROPBase<T> implements SinkROP{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseSinkROP.class);
-
- protected RecordIterator iter;
- protected RecordPointer record;
-
- public BaseSinkROP(T config) {
- super(config);
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) throws SetupException {
- try {
- setupSink();
- } catch (IOException e) {
- throw new SetupException(String.format("failure setting up %s sink rop.", this.getClass()), e);
- }
- }
-
- @Override
- protected void setInput(RecordIterator incoming) {
- iter = incoming;
- record = incoming.getRecordPointer();
- }
-
- @Override
- public RecordIterator getIteratorInternal() {
- throw new UnsupportedOperationException("A ReferenceSink");
- }
-
- @Override
- public RunOutcome run(StatusHandle handle) {
- Throwable exception = null;
- final int runsize = 1000;
- int recordCount = 0;
- OutcomeType outcome = OutcomeType.FAILED;
- long pos = -1;
- try{
- while(true){
- boolean more = true;
- for(;recordCount < runsize; recordCount++){
- NextOutcome r = iter.next();
- if(r == NextOutcome.NONE_LEFT){
- more = false;
- break;
- }else{
- pos = sinkRecord(record);
- }
- }
- handle.progress(pos, recordCount);
- if(!handle.okToContinue()){
- logger.debug("Told to cancel, breaking run.");
- outcome = OutcomeType.CANCELED;
- break;
- }else if(!more){
- outcome = OutcomeType.SUCCESS;
- logger.debug("Breaking because no more records were found.");
- break;
- }else{
- logger.debug("No problems, doing next progress iteration.");
- }
-
- }
- }catch(Exception e){
- exception = e ;
- }
-
- cleanup(outcome);
- return new RunOutcome(outcome, pos, recordCount, exception);
-
- }
-
- /**
- *
- * @param r RecordPointer to record
- * @return The approximate amount of bytes written.
- * @throws IOException
- */
- public abstract long sinkRecord(RecordPointer r) throws IOException;
- protected abstract void setupSink() throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BoundaryListener.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BoundaryListener.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BoundaryListener.java
deleted file mode 100644
index 4d56885..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/BoundaryListener.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.expression.SchemaPath;
-
-public interface BoundaryListener {
- public SchemaPath[] getOrderedBoundaryPaths();
- public void markBoundaryAsCrossed();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/CollapsingAggregateROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/CollapsingAggregateROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/CollapsingAggregateROP.java
deleted file mode 100644
index 030efbb..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/CollapsingAggregateROP.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.CollapsingAggregate;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues;
-
-public class CollapsingAggregateROP extends SingleInputROPBase<CollapsingAggregate> implements BoundaryListener {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CollapsingAggregateROP.class);
-
- private SchemaPath[] boundaryPaths;
- private BasicEvaluator targetEvaluator;
- private boolean foundTarget = false;
- private DataValue currentValue;
- private DataValue previousValue;
- private BasicEvaluator boundaryKey;
- private RecordIterator incoming;
- private BasicEvaluator[] carryovers;
- private FieldReference[] carryoverNames;
- private DataValue[] carryoverValues;
- private AggregatingEvaluator[] aggs;
- private SchemaPath[] aggNames;
- private boolean boundaryCrossed = false;
- private final boolean targetMode;
-
- public CollapsingAggregateROP(CollapsingAggregate config) {
- super(config);
- targetMode = config.getTarget() != null;
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) {
-
- if(config.getWithin() != null){
- boundaryPaths = new SchemaPath[]{config.getWithin()};
- boundaryKey = builder.getBasicEvaluator(record, config.getWithin());
- }else{
- boundaryPaths = new SchemaPath[0];
- boundaryKey = new ScalarValues.IntegerScalar(0);
- }
-
- aggs = new AggregatingEvaluator[config.getAggregations().length];
- carryovers = new BasicEvaluator[config.getCarryovers().length];
- carryoverNames = new FieldReference[config.getCarryovers().length];
- carryoverValues = new DataValue[config.getCarryovers().length];
-
- if(targetMode){
- targetEvaluator = builder.getBasicEvaluator(record, config.getTarget());
- }
- aggNames = new SchemaPath[aggs.length];
- for(int i =0; i < aggs.length; i++){
- aggs[i] = builder.getAggregatingOperator(record, config.getAggregations()[i].getExpr());
- aggNames[i] = config.getAggregations()[i].getRef();
- }
-
- for(int i =0; i < carryovers.length; i++){
- carryovers[i] = builder.getBasicEvaluator(record, config.getCarryovers()[i]);
- carryoverNames[i] = config.getCarryovers()[i];
- }
-
-
-
- }
-
- @Override
- protected void setInput(RecordIterator incoming) {
- this.incoming = incoming;
- }
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return new AggregatingIterator();
- }
-
- private boolean checkBoundaryCrossing() {
- if (boundaryCrossed) {
- boundaryCrossed = false;
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void markBoundaryAsCrossed() {
- this.boundaryCrossed = true;
- }
-
-
-
-
- private class AggregatingIterator implements RecordIterator {
- private boolean remainder = false;
- private boolean more = true;
- private UnbackedRecord outputRecord = new UnbackedRecord();
- public AggregatingIterator() {
- }
-
- private NextOutcome readNext(){
-
- // copy over the previous values.
- previousValue = currentValue;
-
- // increment the parent forward.
- NextOutcome n = incoming.next();
-
- boolean changed = false;
-
- // read boundary values unless no new values were loaded.
- if(n != NextOutcome.NONE_LEFT){
- currentValue = boundaryKey.eval();
- if(!currentValue.equals(previousValue)) changed = true;
-
- // skip first boundary.
- if(previousValue == null) changed = false;
-
- }else{
-
- changed = true;
- }
-
- if(changed){
- markBoundaryAsCrossed();
- }
-
- return n;
- }
-
- private void consumeCurrent(){
- for(int x = 0; x < aggs.length; x++){
- aggs[x].addRecord();
- }
-
- // if we're in target mode and this row matches the target criteria, we're going to copy carry over values and mark foundTarget = true.
- if(targetMode){
- DataValue v = targetEvaluator.eval();
- if(v.getDataType().getMinorType() == MinorType.BIT && v.getAsBooleanValue().getBoolean()){
- foundTarget = true;
- for(int i =0 ; i < carryovers.length; i++){
- carryoverValues[i] = carryovers[i].eval();
- }
- }
- }else{
- for(int i =0 ; i < carryovers.length; i++){
- carryoverValues[i] = carryovers[i].eval();
- }
- }
- }
-
- /**
- * Write the output of the operation to the unbacked record.
- * @return Whether or not the record should be included in the output.
- */
- private boolean writeOutputRecord(){
- outputRecord.clear();
- for(int x = 0; x < aggs.length; x++){
- DataValue dv = aggs[x].eval();
-// logger.debug("Adding Aggregated Values named {} with value {}", outputNames[x], dv);
- outputRecord.addField(aggNames[x], dv);
- }
-
- // Add the carryover keys.
- if(targetMode){
- if(foundTarget){
- for(int y = 0; y < carryoverNames.length; y++){
- outputRecord.addField(carryoverNames[y], carryoverValues[y]);
- }
- foundTarget = false;
- return true;
- }else{
- return false;
- }
- }else{
- for(int y = 0; y < carryoverNames.length; y++){
- outputRecord.addField(carryoverNames[y], carryoverValues[y]);
- }
- return true;
- }
-
-
-
-
- }
-
- @Override
- public NextOutcome next() {
- outside: while(true){
- NextOutcome whatNext = null;
-
- // if we don't have more and there are no more values, exit.
- if (!more) return NextOutcome.NONE_LEFT;
-
- while (true) {
-
- // we shouldn't increment the iterator since we have to consume our remainder.
- if(!remainder){
- whatNext = readNext();
-
- // if we've just crossed a boundary, we should output the previous values.
- if(checkBoundaryCrossing()){
- boolean wroteSomething = writeOutputRecord();
-
- // if there is no future input, we'll flag as !more. Otherwise, will inform that there are pending records that should be consumed.
- if(whatNext == NextOutcome.NONE_LEFT){
- more = false;
- }else{
- remainder = true;
- }
-
- // if we didn't write something, we'll retry the outside loop
- if(!wroteSomething) continue outside;
-
- // always return a next outcome of true since we've just output a record.
- return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
-
- }
- }
-
- // we don't consume the current record until after we've output a record associated with a boundary (as necessary).
- consumeCurrent();
- remainder = false;
- }
-
- }
- }
-
- @Override
- public ROP getParent() {
- return CollapsingAggregateROP.this;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return outputRecord;
- }
-
- }
-
- @Override
- public SchemaPath[] getOrderedBoundaryPaths() {
- return boundaryPaths;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ConstantROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ConstantROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ConstantROP.java
deleted file mode 100644
index 1cfe8a8..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ConstantROP.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.util.Iterator;
-
-import org.apache.drill.common.logical.data.Constant;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.apache.drill.exec.ref.rse.JSONRecordReader;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-public class ConstantROP extends ROPBase<Constant>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanROP.class);
-
- private UnbackedRecord record;
-
- public ConstantROP(Constant config) {
- super(config);
- record = new UnbackedRecord();
- }
-
-
- @Override
- protected void setupIterators(IteratorRegistry registry) throws SetupException {
- // try{
- super.setupIterators(registry);
- // need to assign reader
- // throw new IOException();
- //}catch(IOException e){
- //throw new SetupException("Failure while setting up reader.");
- //}
- }
-
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return new ConstantIterator(ConstantROP.this.config.getContent().getRoot());
- }
-
-
- @Override
- public void cleanup(OutcomeType outcome) {
- super.cleanup(outcome);
- }
-
-
- class ConstantIterator implements RecordIterator {
-
- Iterator<JsonNode> jsonIter;
-
- ConstantIterator(JsonNode json) {
- jsonIter = json.elements();
- }
-
- public RecordPointer getRecordPointer(){
- return record;
- }
-
- public NextOutcome next(){
- if ( ! jsonIter.hasNext()){
- return NextOutcome.NONE_LEFT;
- }
- JsonNode contentJSON = ConstantROP.this.config.getContent().getRoot();
- if (contentJSON.isArray())
- { // list of constant records was specified
- JsonNode node;
- node = jsonIter.next();
- convertJsonToRP(node, record);
- return NextOutcome.INCREMENTED_SCHEMA_UNCHANGED;
- }
- else{
- convertJsonToRP(contentJSON, record);
- return NextOutcome.NONE_LEFT;
- }
- }
-
- private void convertJsonToRP(JsonNode node, RecordPointer rp){
- record.clear();
- record.merge(JSONRecordReader.convert(node));
- }
-
- public ROP getParent(){
- return ConstantROP.this;
- }
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/DataWriter.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/DataWriter.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/DataWriter.java
deleted file mode 100644
index 5c2a908..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/DataWriter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.io.IOException;
-
-public interface DataWriter {
- public void startRecord() throws IOException;
- public void writeArrayStart(int length) throws IOException;
- public void writeArrayElementStart() throws IOException;
- public void writeArrayElementEnd() throws IOException;
- public void writeArrayEnd() throws IOException;
-
- public void writeMapStart() throws IOException;
- public void writeMapKey(CharSequence seq) throws IOException;
- public void writeMapValueStart() throws IOException;
- public void writeMapValueEnd() throws IOException;
- public void writeMapEnd() throws IOException;
-
- public void writeBoolean(boolean b) throws IOException;
- public void writeSInt32(int value) throws IOException;
- public void writeSInt64(long value) throws IOException;
- public void writeBytes(byte[] bytes) throws IOException;
- public void writeSFloat64(double value) throws IOException;
- public void writeSFloat32(float value) throws IOException;
- public void writeNullValue() throws IOException;
- public void writeCharSequence(CharSequence value) throws IOException;
- public void endRecord() throws IOException;
- public void finish() throws IOException;
-
- public enum ConverterType{
- JSON
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FilterROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FilterROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FilterROP.java
deleted file mode 100644
index ff1fd05..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FilterROP.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.logical.data.Filter;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BooleanEvaluator;
-
-public class FilterROP extends SingleInputROPBase<Filter>{
-
- private FilterIterator iter;
- private BooleanEvaluator filterEval;
-
- public FilterROP(Filter config) {
- super(config);
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) {
- filterEval = builder.getBooleanEvaluator(record, config.getExpr());
- }
-
- @Override
- public void setInput(RecordIterator incoming) {
- iter = new FilterIterator(incoming);
- }
-
- @Override
- public RecordIterator getIteratorInternal() {
- return iter;
- }
-
- private class FilterIterator implements RecordIterator{
- RecordIterator incoming;
-
- public FilterIterator(RecordIterator incoming) {
- this.incoming = incoming;
- }
-
- @Override
- public NextOutcome next() {
- NextOutcome r;
- while(true){
- r = incoming.next();
- if(r == NextOutcome.NONE_LEFT) return NextOutcome.NONE_LEFT;
- if(filterEval.eval()) return r;
- }
- }
-
- @Override
- public ROP getParent() {
- return FilterROP.this;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return record;
- }
-
-
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FlattenROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FlattenROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FlattenROP.java
deleted file mode 100644
index db7c916..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/FlattenROP.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.Flatten;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.BaseArrayValue;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.SimpleArrayValue;
-
-
-public class FlattenROP extends SingleInputROPBase<Flatten> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlattenROP.class);
-
- private RecordPointer outputRecord = new UnbackedRecord();
- private BasicEvaluator evaluator;
- private RecordIterator iter;
-
- public FlattenROP(Flatten config) {
- super(config);
- }
-
- @Override
- protected void setInput(RecordIterator incoming) {
- this.iter = new FlattenIterator(incoming);
- }
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return iter;
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) {
- evaluator = builder.getBasicEvaluator(record, config.getExpr());
- }
-
- private class ArrayValueIterator {
- private BaseArrayValue arrayValue;
- private int currentIndex = 0;
-
- public ArrayValueIterator(BaseArrayValue arrayValue) {
- this.arrayValue = arrayValue;
- }
-
- public ArrayValueIterator() {
- this(new SimpleArrayValue());
- }
-
- public DataValue next() {
- DataValue v = null;
- if (currentIndex < arrayValue.size()) {
- v = arrayValue.getByArrayIndex(currentIndex);
- }
-
- currentIndex++;
- return v;
-
- }
- }
-
- private class FlattenIterator implements RecordIterator {
- RecordIterator incoming;
- NextOutcome currentOutcome;
- int currentIndex = 0;
- ArrayValueIterator arrayValueIterator = new ArrayValueIterator();
-
- public FlattenIterator(RecordIterator incoming) {
- super();
- this.incoming = incoming;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return outputRecord;
- }
-
- @Override
- public NextOutcome next() {
- DataValue v;
- if ((v = arrayValueIterator.next()) != null) //if we are already iterating through a sub-array, keep going
- return mergeValue(v);
- else //otherwise, get the next record
- currentOutcome = incoming.next();
-
-
- if (currentOutcome != NextOutcome.NONE_LEFT) {
- if (evaluator.eval().getDataType().getMode() == DataMode.REPEATED) {
- arrayValueIterator = new ArrayValueIterator(evaluator.eval().getAsContainer().getAsArray());
-
- while ((v = arrayValueIterator.next()) != null) {
- return mergeValue(v);
- }
- } else {
- outputRecord.copyFrom(record);
- outputRecord.addField(config.getName(), evaluator.eval());
- if(config.isDrop())
- outputRecord.removeField((SchemaPath)config.getExpr());
- }
- }
- return currentOutcome;
- }
-
- // helper function to merge one of the values from a sub array into the parent record
- private NextOutcome mergeValue(DataValue v) {
- outputRecord.copyFrom(record);
- outputRecord.addField(config.getName(), v);
- if(config.isDrop())
- outputRecord.removeField((SchemaPath)config.getExpr());
- return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
- }
-
- @Override
- public ROP getParent() {
- return FlattenROP.this;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/JoinROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/JoinROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/JoinROP.java
deleted file mode 100644
index 2b46f1a..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/JoinROP.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.logical.data.Join;
-import org.apache.drill.common.logical.data.JoinCondition;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.eval.fn.ComparisonEvaluators;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.apache.drill.exec.ref.values.ComparableValue;
-import org.apache.drill.exec.ref.values.DataValue;
-
-import java.util.List;
-
-public class JoinROP extends ROPBase<Join> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JoinROP.class);
-
- private RecordIterator left;
- private RecordIterator right;
- private UnbackedRecord record;
- private EvaluatorFactory factory;
-
- public JoinROP(Join config) {
- super(config);
- record = new UnbackedRecord();
- }
-
- @Override
- protected void setupIterators(IteratorRegistry builder) {
- left = Iterables.getOnlyElement(builder.getOperator(config.getLeft()));
- right = Iterables.getOnlyElement(builder.getOperator(config.getRight()));
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) throws SetupException {
- factory = builder;
- }
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return createIteratorFromJoin(config.getJointType());
- }
-
- private RecordIterator createIteratorFromJoin(Join.JoinType type) {
- switch (type) {
- case LEFT:
- return new LeftIterator();
- case INNER:
- return new InnerIterator();
- case OUTER:
- return new OuterIterator();
- default:
- throw new UnsupportedOperationException("Type not supported: " + type);
- }
- }
-
- private class RecordBuffer {
- final boolean schemaChanged;
- final RecordPointer pointer;
- boolean hasJoined = false;
-
- private RecordBuffer(RecordPointer pointer, boolean schemaChanged) {
- this.pointer = pointer;
- this.schemaChanged = schemaChanged;
- }
-
- public void setHasJoined(boolean hasJoined) {
- this.hasJoined = hasJoined;
- }
- }
-
- abstract class JoinIterator implements RecordIterator {
- protected List<RecordBuffer> buffer;
- protected int curIdx = 0;
- protected int bufferLength = 0;
-
- protected abstract int setupBuffer();
-
- protected int setupBufferForIterator(RecordIterator iterator) {
- int count = 0;
- NextOutcome outcome = iterator.next();
- while (outcome != NextOutcome.NONE_LEFT) {
- buffer.add(new RecordBuffer(
- iterator.getRecordPointer().copy(),
- outcome == NextOutcome.INCREMENTED_SCHEMA_CHANGED)
- );
- ++count;
- outcome = iterator.next();
- }
- return count;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return record;
- }
-
- public NextOutcome next() {
- if (buffer == null) {
- buffer = Lists.newArrayList();
- setupBuffer();
- bufferLength = buffer.size();
- }
- return getNext();
- }
-
- public abstract NextOutcome getNext();
-
- protected void setOutputRecord(RecordPointer... inputs) {
- boolean first = true;
- for(RecordPointer input : inputs) {
- if(input == null) {
- continue;
- }
-
- if(first) {
- first = false;
- record.copyFrom(input);
- } else {
- record.merge(input);
- }
- }
- }
-
- public boolean eval(DataValue leftVal, DataValue rightVal, String relationship) {
- // Skip join if no comparison can be made
- if (!ComparisonEvaluators.isComparable(leftVal, rightVal)) {
- return false;
- }
-
- //Somehow utilize ComparisonEvaluators?
- switch (relationship) {
- case "!=":
- return !leftVal.equals(rightVal);
- case "==":
- return leftVal.equals(rightVal);
- case "<":
- return ((ComparableValue) leftVal).compareTo(rightVal) < 0;
- case "<=":
- return ((ComparableValue) leftVal).compareTo(rightVal) <= 0;
- case ">":
- return ((ComparableValue) leftVal).compareTo(rightVal) > 0;
- case ">=":
- return ((ComparableValue) leftVal).compareTo(rightVal) >= 0;
- default:
- throw new DrillRuntimeException("Relationship not supported: " + relationship);
- }
- }
-
- @Override
- public ROP getParent() {
- return JoinROP.this;
- }
- }
-
- class InnerIterator extends JoinIterator {
- NextOutcome rightOutcome;
-
- @Override
- protected int setupBuffer() {
- return setupBufferForIterator(left);
- }
-
- @Override
- public NextOutcome getNext() {
- final RecordPointer rightPointer = right.getRecordPointer();
- while (true) {
- if (curIdx == 0) {
- rightOutcome = right.next();
-
- if (rightOutcome == NextOutcome.NONE_LEFT) {
- break;
- }
- }
-
- final RecordBuffer bufferObj = buffer.get(curIdx++);
- Optional<JoinCondition> option = Iterables.tryFind(Lists.newArrayList(config.getConditions()), new Predicate<JoinCondition>() {
- @Override
- public boolean apply(JoinCondition condition) {
- return eval(factory.getBasicEvaluator(rightPointer, condition.getRight()).eval(),
- factory.getBasicEvaluator(bufferObj.pointer, condition.getLeft()).eval(), condition.getRelationship());
- }
- });
-
- if (curIdx >= bufferLength) {
- curIdx = 0;
- }
-
- if (option.isPresent()) {
- setOutputRecord(rightPointer, bufferObj.pointer);
- return (bufferObj.schemaChanged || rightOutcome == NextOutcome.INCREMENTED_SCHEMA_CHANGED) ?
- NextOutcome.INCREMENTED_SCHEMA_CHANGED :
- NextOutcome.INCREMENTED_SCHEMA_UNCHANGED;
- }
-
- }
-
- return NextOutcome.NONE_LEFT;
- }
- }
-
- class LeftIterator extends JoinIterator {
- private NextOutcome leftOutcome;
-
- @Override
- protected int setupBuffer() {
- return setupBufferForIterator(right);
- }
-
- @Override
- public NextOutcome getNext() {
- final RecordPointer leftPointer = left.getRecordPointer();
- boolean isFound = true;
- if(curIdx >= bufferLength) {
- return NextOutcome.NONE_LEFT;
- }
-
- while (true) {
- if (curIdx == 0) {
- if (!isFound) {
- setOutputRecord(leftPointer);
- return leftOutcome;
- }
-
- leftOutcome = left.next();
-
- if (leftOutcome == NextOutcome.NONE_LEFT) {
- break;
- }
-
- isFound = false;
- }
-
- final RecordBuffer bufferObj = buffer.get(curIdx++);
- Optional<JoinCondition> option = Iterables.tryFind(Lists.newArrayList(config.getConditions()), new Predicate<JoinCondition>() {
- @Override
- public boolean apply(JoinCondition condition) {
- return eval(factory.getBasicEvaluator(leftPointer, condition.getLeft()).eval(),
- factory.getBasicEvaluator(bufferObj.pointer, condition.getRight()).eval(), condition.getRelationship());
- }
- });
-
- if (option.isPresent()) {
- setOutputRecord(leftPointer, bufferObj.pointer);
- bufferObj.setHasJoined(true);
- return (bufferObj.schemaChanged || leftOutcome == NextOutcome.INCREMENTED_SCHEMA_CHANGED) ?
- NextOutcome.INCREMENTED_SCHEMA_CHANGED :
- NextOutcome.INCREMENTED_SCHEMA_UNCHANGED;
- }
-
- if (curIdx >= bufferLength) {
- curIdx = 0;
- }
- }
-
- return NextOutcome.NONE_LEFT;
- }
- }
-
- class OuterIterator extends LeftIterator {
- boolean innerJoinCompleted = false;
-
- @Override
- public NextOutcome getNext() {
- if (innerJoinCompleted && curIdx >= bufferLength) {
- return NextOutcome.NONE_LEFT;
- }
-
- if (!innerJoinCompleted) {
- NextOutcome outcome = super.getNext();
- if (outcome != NextOutcome.NONE_LEFT) {
- return outcome;
- } else {
- innerJoinCompleted = true;
- curIdx = 0;
- }
- }
-
- if (innerJoinCompleted) {
- while (curIdx < bufferLength) {
- RecordBuffer recordBuffer = buffer.get(curIdx++);
- if (!recordBuffer.hasJoined) {
- setOutputRecord(recordBuffer.pointer, null);
- return recordBuffer.schemaChanged ? NextOutcome.INCREMENTED_SCHEMA_CHANGED : NextOutcome.INCREMENTED_SCHEMA_UNCHANGED;
- }
- }
- }
- return NextOutcome.NONE_LEFT;
- }
- }
-}
[30/38] git commit: DRILL-363: Custom null handling in hash functions
Posted by ja...@apache.org.
DRILL-363: Custom null handling in hash functions
Added custom null handling for hash functions, to override the privies NULL_IF_NULL handling. Added Integer and Float literals to the logical expression package and corresponding methods in the various visitor implementations. Also added new Hash functions for double and float values based on the doubleToLongBits and floatToIntBits methods in the corresponding boxed primitive classes together with the previously used murmur3_128 hash algorithm used for plain Ints and Longs.
Signed-off-by: Jacques Nadeau <ja...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b7f9e600
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b7f9e600
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b7f9e600
Branch: refs/heads/master
Commit: b7f9e600fa31ac7b9365b31396730719b8334469
Parents: 153c727
Author: Jason Altekruse <al...@gmial.com>
Authored: Sun Feb 9 13:43:42 2014 -0600
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../common/expression/ExpressionValidator.java | 10 ++
.../common/expression/ValueExpressions.java | 63 +++++++++
.../visitors/AbstractExprVisitor.java | 12 ++
.../expression/visitors/AggregateChecker.java | 12 ++
.../expression/visitors/ConstantChecker.java | 12 ++
.../common/expression/visitors/ExprVisitor.java | 7 +-
.../expression/visitors/SimpleExprVisitor.java | 14 ++
.../sig/ConstantExpressionIdentifier.java | 14 +-
.../exec/expr/ExpressionTreeMaterializer.java | 10 ++
.../drill/exec/expr/fn/impl/HashFunctions.java | 128 +++++++++++++++++--
.../apache/drill/exec/expr/fn/impl/IsNull.java | 2 +-
.../exec/resolver/ResolverTypePrecedence.java | 56 ++++----
.../drill/exec/resolver/TypeCastRules.java | 30 +++--
.../exec/physical/impl/TestSimpleFunctions.java | 105 ++++++++++++++-
14 files changed, 426 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
index 11d97e8..076c150 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
@@ -75,6 +75,16 @@ public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, Ru
}
@Override
+ public Void visitIntConstant(ValueExpressions.IntExpression intExpr, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitFloatConstant(ValueExpressions.FloatExpression fExpr, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
public Void visitLongConstant(LongExpression intExpr, ErrorCollector errors) throws RuntimeException {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
index 49f3773..5156786 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -120,6 +120,69 @@ public class ValueExpressions {
}
+ public static class FloatExpression extends LogicalExpressionBase {
+ private float f;
+
+ private static final MajorType FLOAT_CONSTANT = Types.required(MinorType.FLOAT4);
+
+ public FloatExpression(float f, ExpressionPosition pos) {
+ super(pos);
+ this.f = f;
+ }
+
+ public float getFloat() {
+ return f;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return FLOAT_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitFloatConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
+ public static class IntExpression extends LogicalExpressionBase {
+
+ private static final MajorType INT_CONSTANT = Types.required(MinorType.INT);
+
+ private int i;
+
+ public IntExpression(int i, ExpressionPosition pos) {
+ super(pos);
+ this.i = i;
+ }
+
+ public int getInt() {
+ return i;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return INT_CONSTANT;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitIntConstant(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ }
+
public static class DoubleExpression extends LogicalExpressionBase {
private double d;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
index 711bdb8..469cedd 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
@@ -22,6 +22,8 @@ import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
@@ -45,6 +47,16 @@ public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> imple
}
@Override
+ public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP {
+ return visitUnknown(fExpr, value);
+ }
+
+ @Override
+ public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP {
+ return visitUnknown(intExpr, value);
+ }
+
+ @Override
public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP {
return visitUnknown(intExpr, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
index 128f284..36e6a52 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
@@ -25,6 +25,8 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
public final class AggregateChecker extends SimpleExprVisitor<Boolean>{
@@ -66,6 +68,16 @@ public final class AggregateChecker extends SimpleExprVisitor<Boolean>{
}
@Override
+ public Boolean visitIntConstant(IntExpression intExpr) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitFloatConstant(FloatExpression fExpr) {
+ return false;
+ }
+
+ @Override
public Boolean visitLongConstant(LongExpression intExpr) {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
index 02ce231..a951a60 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
@@ -25,6 +25,8 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
public final class ConstantChecker extends SimpleExprVisitor<Boolean>{
@@ -60,6 +62,16 @@ public final class ConstantChecker extends SimpleExprVisitor<Boolean>{
}
@Override
+ public Boolean visitIntConstant(IntExpression intExpr) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitFloatConstant(FloatExpression fExpr) {
+ return true;
+ }
+
+ @Override
public Boolean visitLongConstant(LongExpression intExpr) {
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
index db34ab7..87d146a 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
@@ -25,11 +25,16 @@ import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+
public interface ExprVisitor<T, VAL, EXCEP extends Exception> {
public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP;
public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP;
- public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
+ public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
+ public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP;
+ public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP;
public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP;
public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP;
public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
index 65a939b..71266f7 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
@@ -21,6 +21,8 @@ import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
@@ -43,6 +45,16 @@ public abstract class SimpleExprVisitor<T> implements ExprVisitor<T, Void, Runti
}
@Override
+ public T visitIntConstant(IntExpression intExpr, Void value) throws RuntimeException {
+ return visitIntConstant(intExpr);
+ }
+
+ @Override
+ public T visitFloatConstant(FloatExpression fExpr, Void value) throws RuntimeException {
+ return visitFloatConstant(fExpr);
+ }
+
+ @Override
public T visitLongConstant(LongExpression intExpr, Void value) throws RuntimeException {
return visitLongConstant(intExpr);
}
@@ -66,6 +78,8 @@ public abstract class SimpleExprVisitor<T> implements ExprVisitor<T, Void, Runti
public abstract T visitFunctionCall(FunctionCall call);
public abstract T visitIfExpression(IfExpression ifExpr);
public abstract T visitSchemaPath(SchemaPath path);
+ public abstract T visitIntConstant(IntExpression intExpr);
+ public abstract T visitFloatConstant(FloatExpression fExpr);
public abstract T visitLongConstant(LongExpression intExpr);
public abstract T visitDoubleConstant(DoubleExpression dExpr);
public abstract T visitBooleanConstant(BooleanExpression e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
index 46dbbe9..4d07a8c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
@@ -98,7 +98,19 @@ public class ConstantExpressionIdentifier implements ExprVisitor<Boolean, Identi
public Boolean visitSchemaPath(SchemaPath path, IdentityHashMap<LogicalExpression, Object> value){
return false;
}
-
+
+ @Override
+ public Boolean visitIntConstant(ValueExpressions.IntExpression intExpr, IdentityHashMap<LogicalExpression, Object> value) throws RuntimeException {
+ value.put(intExpr, true);
+ return true;
+ }
+
+ @Override
+ public Boolean visitFloatConstant(ValueExpressions.FloatExpression fExpr, IdentityHashMap<LogicalExpression, Object> value) throws RuntimeException {
+ value.put(fExpr, true);
+ return true;
+ }
+
@Override
public Boolean visitLongConstant(LongExpression intExpr, IdentityHashMap<LogicalExpression, Object> value){
value.put(intExpr, true);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index cbd722c..c0f34f4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -107,6 +107,16 @@ public class ExpressionTreeMaterializer {
}
@Override
+ public LogicalExpression visitIntConstant(ValueExpressions.IntExpression intExpr) {
+ return intExpr;
+ }
+
+ @Override
+ public LogicalExpression visitFloatConstant(ValueExpressions.FloatExpression fExpr) {
+ return fExpr;
+ }
+
+ @Override
public LogicalExpression visitLongConstant(ValueExpressions.LongExpression intExpr) {
return intExpr;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
index 76683f3..83ae35e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
@@ -26,9 +26,7 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.holders.BigIntHolder;
-import org.apache.drill.exec.expr.holders.IntHolder;
-import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.*;
import org.apache.drill.exec.record.RecordBatch;
public class HashFunctions {
@@ -44,8 +42,122 @@ public class HashFunctions {
}
}
-
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableFloatHash implements DrillSimpleFunc {
+
+ @Param NullableFloat4Holder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ if (in.isSet == 0)
+ out.value = 0;
+ else
+ out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(Float.floatToIntBits(in.value)).asInt();
+ }
+ }
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class FloatHash implements DrillSimpleFunc {
+
+ @Param Float4Holder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(Float.floatToIntBits(in.value)).asInt();
+ }
+ }
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableDoubleHash implements DrillSimpleFunc {
+
+ @Param NullableFloat8Holder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ if (in.isSet == 0)
+ out.value = 0;
+ else
+ out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(Double.doubleToLongBits(in.value)).asInt();
+ }
+ }
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class DoubleHash implements DrillSimpleFunc {
+
+ @Param Float8Holder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(Double.doubleToLongBits(in.value)).asInt();
+ }
+ }
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableVarBinaryHash implements DrillSimpleFunc {
+
+ @Param NullableVarBinaryHolder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ if (in.isSet == 0)
+ out.value = 0;
+ else
+ out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(), 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableBigIntHash implements DrillSimpleFunc {
+
+ @Param NullableBigIntHolder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ // TODO: implement hash function for other types
+ if (in.isSet == 0)
+ out.value = 0;
+ else
+ out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(in.value).asInt();
+ }
+ }
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableIntHash implements DrillSimpleFunc {
+ @Param NullableIntHolder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ // TODO: implement hash function for other types
+ if (in.isSet == 0)
+ out.value = 0;
+ else
+ out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(in.value).asInt();
+ }
+ }
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
public static class VarBinaryHash implements DrillSimpleFunc {
@Param VarBinaryHolder in;
@@ -59,7 +171,7 @@ public class HashFunctions {
}
}
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
public static class HashBigInt implements DrillSimpleFunc {
@Param BigIntHolder in;
@@ -74,7 +186,7 @@ public class HashFunctions {
}
}
- @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
public static class IntHash implements DrillSimpleFunc {
@Param IntHolder in;
@Output IntHolder out;
@@ -87,5 +199,5 @@ public class HashFunctions {
out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(in.value).asInt();
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
index a042a5a..a1d3368 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.expr.holders.BitHolder;
import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
import org.apache.drill.exec.record.RecordBatch;
-@FunctionTemplate(name = "isNull", scope = FunctionTemplate.FunctionScope.SIMPLE)
+@FunctionTemplate(name = "isNull", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
public class IsNull implements DrillSimpleFunc {
@Param NullableFloat8Holder input;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
index 26ac961..f35215b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
@@ -41,34 +41,34 @@ public static final Map<MinorType, Integer> precedenceMap;
*/
int i = 0;
precedenceMap = new HashMap<MinorType, Integer>();
- precedenceMap.put(MinorType.NULL, i++); // NULL is legal to implicitly be promoted to any other type
- precedenceMap.put(MinorType.FIXEDBINARY, i++); // Fixed-length is promoted to var length
- precedenceMap.put(MinorType.VARBINARY, i++);
- precedenceMap.put(MinorType.FIXEDCHAR, i++);
- precedenceMap.put(MinorType.VARCHAR, i++);
- precedenceMap.put(MinorType.FIXED16CHAR, i++);
- precedenceMap.put(MinorType.VAR16CHAR, i++);
- precedenceMap.put(MinorType.BIT, i++);
- precedenceMap.put(MinorType.TINYINT, i++); //type with few bytes is promoted to type with more bytes ==> no data loss.
- precedenceMap.put(MinorType.UINT1, i++); //signed is legal to implicitly be promoted to unsigned.
- precedenceMap.put(MinorType.SMALLINT, i++);
- precedenceMap.put(MinorType.UINT2, i++);
- precedenceMap.put(MinorType.INT, i++);
- precedenceMap.put(MinorType.UINT4, i++);
- precedenceMap.put(MinorType.BIGINT, i++);
- precedenceMap.put(MinorType.UINT8, i++);
- precedenceMap.put(MinorType.MONEY, i++);
- precedenceMap.put(MinorType.DECIMAL4, i++);
- precedenceMap.put(MinorType.DECIMAL8, i++);
- precedenceMap.put(MinorType.DECIMAL12, i++);
- precedenceMap.put(MinorType.DECIMAL16, i++);
- precedenceMap.put(MinorType.FLOAT4, i++);
- precedenceMap.put(MinorType.FLOAT8, i++);
- precedenceMap.put(MinorType.TIME, i++);
- precedenceMap.put(MinorType.DATE, i++);
- precedenceMap.put(MinorType.DATETIME, i++);
- precedenceMap.put(MinorType.TIMETZ, i++);
- precedenceMap.put(MinorType.TIMESTAMP, i++);
+ precedenceMap.put(MinorType.NULL, i += 2); // NULL is legal to implicitly be promoted to any other type
+ precedenceMap.put(MinorType.FIXEDBINARY, i += 2); // Fixed-length is promoted to var length
+ precedenceMap.put(MinorType.VARBINARY, i += 2);
+ precedenceMap.put(MinorType.FIXEDCHAR, i += 2);
+ precedenceMap.put(MinorType.VARCHAR, i += 2);
+ precedenceMap.put(MinorType.FIXED16CHAR, i += 2);
+ precedenceMap.put(MinorType.VAR16CHAR, i += 2);
+ precedenceMap.put(MinorType.BIT, i += 2);
+ precedenceMap.put(MinorType.TINYINT, i += 2); //type with few bytes is promoted to type with more bytes ==> no data loss.
+ precedenceMap.put(MinorType.UINT1, i += 2); //signed is legal to implicitly be promoted to unsigned.
+ precedenceMap.put(MinorType.SMALLINT, i += 2);
+ precedenceMap.put(MinorType.UINT2, i += 2);
+ precedenceMap.put(MinorType.INT, i += 2);
+ precedenceMap.put(MinorType.UINT4, i += 2);
+ precedenceMap.put(MinorType.BIGINT, i += 2);
+ precedenceMap.put(MinorType.UINT8, i += 2);
+ precedenceMap.put(MinorType.MONEY, i += 2);
+ precedenceMap.put(MinorType.DECIMAL4, i += 2);
+ precedenceMap.put(MinorType.DECIMAL8, i += 2);
+ precedenceMap.put(MinorType.DECIMAL12, i += 2);
+ precedenceMap.put(MinorType.DECIMAL16, i += 2);
+ precedenceMap.put(MinorType.FLOAT4, i += 2);
+ precedenceMap.put(MinorType.FLOAT8, i += 2);
+ precedenceMap.put(MinorType.TIME, i += 2);
+ precedenceMap.put(MinorType.DATE, i += 2);
+ precedenceMap.put(MinorType.DATETIME, i += 2);
+ precedenceMap.put(MinorType.TIMETZ, i += 2);
+ precedenceMap.put(MinorType.TIMESTAMP, i += 2);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
index b16d1ff..12ca75d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
@@ -534,14 +534,28 @@ public class TypeCastRules {
// Check null vs non-null, using same logic as that in Types.softEqual()
// Only when the function uses NULL_IF_NULL, nullable and non-nullable are inter-changable.
// Otherwise, the function implementation is not a match.
- if (argType.getMode() != parmType.getMode())
- if (!((holder.getNullHandling() == NullHandling.NULL_IF_NULL) &&
- (argType.getMode() == DataMode.OPTIONAL ||
- argType.getMode() == DataMode.REQUIRED ||
- parmType.getMode() == DataMode.OPTIONAL ||
- parmType.getMode() == DataMode.REQUIRED )))
- return -1;
-
+ if (argType.getMode() != parmType.getMode()) {
+ // TODO - this does not seem to do what it is intended to
+// if (!((holder.getNullHandling() == NullHandling.NULL_IF_NULL) &&
+// (argType.getMode() == DataMode.OPTIONAL ||
+// argType.getMode() == DataMode.REQUIRED ||
+// parmType.getMode() == DataMode.OPTIONAL ||
+// parmType.getMode() == DataMode.REQUIRED )))
+// return -1;
+ // if the function is designed to take optional with custom null handling, and a required
+ // is being passed, increase the cost to account for a null check
+ // this allows for a non-nullable implementation to be preferred
+ if (holder.getNullHandling() == NullHandling.INTERNAL) {
+ // a function that expects required output, but nullable was provided
+ if (parmType.getMode() == DataMode.REQUIRED && argType.getMode() == DataMode.OPTIONAL) {
+ return -1;
+ }
+ else if (parmType.getMode() == DataMode.OPTIONAL && argType.getMode() == DataMode.REQUIRED) {
+ cost++;
+ }
+ }
+ }
+
cost += (parmVal - argVal);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index 25daa7a..f6a8096 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -19,13 +19,23 @@ package org.apache.drill.exec.physical.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Lists;
+import com.sun.codemodel.JClassAlreadyExistsException;
+import com.sun.codemodel.JVar;
import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.*;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.compile.sig.GeneratorMapping;
+import org.apache.drill.exec.compile.sig.MappingSet;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
@@ -34,10 +44,14 @@ import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.aggregate.Aggregator;
+import org.apache.drill.exec.physical.impl.project.Projector;
import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.resolver.FunctionResolver;
+import org.apache.drill.exec.resolver.FunctionResolverFactory;
import org.apache.drill.exec.rpc.user.UserServer;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.vector.NullableVarBinaryVector;
@@ -49,12 +63,99 @@ import com.google.common.base.Charsets;
import com.google.common.io.Files;
import com.codahale.metrics.MetricRegistry;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
public class TestSimpleFunctions {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class);
DrillConfig c = DrillConfig.create();
@Test
+ public void testHashFunctionResolution(@Injectable DrillConfig config) throws JClassAlreadyExistsException, IOException {
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config);
+ // test required vs nullable Int input
+ resolveHash(config,
+ new TypedNullConstant(Types.optional(TypeProtos.MinorType.INT)),
+ Types.optional(TypeProtos.MinorType.INT),
+ Types.required(TypeProtos.MinorType.INT),
+ TypeProtos.DataMode.OPTIONAL,
+ registry);
+
+ resolveHash(config,
+ new ValueExpressions.IntExpression(1, ExpressionPosition.UNKNOWN),
+ Types.required(TypeProtos.MinorType.INT),
+ Types.required(TypeProtos.MinorType.INT),
+ TypeProtos.DataMode.REQUIRED,
+ registry);
+
+ // test required vs nullable float input
+ resolveHash(config,
+ new TypedNullConstant(Types.optional(TypeProtos.MinorType.FLOAT4)),
+ Types.optional(TypeProtos.MinorType.FLOAT4),
+ Types.required(TypeProtos.MinorType.FLOAT4),
+ TypeProtos.DataMode.OPTIONAL,
+ registry);
+
+ resolveHash(config,
+ new ValueExpressions.FloatExpression(5.0f, ExpressionPosition.UNKNOWN),
+ Types.required(TypeProtos.MinorType.FLOAT4),
+ Types.required(TypeProtos.MinorType.FLOAT4),
+ TypeProtos.DataMode.REQUIRED,
+ registry);
+
+ // test required vs nullable long input
+ resolveHash(config,
+ new TypedNullConstant(Types.optional(TypeProtos.MinorType.BIGINT)),
+ Types.optional(TypeProtos.MinorType.BIGINT),
+ Types.required(TypeProtos.MinorType.BIGINT),
+ TypeProtos.DataMode.OPTIONAL,
+ registry);
+
+ resolveHash(config,
+ new ValueExpressions.LongExpression(100L, ExpressionPosition.UNKNOWN),
+ Types.required(TypeProtos.MinorType.BIGINT),
+ Types.required(TypeProtos.MinorType.BIGINT),
+ TypeProtos.DataMode.REQUIRED,
+ registry);
+
+ // test required vs nullable double input
+ resolveHash(config,
+ new TypedNullConstant(Types.optional(TypeProtos.MinorType.FLOAT8)),
+ Types.optional(TypeProtos.MinorType.FLOAT8),
+ Types.required(TypeProtos.MinorType.FLOAT8),
+ TypeProtos.DataMode.OPTIONAL,
+ registry);
+
+ resolveHash(config,
+ new ValueExpressions.DoubleExpression(100.0, ExpressionPosition.UNKNOWN),
+ Types.required(TypeProtos.MinorType.FLOAT8),
+ Types.required(TypeProtos.MinorType.FLOAT8),
+ TypeProtos.DataMode.REQUIRED,
+ registry);
+ }
+
+ public void resolveHash(DrillConfig config, LogicalExpression arg, TypeProtos.MajorType expectedArg,
+ TypeProtos.MajorType expectedOut, TypeProtos.DataMode expectedBestInputMode,
+ FunctionImplementationRegistry registry) throws JClassAlreadyExistsException, IOException {
+ List<LogicalExpression> args = new ArrayList<>();
+ args.add(arg);
+ String[] registeredNames = { "hash" };
+ FunctionCall call = new FunctionCall(
+ FunctionDefinition.simple("hash",
+ new BasicArgumentValidator(new Arg[]{ new Arg(expectedArg)}),
+ new OutputTypeDeterminer.FixedType(expectedOut),
+ registeredNames),
+ args,
+ ExpressionPosition.UNKNOWN
+ );
+ FunctionResolver resolver = FunctionResolverFactory.getResolver(call);
+ DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getMethods().get(call.getDefinition().getName()), call);
+ assertEquals( expectedBestInputMode, matchedFuncHolder.getParmMajorType(0).getMode());
+ }
+
+ @Test
public void testIsNull(@Injectable final DrillbitContext bitContext,
@Injectable UserServer.UserClientConnection connection) throws Throwable {
[28/38] git commit: DRILL-366: While casting VarChar to BigInt make
sure the getByte() method is invoked with the appropriate byte offset.
Posted by ja...@apache.org.
DRILL-366: While casting VarChar to BigInt make sure the getByte() method is invoked with the appropriate byte offset.
CastFunctionsSrcVarLen.java:
Use in.start as starting index to invoke getBytes() instead of 0. We always end up with the same row when we use 0
TestCastVarCharToBigInt.java, *.json
Added a test case to read varchar and convert it to BigInt.
Signed-off-by: Jacques Nadeau <ja...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ee9eaa13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ee9eaa13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ee9eaa13
Branch: refs/heads/master
Commit: ee9eaa13c31838be5e080e7915efbd25f0a1e310
Parents: ed9dec7
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Feb 11 20:07:20 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../templates/CastFunctionsSrcVarLen.java | 18 ++--
.../physical/impl/TestCastVarCharToBigInt.java | 101 +++++++++++++++++++
.../cast/test_cast_varchar_to_bigint.json | 38 +++++++
.../src/test/resources/scan_json_test_cast.json | 9 ++
4 files changed, 157 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ee9eaa13/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
index 370597d..5855381 100644
--- a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
+++ b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java
@@ -61,19 +61,19 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
out.value = ${type.javaType}.parse${type.parse}(new String(buf));
<#elseif type.to=="Int" || type.to == "BigInt">
- int i = 0;
- int length = in.end - in.start;
-
- if (length==0) {
+
+ if ((in.end - in.start) ==0) {
//empty, not a valid number
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
throw new NumberFormatException(new String(buf));
}
+
+ int readIndex = in.start;
+
+ boolean negative = in.buffer.getByte(readIndex) == '-';
- boolean negative = in.buffer.getByte(0)=='-';
-
- if (negative && ++i == length ) {
+ if (negative && ++readIndex == in.end) {
//only one single '-'
byte[] buf = new byte[in.end - in.start];
in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
@@ -85,8 +85,8 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{
${type.primeType} result = 0;
int digit;
- while (i < length) {
- digit = Character.digit(in.buffer.getByte(i++),radix);
+ while (readIndex < in.end) {
+ digit = Character.digit(in.buffer.getByte(readIndex++),radix);
//not valid digit.
if (digit == -1) {
byte[] buf = new byte[in.end - in.start];
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ee9eaa13/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
new file mode 100644
index 0000000..ba8b9da
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
+import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.SimpleRootExec;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.pop.PopUnitTestBase;
+import org.apache.drill.exec.proto.BitControl;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.Float8Vector;
+
+import org.apache.drill.exec.vector.ValueVector;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.codahale.metrics.MetricRegistry;
+
+import java.util.List;
+
+
+public class TestCastVarCharToBigInt extends PopUnitTestBase {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestCastVarCharToBigInt.class);
+
+ @Test
+ public void testCastToBigInt() throws Exception {
+ try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
+
+ // run query.
+ bit.run();
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
+ Files.toString(FileUtils.getResourceAsFile("/functions/cast/test_cast_varchar_to_bigint.json"), Charsets.UTF_8)
+ .replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/scan_json_test_cast.json").toURI().toString())
+ );
+
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
+
+ QueryResultBatch batch = results.get(0);
+ assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));
+
+ batchLoader.getValueAccessorById(0, BigIntVector.class);
+
+ for (VectorWrapper<?> v : batchLoader) {
+
+ ValueVector.Accessor accessor = v.getValueVector().getAccessor();
+
+ assertEquals(accessor.getObject(0), 2008L);
+ assertEquals(accessor.getObject(1), 2007L);
+ assertEquals(accessor.getObject(2), 2006L);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ee9eaa13/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json b/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json
new file mode 100644
index 0000000..36439c5
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json
@@ -0,0 +1,38 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"json-scan",
+ entries: [
+ {
+ path : "#{TEST_FILE}"
+ }
+ ],
+ storageengine: {
+ "type": "json",
+ "dfsName": "file:///"
+ }
+ },
+ {
+ pop:"project",
+ @id:2,
+ child: 1,
+ exprs: [ {
+ ref: "INTEGER",
+ expr: "cast(integer as bigint)"
+ } ]
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ee9eaa13/exec/java-exec/src/test/resources/scan_json_test_cast.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/scan_json_test_cast.json b/exec/java-exec/src/test/resources/scan_json_test_cast.json
new file mode 100644
index 0000000..dd9119a
--- /dev/null
+++ b/exec/java-exec/src/test/resources/scan_json_test_cast.json
@@ -0,0 +1,9 @@
+{
+ "integer" : "2008"
+}
+{
+ "integer" : "2007"
+}
+{
+ "integer" : "2006"
+}
\ No newline at end of file
[18/38] git commit: DRILL-320: Rework math functions to use code
generated using free marker.
Posted by ja...@apache.org.
DRILL-320: Rework math functions to use code generated using free marker.
Contains implementation for basic functions: +, -, *, /
Signed-off-by: Jacques Nadeau <ja...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ed9dec7c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ed9dec7c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ed9dec7c
Branch: refs/heads/master
Commit: ed9dec7c3f289e4f68466b989a500523f3d18b1e
Parents: 8b90cae
Author: Mehant Baid <me...@gmail.com>
Authored: Fri Feb 7 22:54:06 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
exec/java-exec/src/main/codegen/config.fmpp | 3 +-
.../src/main/codegen/data/MathFunctionTypes.tdd | 72 +++++++++++++++
.../templates/MathFunctionTemplates.java | 70 ++++++++++++++
.../drill/exec/expr/fn/impl/MathFunctions.java | 59 ------------
.../drill/exec/fn/impl/TestMathFunctions.java | 96 ++++++++++++++++++++
.../functions/simple_math_functions.json | 34 +++++++
6 files changed, 274 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/config.fmpp b/exec/java-exec/src/main/codegen/config.fmpp
index cd2b2cc..8f1060a 100644
--- a/exec/java-exec/src/main/codegen/config.fmpp
+++ b/exec/java-exec/src/main/codegen/config.fmpp
@@ -17,7 +17,8 @@
data: {
vv: tdd(../data/ValueVectorTypes.tdd),
compare: tdd(../data/CompareTypes.tdd),
- cast: tdd(../data/Casts.tdd)
+ cast: tdd(../data/Casts.tdd),
+ MathFunctionTypes: tdd(../data/MathFunctionTypes.tdd)
}
freemarkerLinks: {
includes: includes/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/main/codegen/data/MathFunctionTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/MathFunctionTypes.tdd b/exec/java-exec/src/main/codegen/data/MathFunctionTypes.tdd
new file mode 100644
index 0000000..1724373
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/data/MathFunctionTypes.tdd
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ mathFunctionTypes: [
+ {className: "Add", funcName: "add", op: "+", types: [
+ {input1: "Int", input2: "Int", outputType: "Int", castType: "int"},
+ {input1: "BigInt", input2: "BigInt", outputType: "BigInt", castType: "long"},
+ {input1: "Float4", input2: "Float4", outputType: "Float4", castType: "float"},
+ {input1: "Float8", input2: "Float8", outputType: "Float8", castType: "double"},
+ {input1: "SmallInt", input2: "SmallInt", outputType: "SmallInt", castType: "short"},
+ {input1: "TinyInt", input2: "TinyInt", outputType: "TinyInt", castType: "byte"},
+ {input1: "UInt1", input2: "UInt1", outputType: "UInt1", castType: "byte"},
+ {input1: "UInt2", input2: "UInt2", outputType: "UInt2", castType: "char"},
+ {input1: "UInt4", input2: "UInt4", outputType: "UInt4", castType: "int"},
+ {input1: "UInt8", input2: "UInt8", outputType: "UInt8", castType: "long"}
+ ]
+ },
+ {className: "Subtract", funcName: "subtract", op: "-", types: [
+ {input1: "Int", input2: "Int", outputType: "Int", castType: "int"},
+ {input1: "BigInt", input2: "BigInt", outputType: "BigInt", castType: "long"},
+ {input1: "Float4", input2: "Float4", outputType: "Float4", castType: "float"},
+ {input1: "Float8", input2: "Float8", outputType: "Float8", castType: "double"},
+ {input1: "SmallInt", input2: "SmallInt", outputType: "SmallInt", castType: "short"},
+ {input1: "TinyInt", input2: "TinyInt", outputType: "TinyInt", castType: "byte"},
+ {input1: "UInt1", input2: "UInt1", outputType: "UInt1", castType: "byte"},
+ {input1: "UInt2", input2: "UInt2", outputType: "UInt2", castType: "char"},
+ {input1: "UInt4", input2: "UInt4", outputType: "UInt4", castType: "int"},
+ {input1: "UInt8", input2: "UInt8", outputType: "UInt8", castType: "long"}
+ ]
+ },
+ {className: "Multiply", funcName: "multiply", op: "*", types: [
+ {input1: "Int", input2: "Int", outputType: "Int", castType: "int"},
+ {input1: "BigInt", input2: "BigInt", outputType: "BigInt", castType: "long"},
+ {input1: "Float4", input2: "Float4", outputType: "Float4", castType: "float"},
+ {input1: "Float8", input2: "Float8", outputType: "Float8", castType: "double"},
+ {input1: "SmallInt", input2: "SmallInt", outputType: "SmallInt", castType: "short"},
+ {input1: "TinyInt", input2: "TinyInt", outputType: "TinyInt", castType: "byte"},
+ {input1: "UInt1", input2: "UInt1", outputType: "UInt1", castType: "byte"},
+ {input1: "UInt2", input2: "UInt2", outputType: "UInt2", castType: "char"},
+ {input1: "UInt4", input2: "UInt4", outputType: "UInt4", castType: "int"},
+ {input1: "UInt8", input2: "UInt8", outputType: "UInt8", castType: "long"}
+ ]
+ },
+ {className: "Divide", funcName: "divide", op: "/", types: [
+ {input1: "Int", input2: "Int", outputType: "Int", castType: "int"},
+ {input1: "BigInt", input2: "BigInt", outputType: "BigInt", castType: "long"},
+ {input1: "Float4", input2: "Float4", outputType: "Float4", castType: "float"},
+ {input1: "Float8", input2: "Float8", outputType: "Float8", castType: "double"},
+ {input1: "SmallInt", input2: "SmallInt", outputType: "SmallInt", castType: "short"},
+ {input1: "TinyInt", input2: "TinyInt", outputType: "TinyInt", castType: "byte"},
+ {input1: "UInt1", input2: "UInt1", outputType: "UInt1", castType: "byte"},
+ {input1: "UInt2", input2: "UInt2", outputType: "UInt2", castType: "char"},
+ {input1: "UInt4", input2: "UInt4", outputType: "UInt4", castType: "int"},
+ {input1: "UInt8", input2: "UInt8", outputType: "UInt8", castType: "long"}
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/main/codegen/templates/MathFunctionTemplates.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/MathFunctionTemplates.java b/exec/java-exec/src/main/codegen/templates/MathFunctionTemplates.java
new file mode 100644
index 0000000..9ec2178
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/MathFunctionTemplates.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+
+
+<#list MathFunctionTypes.mathFunctionTypes as inputType>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/${inputType.className}Functions.java" />
+
+<#include "/@includes/license.ftl" />
+
+<#-- A utility class that is used to generate java code for add function -->
+
+/*
+ * This class is automatically generated from AddTypes.tdd using FreeMarker.
+ */
+
+
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+
+@SuppressWarnings("unused")
+
+public class ${inputType.className}Functions {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${inputType.className}Functions.class);
+
+<#list inputType.types as type>
+
+ @FunctionTemplate(name = "${inputType.funcName}", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ public static class ${type.input1}${type.input2}${inputType.className} implements DrillSimpleFunc {
+
+ @Param ${type.input1}Holder in1;
+ @Param ${type.input2}Holder in2;
+ @Output ${type.outputType}Holder out;
+
+ public void setup(RecordBatch b) {
+ }
+
+ public void eval() {
+ out.value = (${type.castType}) (in1.value ${inputType.op} in2.value);
+ }
+ }
+</#list>
+}
+</#list>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
index ea251c3..f4e2060 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
@@ -34,65 +34,6 @@ public class MathFunctions{
private MathFunctions(){}
- @FunctionTemplate(name = "add", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
- public static class Add1 implements DrillSimpleFunc{
-
- @Param IntHolder left;
- @Param IntHolder right;
- @Output IntHolder out;
-
- public void setup(RecordBatch b){}
-
- public void eval(){
- out.value = left.value + right.value;
- }
-
- }
-
- @FunctionTemplate(name = "add", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
- public static class LongAdd1 implements DrillSimpleFunc{
-
- @Param BigIntHolder left;
- @Param BigIntHolder right;
- @Output BigIntHolder out;
-
- public void setup(RecordBatch b){}
-
- public void eval(){
- out.value = left.value + right.value;
- }
-
- }
-
- @FunctionTemplate(name = "add", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
- public static class Float4Add implements DrillSimpleFunc{
-
- @Param Float4Holder left;
- @Param Float4Holder right;
- @Output Float4Holder out;
-
- public void setup(RecordBatch b){}
-
- public void eval(){
- out.value = left.value + right.value;
- }
- }
-
-
- @FunctionTemplate(name = "add", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
- public static class Float8Add implements DrillSimpleFunc{
-
- @Param Float8Holder left;
- @Param Float8Holder right;
- @Output Float8Holder out;
-
- public void setup(RecordBatch b){}
-
- public void eval(){
- out.value = left.value + right.value;
- }
- }
-
@FunctionTemplate(name = "negative", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class Negative implements DrillSimpleFunc{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
new file mode 100644
index 0000000..cc90f13
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.fn.impl;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
+import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.SimpleRootExec;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.BitControl;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.Float8Vector;
+
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.codahale.metrics.MetricRegistry;
+
+
+public class TestMathFunctions {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestMathFunctions.class);
+ DrillConfig c = DrillConfig.create();
+
+
+ @Test
+ public void testBasicMathFunctions(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable
+ {
+
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry();
+ bitContext.getAllocator(); result = new TopLevelAllocator();
+ bitContext.getConfig(); result = c;
+ bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ }};
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/simple_math_functions.json"), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ while(exec.next()) {
+ BigIntVector intMulVector = exec.getValueVectorById(new SchemaPath("INTMUL", ExpressionPosition.UNKNOWN), BigIntVector.class);
+ Float8Vector floatMulVector = exec.getValueVectorById(new SchemaPath("FLOATMUL", ExpressionPosition.UNKNOWN), Float8Vector.class);
+ BigIntVector intAddVector = exec.getValueVectorById(new SchemaPath("INTADD", ExpressionPosition.UNKNOWN), BigIntVector.class);
+ Float8Vector floatAddVector = exec.getValueVectorById(new SchemaPath("FLOATADD", ExpressionPosition.UNKNOWN), Float8Vector.class);
+ assertEquals(exec.getRecordCount(), 1);
+ assertEquals(intMulVector.getAccessor().get(0), 2);
+ assertEquals(floatMulVector.getAccessor().get(0), (1.1 * 2.2), 0);
+ assertEquals(intAddVector.getAccessor().get(0), 3);
+ assertEquals(floatAddVector.getAccessor().get(0), (1.1 + 2.2), 0);
+ }
+
+ if(context.getFailureCause() != null){
+ throw context.getFailureCause();
+ }
+ assertTrue(!context.isFailed());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed9dec7c/exec/java-exec/src/test/resources/functions/simple_math_functions.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/simple_math_functions.json b/exec/java-exec/src/test/resources/functions/simple_math_functions.json
new file mode 100644
index 0000000..28ac5b2
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/simple_math_functions.json
@@ -0,0 +1,34 @@
+{
+ head : {
+ version : 1,
+ generator : {
+ type : "optiq",
+ info : "na"
+ },
+ type : "APACHE_DRILL_PHYSICAL"
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-sub-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 1, types: [
+ {name: "blue", type: "BIGINT", mode: "REQUIRED"}
+ ]}
+ ]
+ }, {
+ pop : "project",
+ @id : 2,
+ exprs : [
+ { ref : "output.INTMUL", expr : " 1 * 2 "},
+ { ref : "output.FLOATMUL", expr : " 1.1 * 2.2 "},
+ { ref : "output.INTADD", expr : " 1 + 2 " },
+ { ref : "output.FLOATADD", expr : " 1.1 + 2.2 " } ],
+ child : 1
+ }, {
+ pop : "screen",
+ @id : 3,
+ child : 2
+ } ]
+}
\ No newline at end of file
[22/38] Integrate new SQL changes with Hive storage engine,
moving to automatic file detection. Rename storage engine to storage
plugin. Separate storage plugins from format plugins,
updating Parquet and JSON to format engines. Refactor distribution log
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
new file mode 100644
index 0000000..6d02046
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.QueryOptimizerRule;
+import org.apache.drill.exec.store.dfs.BasicFormatMatcher;
+import org.apache.drill.exec.store.dfs.FileSelection;
+import org.apache.drill.exec.store.dfs.FormatMatcher;
+import org.apache.drill.exec.store.dfs.FormatPlugin;
+import org.apache.drill.exec.store.dfs.FormatSelection;
+import org.apache.drill.exec.store.dfs.MagicString;
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+import org.apache.drill.exec.store.mock.MockStorageEngine;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import parquet.format.converter.ParquetMetadataConverter;
+import parquet.hadoop.CodecFactoryExposer;
+import parquet.hadoop.ParquetFileWriter;
+
+import com.google.common.collect.Lists;
+
+public class ParquetFormatPlugin implements FormatPlugin{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class);
+
+ private final DrillbitContext context;
+ static final ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
+ private CodecFactoryExposer codecFactoryExposer;
+ private final DrillFileSystem fs;
+ private final ParquetFormatMatcher formatMatcher;
+ private final ParquetFormatConfig config;
+ private final StoragePluginConfig storageConfig;
+ private final String name;
+
+ public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig){
+ this(name, context, fs, storageConfig, new ParquetFormatConfig());
+ }
+
+ public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){
+ this.context = context;
+ this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf());
+ this.config = formatConfig;
+ this.formatMatcher = new ParquetFormatMatcher(this, fs);
+ this.storageConfig = storageConfig;
+ this.fs = fs;
+ this.name = name == null ? "parquet" : name;
+ }
+
+ Configuration getHadoopConfig() {
+ return fs.getUnderlying().getConf();
+ }
+
+ public DrillFileSystem getFileSystem() {
+ return fs;
+ }
+
+ @Override
+ public ParquetFormatConfig getConfig() {
+ return config;
+ }
+
+ public DrillbitContext getContext() {
+ return this.context;
+ }
+
+ @Override
+ public boolean supportsRead() {
+ return true;
+ }
+
+ @Override
+ public List<QueryOptimizerRule> getOptimizerRules() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public ParquetGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException {
+ return new ParquetGroupScan( selection.getFileStatusList(fs), this, outputRef);
+ }
+
+ @Override
+ public StoragePluginConfig getStorageConfig() {
+ return storageConfig;
+ }
+
+ public CodecFactoryExposer getCodecFactoryExposer() {
+ return codecFactoryExposer;
+ }
+
+ public String getName(){
+ return name;
+ }
+
+ @Override
+ public boolean supportsWrite() {
+ return false;
+ }
+
+
+
+ @Override
+ public FormatMatcher getMatcher() {
+ return formatMatcher;
+ }
+
+ private static class ParquetFormatMatcher extends BasicFormatMatcher{
+
+ private final DrillFileSystem fs;
+
+ public ParquetFormatMatcher(ParquetFormatPlugin plugin, DrillFileSystem fs) {
+ super(plugin, fs, //
+ Lists.newArrayList( //
+ Pattern.compile(".*\\.parquet$"), //
+ Pattern.compile(".*/" + ParquetFileWriter.PARQUET_METADATA_FILE) //
+ //
+ ),
+ Lists.newArrayList(new MagicString(0, ParquetFileWriter.MAGIC))
+
+ );
+ this.fs = fs;
+
+ }
+
+ @Override
+ public boolean supportDirectoryReads() {
+ return true;
+ }
+
+ @Override
+ public FormatSelection isReadable(FileSelection file) throws IOException {
+ // TODO: we only check the first file for directory reading. This is because
+ if(file.containsDirectories(fs)){
+ if(isDirReadable(file.getFirstPath(fs))){
+ return new FormatSelection(plugin.getConfig(), file);
+ }
+ }
+ return super.isReadable(file);
+ }
+
+ boolean isDirReadable(FileStatus dir) {
+ Path p = new Path(dir.getPath(), "/" + ParquetFileWriter.PARQUET_METADATA_FILE);
+ try {
+ return fs.getUnderlying().exists(p);
+ } catch (IOException e) {
+ logger.info("Failure while attempting to check for Parquet metadata file.", e);
+ return false;
+ }
+ }
+
+
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index aa01115..f76e59a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -18,37 +18,35 @@
package org.apache.drill.exec.store.parquet;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.Timer;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.ReadEntryFromHDFS;
-import org.apache.drill.exec.physical.ReadEntryWithPath;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.store.AffinityCalculator;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.dfs.FileSystemPlugin;
+import org.apache.drill.exec.store.dfs.ReadEntryFromHDFS;
+import org.apache.drill.exec.store.dfs.ReadEntryWithPath;
+import org.apache.drill.exec.store.dfs.easy.FileWork;
+import org.apache.drill.exec.store.schedule.AffinityCreator;
+import org.apache.drill.exec.store.schedule.AssignmentCreator;
+import org.apache.drill.exec.store.schedule.BlockMapBuilder;
+import org.apache.drill.exec.store.schedule.CompleteWork;
+import org.apache.drill.exec.store.schedule.EndpointByteMap;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -59,16 +57,17 @@ import parquet.hadoop.metadata.ColumnChunkMetaData;
import parquet.hadoop.metadata.ParquetMetadata;
import parquet.org.codehaus.jackson.annotate.JsonCreator;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
-import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-
@JsonTypeName("parquet-scan")
public class ParquetGroupScan extends AbstractGroupScan {
@@ -78,11 +77,18 @@ public class ParquetGroupScan extends AbstractGroupScan {
static final String ENDPOINT_BYTES_TIMER = MetricRegistry.name(ParquetGroupScan.class, "endpointBytes");
static final String ASSIGNMENT_TIMER = MetricRegistry.name(ParquetGroupScan.class, "applyAssignments");
static final String ASSIGNMENT_AFFINITY_HIST = MetricRegistry.name(ParquetGroupScan.class, "assignmentAffinity");
+
final Histogram assignmentAffinityStats = metrics.histogram(ASSIGNMENT_AFFINITY_HIST);
- private ArrayListMultimap<Integer, ParquetRowGroupScan.RowGroupReadEntry> mappings;
+ private ListMultimap<Integer, RowGroupInfo> mappings;
private List<RowGroupInfo> rowGroupInfos;
- private Stopwatch watch = new Stopwatch();
+ private final List<ReadEntryWithPath> entries;
+ private final Stopwatch watch = new Stopwatch();
+ private final ParquetFormatPlugin formatPlugin;
+ private final ParquetFormatConfig formatConfig;
+ private final FileSystem fs;
+ private final FieldReference ref;
+ private List<EndpointAffinity> endpointAffinities;
private List<SchemaPath> columns;
@@ -91,79 +97,87 @@ public class ParquetGroupScan extends AbstractGroupScan {
}
@JsonProperty("storageengine")
- public ParquetStorageEngineConfig getEngineConfig() {
- return this.engineConfig;
+ public ParquetFormatConfig getEngineConfig() {
+ return this.formatConfig;
}
- private List<ReadEntryWithPath> entries;
- private long totalBytes;
- private Collection<DrillbitEndpoint> availableEndpoints;
- private ParquetStorageEngine storageEngine;
- private ParquetStorageEngineConfig engineConfig;
- private FileSystem fs;
- private final FieldReference ref;
- private List<EndpointAffinity> endpointAffinities;
-
@JsonCreator
- public ParquetGroupScan(@JsonProperty("entries") List<ReadEntryWithPath> entries,
- @JsonProperty("storageengine") ParquetStorageEngineConfig storageEngineConfig,
- @JacksonInject StorageEngineRegistry engineRegistry,
- @JsonProperty("ref") FieldReference ref,
- @JsonProperty("columns") List<SchemaPath> columns
- )throws IOException, ExecutionSetupException {
+ public ParquetGroupScan( //
+ @JsonProperty("entries") List<ReadEntryWithPath> entries, //
+ @JsonProperty("storage") StoragePluginConfig storageConfig, //
+ @JsonProperty("format") FormatPluginConfig formatConfig, //
+ @JacksonInject StoragePluginRegistry engineRegistry, //
+ @JsonProperty("ref") FieldReference ref, //
+ @JsonProperty("columns") List<SchemaPath> columns //
+ ) throws IOException, ExecutionSetupException {
engineRegistry.init(DrillConfig.create());
this.columns = columns;
- this.storageEngine = (ParquetStorageEngine) engineRegistry.getEngine(storageEngineConfig);
- this.availableEndpoints = storageEngine.getContext().getBits();
- this.fs = storageEngine.getFileSystem();
- this.engineConfig = storageEngineConfig;
+ if(formatConfig == null) formatConfig = new ParquetFormatConfig();
+ Preconditions.checkNotNull(storageConfig);
+ Preconditions.checkNotNull(formatConfig);
+ this.formatPlugin = (ParquetFormatPlugin) engineRegistry.getFormatPlugin(storageConfig, formatConfig);
+ Preconditions.checkNotNull(formatPlugin);
+ this.fs = formatPlugin.getFileSystem().getUnderlying();
+ this.formatConfig = formatPlugin.getConfig();
this.entries = entries;
this.ref = ref;
- readFooter();
- calculateEndpointBytes();
+ this.readFooterFromEntries();
+
}
- public ParquetGroupScan(List<ReadEntryWithPath> entries, //
- ParquetStorageEngine storageEngine, //
- FieldReference ref, //
- List<SchemaPath> columns) throws IOException {
- this.storageEngine = storageEngine;
- this.columns = columns;
- this.engineConfig = storageEngine.getEngineConfig();
- this.availableEndpoints = storageEngine.getContext().getBits();
- this.fs = storageEngine.getFileSystem();
- this.entries = entries;
+ public ParquetGroupScan(List<FileStatus> files, //
+ ParquetFormatPlugin formatPlugin, //
+ FieldReference ref) //
+ throws IOException {
+ this.formatPlugin = formatPlugin;
+ this.columns = null;
+ this.formatConfig = formatPlugin.getConfig();
+ this.fs = formatPlugin.getFileSystem().getUnderlying();
+
+ this.entries = Lists.newArrayList();
+ for(FileStatus file : files){
+ entries.add(new ReadEntryWithPath(file.getPath().toString()));
+ }
+
this.ref = ref;
- readFooter();
- calculateEndpointBytes();
+ readFooter(files);
}
- private void readFooter() throws IOException {
+ private void readFooterFromEntries() throws IOException {
+ List<FileStatus> files = Lists.newArrayList();
+ for(ReadEntryWithPath e : entries){
+ files.add(fs.getFileStatus(new Path(e.getPath())));
+ }
+ readFooter(files);
+ }
+
+ private void readFooter(List<FileStatus> statuses) throws IOException {
watch.reset();
watch.start();
Timer.Context tContext = metrics.timer(READ_FOOTER_TIMER).time();
- rowGroupInfos = new ArrayList();
+
+
+ rowGroupInfos = Lists.newArrayList();
long start = 0, length = 0;
ColumnChunkMetaData columnChunkMetaData;
- for (ReadEntryWithPath readEntryWithPath : entries){
- Path path = new Path(readEntryWithPath.getPath());
- List<Footer> footers = ParquetFileReader.readFooters(this.storageEngine.getHadoopConfig(), path);
+ for (FileStatus status : statuses) {
+ List<Footer> footers = ParquetFileReader.readFooters(formatPlugin.getHadoopConfig(), status);
if (footers.size() == 0) {
- logger.warn("No footers found");
+ throw new IOException(String.format("Unable to find footer for file %s", status.getPath().getName()));
}
-// readEntryWithPath.getPath();
for (Footer footer : footers) {
int index = 0;
ParquetMetadata metadata = footer.getParquetMetadata();
- for (BlockMetaData rowGroup : metadata.getBlocks()){
+ for (BlockMetaData rowGroup : metadata.getBlocks()) {
// need to grab block information from HDFS
columnChunkMetaData = rowGroup.getColumns().iterator().next();
start = columnChunkMetaData.getFirstDataPageOffset();
- // this field is not being populated correctly, but the column chunks know their sizes, just summing them for now
- //end = start + rowGroup.getTotalByteSize();
+ // this field is not being populated correctly, but the column chunks know their sizes, just summing them for
+ // now
+ // end = start + rowGroup.getTotalByteSize();
length = 0;
- for (ColumnChunkMetaData col : rowGroup.getColumns()){
+ for (ColumnChunkMetaData col : rowGroup.getColumns()) {
length += col.getTotalSize();
}
String filePath = footer.getFile().toUri().getPath();
@@ -179,203 +193,109 @@ public class ParquetGroupScan extends AbstractGroupScan {
logger.debug("Took {} ms to get row group infos", watch.elapsed(TimeUnit.MILLISECONDS));
}
- private void calculateEndpointBytes() {
- Timer.Context tContext = metrics.timer(ENDPOINT_BYTES_TIMER).time();
- watch.reset();
- watch.start();
- AffinityCalculator ac = new AffinityCalculator(fs, availableEndpoints);
- for (RowGroupInfo e : rowGroupInfos) {
- ac.setEndpointBytes(e);
- totalBytes += e.getLength();
- }
- watch.stop();
- tContext.stop();
- logger.debug("Took {} ms to calculate EndpointBytes", watch.elapsed(TimeUnit.MILLISECONDS));
- }
-
@JsonIgnore
public FileSystem getFileSystem() {
return this.fs;
}
- public static class RowGroupInfo extends ReadEntryFromHDFS {
+ public static class RowGroupInfo extends ReadEntryFromHDFS implements CompleteWork, FileWork {
- private HashMap<DrillbitEndpoint,Long> endpointBytes;
- private long maxBytes;
+ private EndpointByteMap byteMap;
private int rowGroupIndex;
@JsonCreator
public RowGroupInfo(@JsonProperty("path") String path, @JsonProperty("start") long start,
- @JsonProperty("length") long length, @JsonProperty("rowGroupIndex") int rowGroupIndex) {
+ @JsonProperty("length") long length, @JsonProperty("rowGroupIndex") int rowGroupIndex) {
super(path, start, length);
this.rowGroupIndex = rowGroupIndex;
}
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(1, 2, 1, 1);
+ public RowGroupReadEntry getRowGroupReadEntry() {
+ return new RowGroupReadEntry(this.getPath(), this.getStart(), this.getLength(), this.rowGroupIndex);
}
- @Override
- public Size getSize() {
- // TODO - these values are wrong, I cannot know these until after I read a file
- return new Size(10, 10);
- }
-
- public HashMap<DrillbitEndpoint,Long> getEndpointBytes() {
- return endpointBytes;
- }
-
- public void setEndpointBytes(HashMap<DrillbitEndpoint,Long> endpointBytes) {
- this.endpointBytes = endpointBytes;
- }
-
- public void setMaxBytes(long bytes) {
- this.maxBytes = bytes;
+ public int getRowGroupIndex() {
+ return this.rowGroupIndex;
}
- public long getMaxBytes() {
- return maxBytes;
+ @Override
+ public int compareTo(CompleteWork o) {
+ return Long.compare(getTotalBytes(), o.getTotalBytes());
}
- public ParquetRowGroupScan.RowGroupReadEntry getRowGroupReadEntry() {
- return new ParquetRowGroupScan.RowGroupReadEntry(this.getPath(), this.getStart(), this.getLength(), this.rowGroupIndex);
+ @Override
+ public long getTotalBytes() {
+ return this.getLength();
}
- public int getRowGroupIndex() {
- return this.rowGroupIndex;
+ @Override
+ public EndpointByteMap getByteMap() {
+ return byteMap;
}
- }
- private class ParquetReadEntryComparator implements Comparator<RowGroupInfo> {
- public int compare(RowGroupInfo e1, RowGroupInfo e2) {
- if (e1.getMaxBytes() == e2.getMaxBytes()) return 0;
- return (e1.getMaxBytes() > e2.getMaxBytes()) ? 1 : -1;
+ public void setEndpointByteMap(EndpointByteMap byteMap) {
+ this.byteMap = byteMap;
}
}
/**
- *Calculates the affinity each endpoint has for this scan, by adding up the affinity each endpoint has for each
+ * Calculates the affinity each endpoint has for this scan, by adding up the affinity each endpoint has for each
* rowGroup
+ *
* @return a list of EndpointAffinity objects
*/
@Override
public List<EndpointAffinity> getOperatorAffinity() {
- watch.reset();
- watch.start();
+
if (this.endpointAffinities == null) {
- HashMap<DrillbitEndpoint, Float> affinities = new HashMap<>();
- for (RowGroupInfo entry : rowGroupInfos) {
- for (DrillbitEndpoint d : entry.getEndpointBytes().keySet()) {
- long bytes = entry.getEndpointBytes().get(d);
- float affinity = (float)bytes / (float)totalBytes;
- logger.debug("RowGroup: {} Endpoint: {} Bytes: {}", entry.getRowGroupIndex(), d.getAddress(), bytes);
- if (affinities.keySet().contains(d)) {
- affinities.put(d, affinities.get(d) + affinity);
- } else {
- affinities.put(d, affinity);
- }
+ BlockMapBuilder bmb = new BlockMapBuilder(fs, formatPlugin.getContext().getBits());
+ try{
+ for (RowGroupInfo rgi : rowGroupInfos) {
+ EndpointByteMap ebm = bmb.getEndpointByteMap(rgi);
+ rgi.setEndpointByteMap(ebm);
}
+ } catch (IOException e) {
+ logger.warn("Failure while determining operator affinity.", e);
+ return Collections.emptyList();
}
- List<EndpointAffinity> affinityList = new LinkedList<>();
- for (DrillbitEndpoint d : affinities.keySet()) {
- logger.debug("Endpoint {} has affinity {}", d.getAddress(), affinities.get(d).floatValue());
- affinityList.add(new EndpointAffinity(d,affinities.get(d).floatValue()));
- }
- this.endpointAffinities = affinityList;
+
+ this.endpointAffinities = AffinityCreator.getAffinityMap(rowGroupInfos);
}
- watch.stop();
- logger.debug("Took {} ms to get operator affinity", watch.elapsed(TimeUnit.MILLISECONDS));
return this.endpointAffinities;
}
+ @Override
+ public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) throws PhysicalOperatorSetupException {
- static final double[] ASSIGNMENT_CUTOFFS = {0.99, 0.50, 0.25, 0.00};
+ this.mappings = AssignmentCreator.getMappings(incomingEndpoints, rowGroupInfos);
- /**
- *
- * @param incomingEndpoints
- */
- @Override
- public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) {
- watch.reset();
- watch.start();
- final Timer.Context tcontext = metrics.timer(ASSIGNMENT_TIMER).time();
- Preconditions.checkArgument(incomingEndpoints.size() <= rowGroupInfos.size(), String.format("Incoming endpoints %d " +
- "is greater than number of row groups %d", incomingEndpoints.size(), rowGroupInfos.size()));
- mappings = ArrayListMultimap.create();
- ArrayList rowGroupList = new ArrayList(rowGroupInfos);
- List<DrillbitEndpoint> endpointLinkedlist = Lists.newLinkedList(incomingEndpoints);
- for(double cutoff : ASSIGNMENT_CUTOFFS ){
- scanAndAssign(mappings, endpointLinkedlist, rowGroupList, cutoff, false);
- }
- scanAndAssign(mappings, endpointLinkedlist, rowGroupList, 0.0, true);
- tcontext.stop();
- watch.stop();
- logger.debug("Took {} ms to apply assignments", watch.elapsed(TimeUnit.MILLISECONDS));
- Preconditions.checkState(rowGroupList.isEmpty(), "All readEntries should be assigned by now, but some are still unassigned");
- Preconditions.checkState(!rowGroupInfos.isEmpty());
}
- public int fragmentPointer = 0;
+ @Override
+ public ParquetRowGroupScan getSpecificScan(int minorFragmentId) {
+ assert minorFragmentId < mappings.size() : String.format(
+ "Mappings length [%d] should be longer than minor fragment id [%d] but it isn't.", mappings.size(),
+ minorFragmentId);
- /**
- *
- * @param endpointAssignments the mapping between fragment/endpoint and rowGroup
- * @param endpoints the list of drillbits, ordered by the corresponding fragment
- * @param rowGroups the list of rowGroups to assign
- * @param requiredPercentage the percentage of max bytes required to make an assignment
- * @param assignAll if true, will assign even if no affinity
- */
- private void scanAndAssign (Multimap<Integer, ParquetRowGroupScan.RowGroupReadEntry> endpointAssignments, List<DrillbitEndpoint> endpoints,
- List<RowGroupInfo> rowGroups, double requiredPercentage, boolean assignAll) {
- Collections.sort(rowGroups, new ParquetReadEntryComparator());
- final boolean requireAffinity = requiredPercentage > 0;
- int maxAssignments = (int) (rowGroups.size() / endpoints.size());
-
- if (maxAssignments < 1) maxAssignments = 1;
-
- for(Iterator<RowGroupInfo> iter = rowGroups.iterator(); iter.hasNext();){
- RowGroupInfo rowGroupInfo = iter.next();
- for (int i = 0; i < endpoints.size(); i++) {
- int minorFragmentId = (fragmentPointer + i) % endpoints.size();
- DrillbitEndpoint currentEndpoint = endpoints.get(minorFragmentId);
- Map<DrillbitEndpoint, Long> bytesPerEndpoint = rowGroupInfo.getEndpointBytes();
- boolean haveAffinity = bytesPerEndpoint.containsKey(currentEndpoint) ;
-
- if (assignAll ||
- (!bytesPerEndpoint.isEmpty() &&
- (!requireAffinity || haveAffinity) &&
- (!endpointAssignments.containsKey(minorFragmentId) || endpointAssignments.get(minorFragmentId).size() < maxAssignments) &&
- (!requireAffinity || bytesPerEndpoint.get(currentEndpoint) >= rowGroupInfo.getMaxBytes() * requiredPercentage))) {
-
- endpointAssignments.put(minorFragmentId, rowGroupInfo.getRowGroupReadEntry());
- logger.debug("Assigned rowGroup {} to minorFragmentId {} endpoint {}", rowGroupInfo.getRowGroupIndex(), minorFragmentId, endpoints.get(minorFragmentId).getAddress());
- if (bytesPerEndpoint.get(currentEndpoint) != null) {
- assignmentAffinityStats.update(bytesPerEndpoint.get(currentEndpoint) / rowGroupInfo.getLength());
- } else {
- assignmentAffinityStats.update(0);
- }
- iter.remove();
- fragmentPointer = (minorFragmentId + 1) % endpoints.size();
- break;
- }
- }
+ List<RowGroupInfo> rowGroupsForMinor = mappings.get(minorFragmentId);
- }
+ Preconditions.checkArgument(!rowGroupsForMinor.isEmpty(),
+ String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId));
+
+ return new ParquetRowGroupScan(formatPlugin, convertToReadEntries(rowGroupsForMinor), ref, columns);
}
- @Override
- public ParquetRowGroupScan getSpecificScan(int minorFragmentId) {
- assert minorFragmentId < mappings.size() : String.format("Mappings length [%d] should be longer than minor fragment id [%d] but it isn't.", mappings.size(), minorFragmentId);
- for (ParquetRowGroupScan.RowGroupReadEntry rg : mappings.get(minorFragmentId)) {
- logger.debug("minorFragmentId: {} Path: {} RowGroupIndex: {}",minorFragmentId, rg.getPath(),rg.getRowGroupIndex());
+
+
+ private List<RowGroupReadEntry> convertToReadEntries(List<RowGroupInfo> rowGroups){
+ List<RowGroupReadEntry> entries = Lists.newArrayList();
+ for (RowGroupInfo rgi : rowGroups) {
+ RowGroupReadEntry rgre = new RowGroupReadEntry(rgi.getPath(), rgi.getStart(), rgi.getLength(),
+ rgi.getRowGroupIndex());
+ entries.add(rgre);
}
- Preconditions.checkArgument(!mappings.get(minorFragmentId).isEmpty(), String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId));
- return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId), ref,
- columns);
+ return entries;
}
-
public FieldReference getRef() {
return ref;
@@ -392,21 +312,21 @@ public class ParquetGroupScan extends AbstractGroupScan {
@Override
public OperatorCost getCost() {
- //TODO Figure out how to properly calculate cost
- return new OperatorCost(1,rowGroupInfos.size(),1,1);
+ // TODO Figure out how to properly calculate cost
+ return new OperatorCost(1, rowGroupInfos.size(), 1, 1);
}
@Override
public Size getSize() {
// TODO - this is wrong, need to populate correctly
- return new Size(10,10);
+ return new Size(10, 10);
}
@Override
@JsonIgnore
public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
Preconditions.checkArgument(children.isEmpty());
- //TODO return copy of self
+ // TODO return copy of self
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
index 1e6c31a..9e1cc66 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
@@ -49,7 +49,7 @@ import parquet.schema.PrimitiveType;
import com.google.common.base.Joiner;
-public class ParquetRecordReader implements RecordReader {
+class ParquetRecordReader implements RecordReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReader.class);
// this value has been inflated to read in multiple value vectors at once, and then break them up into smaller vectors
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
index b3ce9b4..0e672d0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
@@ -25,15 +25,16 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.ReadEntryFromHDFS;
import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.physical.base.SubScan;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.dfs.FileSystemPlugin;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -48,43 +49,53 @@ import com.google.common.collect.Iterators;
public class ParquetRowGroupScan extends AbstractBase implements SubScan {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRowGroupScan.class);
- public final StorageEngineConfig engineConfig;
- private final ParquetStorageEngine parquetStorageEngine;
+ public final ParquetFormatConfig formatConfig;
+ private final ParquetFormatPlugin formatPlugin;
private final List<RowGroupReadEntry> rowGroupReadEntries;
private final FieldReference ref;
private final List<SchemaPath> columns;
@JsonCreator
- public ParquetRowGroupScan(@JacksonInject StorageEngineRegistry registry,
- @JsonProperty("engineConfig") StorageEngineConfig engineConfig,
- @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries,
- @JsonProperty("ref") FieldReference ref,
- @JsonProperty("columns") List<SchemaPath> columns
- ) throws ExecutionSetupException {
- parquetStorageEngine = (ParquetStorageEngine) registry.getEngine(engineConfig);
+ public ParquetRowGroupScan( //
+ @JacksonInject StoragePluginRegistry registry, //
+ @JsonProperty("storage") StoragePluginConfig storageConfig, //
+ @JsonProperty("format") FormatPluginConfig formatConfig, //
+ @JsonProperty("entries") LinkedList<RowGroupReadEntry> rowGroupReadEntries, //
+ @JsonProperty("ref") FieldReference ref, //
+ @JsonProperty("columns") List<SchemaPath> columns //
+ ) throws ExecutionSetupException {
+
+ if(formatConfig == null) formatConfig = new ParquetFormatConfig();
+ Preconditions.checkNotNull(storageConfig);
+ Preconditions.checkNotNull(formatConfig);
+ this.formatPlugin = (ParquetFormatPlugin) registry.getFormatPlugin(storageConfig, formatConfig);
+ Preconditions.checkNotNull(formatPlugin);
this.rowGroupReadEntries = rowGroupReadEntries;
- this.engineConfig = engineConfig;
+ this.formatConfig = formatPlugin.getConfig();
this.ref = ref;
this.columns = columns;
}
- public ParquetRowGroupScan(ParquetStorageEngine engine, ParquetStorageEngineConfig config,
- List<RowGroupReadEntry> rowGroupReadEntries, FieldReference ref,
- List<SchemaPath> columns
- ) {
- parquetStorageEngine = engine;
- engineConfig = config;
+ public ParquetRowGroupScan( //
+ ParquetFormatPlugin formatPlugin, //
+ List<RowGroupReadEntry> rowGroupReadEntries, //
+ FieldReference ref, //
+ List<SchemaPath> columns) {
+ this.formatPlugin = formatPlugin;
+ this.formatConfig = formatPlugin.getConfig();
this.rowGroupReadEntries = rowGroupReadEntries;
this.ref = ref;
this.columns = columns;
}
+ @JsonProperty("entries")
public List<RowGroupReadEntry> getRowGroupReadEntries() {
return rowGroupReadEntries;
}
- public StorageEngineConfig getEngineConfig() {
- return engineConfig;
+ @JsonProperty("storage")
+ public StoragePluginConfig getEngineConfig() {
+ return formatPlugin.getStorageConfig();
}
@Override
@@ -92,7 +103,6 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
return null;
}
-
public FieldReference getRef() {
return ref;
}
@@ -108,8 +118,8 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
}
@JsonIgnore
- public ParquetStorageEngine getStorageEngine(){
- return parquetStorageEngine;
+ public ParquetFormatPlugin getStorageEngine() {
+ return formatPlugin;
}
@Override
@@ -120,8 +130,7 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
@Override
public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.isEmpty());
- return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries,
- ref, columns);
+ return new ParquetRowGroupScan(formatPlugin, rowGroupReadEntries, ref, columns);
}
@Override
@@ -133,36 +142,4 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
return columns;
}
- public static class RowGroupReadEntry extends ReadEntryFromHDFS {
-
- private int rowGroupIndex;
-
- @parquet.org.codehaus.jackson.annotate.JsonCreator
- public RowGroupReadEntry(@JsonProperty("path") String path, @JsonProperty("start") long start,
- @JsonProperty("length") long length, @JsonProperty("rowGroupIndex") int rowGroupIndex) {
- super(path, start, length);
- this.rowGroupIndex = rowGroupIndex;
- }
-
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(1, 2, 1, 1);
- }
-
- @Override
- public Size getSize() {
- // TODO - these values are wrong, I cannot know these until after I read a file
- return new Size(10, 10);
- }
-
- @JsonIgnore
- public RowGroupReadEntry getRowGroupReadEntry() {
- return new RowGroupReadEntry(this.getPath(), this.getStart(), this.getLength(), this.rowGroupIndex);
- }
-
- public int getRowGroupIndex(){
- return rowGroupIndex;
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index 966a16b..17e7da2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -26,7 +26,10 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.BatchCreator;
import org.apache.drill.exec.physical.impl.ScanBatch;
@@ -38,8 +41,10 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import parquet.hadoop.CodecFactoryExposer;
import parquet.hadoop.ParquetFileReader;
import parquet.hadoop.metadata.ParquetMetadata;
@@ -50,9 +55,12 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
public RecordBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan, List<RecordBatch> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.isEmpty());
List<RecordReader> readers = Lists.newArrayList();
+
+ FileSystem fs = rowGroupScan.getStorageEngine().getFileSystem().getUnderlying();
+
// keep footers in a map to avoid re-reading them
Map<String, ParquetMetadata> footers = new HashMap<String, ParquetMetadata>();
- for(ParquetRowGroupScan.RowGroupReadEntry e : rowGroupScan.getRowGroupReadEntries()){
+ for(RowGroupReadEntry e : rowGroupScan.getRowGroupReadEntries()){
/*
Here we could store a map from file names to footers, to prevent re-reading the footer for each row group in a file
TODO - to prevent reading the footer again in the parquet record reader (it is read earlier in the ParquetStorageEngine)
@@ -63,11 +71,11 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
try {
if ( ! footers.containsKey(e.getPath())){
footers.put(e.getPath(),
- ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath())));
+ ParquetFileReader.readFooter( fs.getConf(), new Path(e.getPath())));
}
readers.add(
new ParquetRecordReader(
- context, e.getPath(), e.getRowGroupIndex(), rowGroupScan.getStorageEngine().getFileSystem(),
+ context, e.getPath(), e.getRowGroupIndex(), fs,
rowGroupScan.getStorageEngine().getCodecFactoryExposer(),
footers.get(e.getPath()),
rowGroupScan.getRef(),
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
deleted file mode 100644
index c17a9e3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.parquet;
-
-import java.io.IOException;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.physical.ReadEntryWithPath;
-import org.apache.drill.exec.store.ClassPathFileSystem;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-
-import com.beust.jcommander.internal.Lists;
-
-public class ParquetSchemaProvider implements SchemaProvider{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetSchemaProvider.class);
-
- public static final String HADOOP_DEFAULT_NAME = "fs.default.name";
- final ParquetStorageEngineConfig configuration;
- final FileSystem fs;
- final Configuration conf;
-
- public ParquetSchemaProvider(ParquetStorageEngineConfig configuration, DrillConfig config){
- this.configuration = configuration;
- try {
- this.conf = new Configuration();
- this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
- this.conf.set(HADOOP_DEFAULT_NAME, configuration.getDfsName());
- logger.debug("{}: {}",HADOOP_DEFAULT_NAME, configuration.getDfsName());
- this.fs = FileSystem.get(conf);
- } catch (IOException ie) {
- throw new RuntimeException("Error setting up filesystem", ie);
- }
- }
-
- @Override
- public Object getSelectionBaseOnName(String tableName) {
- try{
- if(!fs.exists(new Path(tableName))) return null;
- ReadEntryWithPath re = new ReadEntryWithPath(tableName);
- return Lists.newArrayList(re);
- }catch(Exception e){
- logger.warn(String.format("Failure while checking table name %s.", tableName), e);
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
deleted file mode 100644
index ad9756e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.parquet;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import com.beust.jcommander.internal.Lists;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.ReadEntryWithPath;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.AbstractStorageEngine;
-import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.drill.exec.store.mock.MockStorageEngine;
-
-import com.google.common.collect.ListMultimap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
-
-import parquet.format.converter.ParquetMetadataConverter;
-import parquet.hadoop.CodecFactoryExposer;
-import parquet.hadoop.ParquetFileReader;
-import parquet.hadoop.metadata.BlockMetaData;
-import parquet.hadoop.metadata.ColumnChunkMetaData;
-import parquet.hadoop.metadata.ParquetMetadata;
-
-public class ParquetStorageEngine extends AbstractStorageEngine{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class);
-
- private final DrillbitContext context;
- static final ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
- private CodecFactoryExposer codecFactoryExposer;
- private final ParquetSchemaProvider schemaProvider;
- private final ParquetStorageEngineConfig engineConfig;
-
- public ParquetStorageEngine(ParquetStorageEngineConfig configuration, DrillbitContext context){
- this.context = context;
- this.schemaProvider = new ParquetSchemaProvider(configuration, context.getConfig());
- codecFactoryExposer = new CodecFactoryExposer(schemaProvider.conf);
- this.engineConfig = configuration;
- }
-
- public Configuration getHadoopConfig() {
- double y = 5;
- int x = (int) y;
- return schemaProvider.conf;
- }
-
- public FileSystem getFileSystem() {
- return schemaProvider.fs;
- }
-
- public ParquetStorageEngineConfig getEngineConfig() {
- return engineConfig;
- }
-
- public DrillbitContext getContext() {
- return this.context;
- }
-
- @Override
- public boolean supportsRead() {
- return true;
- }
-
- @Override
- public ParquetGroupScan getPhysicalScan(Scan scan) throws IOException {
-
- ArrayList<ReadEntryWithPath> readEntries = scan.getSelection().getListWith(new ObjectMapper(),
- new TypeReference<ArrayList<ReadEntryWithPath>>() {});
-
- return new ParquetGroupScan( readEntries, this, scan.getOutputReference(), null);
- }
-
- @Override
- public ListMultimap<ReadEntry, DrillbitEndpoint> getReadLocations(Collection<ReadEntry> entries) {
- return null;
- }
-
- @Override
- public RecordReader getReader(FragmentContext context, ReadEntry readEntry) throws IOException {
- return null;
- }
-
-
- public CodecFactoryExposer getCodecFactoryExposer() {
- return codecFactoryExposer;
- }
-
- @Override
- public ParquetSchemaProvider getSchemaProvider() {
- return schemaProvider;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java
deleted file mode 100644
index f2d6124..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.parquet;
-
-import java.util.HashMap;
-
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-import org.apache.drill.exec.store.DistributedStorageEngine;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-@JsonTypeName("parquet")
-public class ParquetStorageEngineConfig extends StorageEngineConfigBase implements DistributedStorageEngine{
-
- public String getDfsName() {
- return dfsName;
- }
-
- // information needed to identify an HDFS instance
- private String dfsName;
- private HashMap<String,String> map;
-
- @JsonCreator
- public ParquetStorageEngineConfig(@JsonProperty("dfsName") String dfsName) {
- this.dfsName = dfsName;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- ParquetStorageEngineConfig that = (ParquetStorageEngineConfig) o;
-
- if (dfsName != null ? !dfsName.equals(that.dfsName) : that.dfsName != null) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return dfsName != null ? dfsName.hashCode() : 0;
- }
- public void set(String key, String value) {
- map.put(key, value);
- }
-
- public String get(String key) {
- return map.get(key);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupReadEntry.java
new file mode 100644
index 0000000..986328e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupReadEntry.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.exec.store.dfs.ReadEntryFromHDFS;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class RowGroupReadEntry extends ReadEntryFromHDFS {
+
+ private int rowGroupIndex;
+
+ @parquet.org.codehaus.jackson.annotate.JsonCreator
+ public RowGroupReadEntry(@JsonProperty("path") String path, @JsonProperty("start") long start,
+ @JsonProperty("length") long length, @JsonProperty("rowGroupIndex") int rowGroupIndex) {
+ super(path, start, length);
+ this.rowGroupIndex = rowGroupIndex;
+ }
+
+ @JsonIgnore
+ public RowGroupReadEntry getRowGroupReadEntry() {
+ return new RowGroupReadEntry(this.getPath(), this.getStart(), this.getLength(), this.rowGroupIndex);
+ }
+
+ public int getRowGroupIndex(){
+ return rowGroupIndex;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
index 0321838..d9e498e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
@@ -103,6 +103,10 @@ public class VarLenBinaryReader {
do {
lengthVarFieldsInCurrentRecord = 0;
for (ColumnReader columnReader : columns) {
+ if (recordsReadInCurrentPass == columnReader.valueVecHolder.getValueVector().getValueCapacity()){
+ rowGroupFinished = true;
+ break;
+ }
if (columnReader.pageReadStatus.currentPage == null
|| columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) {
columnReader.totalValuesRead += columnReader.pageReadStatus.valuesRead;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java
new file mode 100644
index 0000000..d25abad
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.schedule;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.exec.physical.EndpointAffinity;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+
+import com.beust.jcommander.internal.Lists;
+import com.carrotsearch.hppc.ObjectFloatOpenHashMap;
+import com.carrotsearch.hppc.cursors.ObjectFloatCursor;
+import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+import com.google.common.base.Stopwatch;
+
+public class AffinityCreator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AffinityCreator.class);
+
+ public static <T extends CompleteWork> List<EndpointAffinity> getAffinityMap(List<T> work){
+ Stopwatch watch = new Stopwatch();
+
+ long totalBytes = 0;
+ for (CompleteWork entry : work) {
+ totalBytes += entry.getTotalBytes();
+ }
+
+ ObjectFloatOpenHashMap<DrillbitEndpoint> affinities = new ObjectFloatOpenHashMap<DrillbitEndpoint>();
+ for (CompleteWork entry : work) {
+ for (ObjectLongCursor<DrillbitEndpoint> cursor : entry.getByteMap()) {
+ long bytes = cursor.value;
+ float affinity = (float)bytes / (float)totalBytes;
+ logger.debug("Work: {} Endpoint: {} Bytes: {}", work, cursor.key.getAddress(), bytes);
+ affinities.putOrAdd(cursor.key, affinity, affinity);
+ }
+ }
+
+ List<EndpointAffinity> affinityList = Lists.newLinkedList();
+ for (ObjectFloatCursor<DrillbitEndpoint> d : affinities) {
+ logger.debug("Endpoint {} has affinity {}", d.key.getAddress(), d.value);
+ affinityList.add(new EndpointAffinity(d.key, d.value));
+ }
+
+ logger.debug("Took {} ms to get operator affinity", watch.elapsed(TimeUnit.MILLISECONDS));
+ return affinityList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
new file mode 100644
index 0000000..eaa4f17
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.schedule;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+
+/**
+ * The AssignmentCreator is responsible for assigning a set of work units to the available slices.
+ */
+public class AssignmentCreator<T extends CompleteWork> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssignmentCreator.class);
+
+ static final double[] ASSIGNMENT_CUTOFFS = { 0.99, 0.50, 0.25, 0.00 };
+ private final ArrayListMultimap<Integer, T> mappings;
+ private final List<DrillbitEndpoint> endpoints;
+
+
+
+ /**
+ * Given a set of endpoints to assign work to, attempt to evenly assign work based on affinity of work units to
+ * Drillbits.
+ *
+ * @param incomingEndpoints
+ * The set of nodes to assign work to. Note that nodes can be listed multiple times if we want to have
+ * multiple slices on a node working on the task simultaneously.
+ * @param units
+ * The work units to assign.
+ * @return ListMultimap of Integer > List<CompleteWork> (based on their incoming order) to with
+ */
+ public static <T extends CompleteWork> ListMultimap<Integer, T> getMappings(List<DrillbitEndpoint> incomingEndpoints,
+ List<T> units) {
+ AssignmentCreator<T> creator = new AssignmentCreator<T>(incomingEndpoints, units);
+ return creator.mappings;
+ }
+
+ private AssignmentCreator(List<DrillbitEndpoint> incomingEndpoints, List<T> units) {
+ Stopwatch watch = new Stopwatch();
+
+ Preconditions.checkArgument(incomingEndpoints.size() <= units.size(), String.format("Incoming endpoints %d "
+ + "is greater than number of row groups %d", incomingEndpoints.size(), units.size()));
+ this.mappings = ArrayListMultimap.create();
+ this.endpoints = Lists.newLinkedList(incomingEndpoints);
+
+ ArrayList<T> rowGroupList = new ArrayList<>(units);
+ for (double cutoff : ASSIGNMENT_CUTOFFS) {
+ scanAndAssign(rowGroupList, cutoff, false);
+ }
+ scanAndAssign(rowGroupList, 0.0, true);
+
+ logger.debug("Took {} ms to apply assignments", watch.elapsed(TimeUnit.MILLISECONDS));
+ Preconditions.checkState(rowGroupList.isEmpty(), "All readEntries should be assigned by now, but some are still unassigned");
+ Preconditions.checkState(!units.isEmpty());
+
+ }
+
+ /**
+ *
+ * @param mappings
+ * the mapping between fragment/endpoint and rowGroup
+ * @param endpoints
+ * the list of drillbits, ordered by the corresponding fragment
+ * @param workunits
+ * the list of rowGroups to assign
+ * @param requiredPercentage
+ * the percentage of max bytes required to make an assignment
+ * @param assignAll
+ * if true, will assign even if no affinity
+ */
+ private void scanAndAssign(List<T> workunits, double requiredPercentage, boolean assignAll) {
+ Collections.sort(workunits);
+ int fragmentPointer = 0;
+ final boolean requireAffinity = requiredPercentage > 0;
+ int maxAssignments = (int) (workunits.size() / endpoints.size());
+
+ if (maxAssignments < 1)
+ maxAssignments = 1;
+
+ for (Iterator<T> iter = workunits.iterator(); iter.hasNext();) {
+ T unit = iter.next();
+ for (int i = 0; i < endpoints.size(); i++) {
+ int minorFragmentId = (fragmentPointer + i) % endpoints.size();
+ DrillbitEndpoint currentEndpoint = endpoints.get(minorFragmentId);
+ EndpointByteMap endpointByteMap = unit.getByteMap();
+ boolean haveAffinity = endpointByteMap.isSet(currentEndpoint);
+
+ if (assignAll
+ || (!endpointByteMap.isEmpty() && (!requireAffinity || haveAffinity)
+ && (!mappings.containsKey(minorFragmentId) || mappings.get(minorFragmentId).size() < maxAssignments) && (!requireAffinity || endpointByteMap
+ .get(currentEndpoint) >= endpointByteMap.getMaxBytes() * requiredPercentage))) {
+
+ mappings.put(minorFragmentId, unit);
+ // logger.debug("Assigned rowGroup {} to minorFragmentId {} endpoint {}", rowGroupInfo.getRowGroupIndex(),
+ // minorFragmentId, endpoints.get(minorFragmentId).getAddress());
+ // if (bytesPerEndpoint.get(currentEndpoint) != null) {
+ // // assignmentAffinityStats.update(bytesPerEndpoint.get(currentEndpoint) / rowGroupInfo.getLength());
+ // } else {
+ // // assignmentAffinityStats.update(0);
+ // }
+ iter.remove();
+ fragmentPointer = (minorFragmentId + 1) % endpoints.size();
+ break;
+ }
+ }
+
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
new file mode 100644
index 0000000..432c1d7
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.schedule;
+
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.exec.metrics.DrillMetrics;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.store.dfs.easy.FileWork;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.beust.jcommander.internal.Lists;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableRangeMap;
+import com.google.common.collect.Range;
+
+public class BlockMapBuilder {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BlockMapBuilder.class);
+ static final MetricRegistry metrics = DrillMetrics.getInstance();
+ static final String BLOCK_MAP_BUILDER_TIMER = MetricRegistry.name(BlockMapBuilder.class, "blockMapBuilderTimer");
+
+ private HashMap<Path,ImmutableRangeMap<Long,BlockLocation>> blockMapMap = new HashMap<>();
+ private Collection<DrillbitEndpoint> endpoints;
+ private FileSystem fs;
+ private HashMap<String,DrillbitEndpoint> endPointMap;
+
+ public BlockMapBuilder(FileSystem fs, Collection<DrillbitEndpoint> endpoints) {
+ this.fs = fs;
+ this.endpoints = endpoints;
+ buildEndpointMap();
+ }
+
+
+ public List<CompleteFileWork> generateFileWork(List<FileStatus> files, boolean blockify) throws IOException{
+ List<CompleteFileWork> work = Lists.newArrayList();
+ for(FileStatus f : files){
+ ImmutableRangeMap<Long,BlockLocation> rangeMap = getBlockMap(f);
+ if(!blockify){
+ work.add(new CompleteFileWork(this.getEndpointByteMap(new FileStatusWork(f)), 0, f.getLen(), f.getPath().toString()));
+ continue;
+ }
+
+ for(Entry<Range<Long>, BlockLocation> l : rangeMap.asMapOfRanges().entrySet()){
+ work.add(new CompleteFileWork(this.getEndpointByteMap(new FileStatusWork(f)), l.getValue().getOffset(), l.getValue().getLength(), f.getPath().toString()));
+ }
+ }
+ return work;
+ }
+
+ private class FileStatusWork implements FileWork{
+ private FileStatus status;
+
+ public FileStatusWork(FileStatus status) {
+ if(status.isDir()) throw new IllegalStateException("FileStatus work only works with files, not directories.");
+ this.status = status;
+ }
+
+ @Override
+ public String getPath() {
+ return status.getPath().toString();
+ }
+
+ @Override
+ public long getStart() {
+ return 0;
+ }
+
+ @Override
+ public long getLength() {
+ return status.getLen();
+ }
+
+
+
+ }
+
+ private ImmutableRangeMap<Long,BlockLocation> buildBlockMap(Path path) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ return buildBlockMap(status);
+ }
+
+ /**
+ * Builds a mapping of block locations to file byte range
+ */
+ private ImmutableRangeMap<Long,BlockLocation> buildBlockMap(FileStatus status) throws IOException {
+ final Timer.Context context = metrics.timer(BLOCK_MAP_BUILDER_TIMER).time();
+ BlockLocation[] blocks;
+ ImmutableRangeMap<Long,BlockLocation> blockMap;
+ blocks = fs.getFileBlockLocations(status, 0 , status.getLen());
+ ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long,BlockLocation>();
+ for (BlockLocation block : blocks) {
+ long start = block.getOffset();
+ long end = start + block.getLength();
+ Range<Long> range = Range.closedOpen(start, end);
+ blockMapBuilder = blockMapBuilder.put(range, block);
+ }
+ blockMap = blockMapBuilder.build();
+ blockMapMap.put(status.getPath(), blockMap);
+ context.stop();
+ return blockMap;
+ }
+
+ private ImmutableRangeMap<Long,BlockLocation> getBlockMap(Path path) throws IOException{
+ ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(path);
+ if(blockMap == null){
+ blockMap = buildBlockMap(path);
+ }
+ return blockMap;
+ }
+
+ private ImmutableRangeMap<Long,BlockLocation> getBlockMap(FileStatus status) throws IOException{
+ ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(status.getPath());
+ if(blockMap == null){
+ blockMap = buildBlockMap(status);
+ }
+ return blockMap;
+ }
+
+
+ /**
+ * For a given FileWork, calculate how many bytes are available on each on drillbit endpoint
+ *
+ * @param work the FileWork to calculate endpoint bytes for
+ * @throws IOException
+ */
+ public EndpointByteMap getEndpointByteMap(FileWork work) throws IOException {
+ Stopwatch watch = new Stopwatch();
+ watch.start();
+ Path fileName = new Path(work.getPath());
+
+
+ ImmutableRangeMap<Long,BlockLocation> blockMap = getBlockMap(fileName);
+ EndpointByteMapImpl endpointByteMap = new EndpointByteMapImpl();
+ long start = work.getStart();
+ long end = start + work.getLength();
+ Range<Long> rowGroupRange = Range.closedOpen(start, end);
+
+ // Find submap of ranges that intersect with the rowGroup
+ ImmutableRangeMap<Long,BlockLocation> subRangeMap = blockMap.subRangeMap(rowGroupRange);
+
+ // Iterate through each block in this submap and get the host for the block location
+ for (Map.Entry<Range<Long>,BlockLocation> block : subRangeMap.asMapOfRanges().entrySet()) {
+ String[] hosts;
+ Range<Long> blockRange = block.getKey();
+ try {
+ hosts = block.getValue().getHosts();
+ } catch (IOException ioe) {
+ throw new RuntimeException("Failed to get hosts for block location", ioe);
+ }
+ Range<Long> intersection = rowGroupRange.intersection(blockRange);
+ long bytes = intersection.upperEndpoint() - intersection.lowerEndpoint();
+
+ // For each host in the current block location, add the intersecting bytes to the corresponding endpoint
+ for (String host : hosts) {
+ DrillbitEndpoint endpoint = getDrillBitEndpoint(host);
+ if(endpoint != null){
+ endpointByteMap.add(endpoint, bytes);
+ }else{
+ logger.debug("Failure finding Drillbit running on host {}. Skipping affinity to that host.", host);
+ }
+ }
+ }
+
+ logger.debug("FileWork group ({},{}) max bytes {}", work.getPath(), work.getStart(), endpointByteMap.getMaxBytes());
+
+ logger.debug("Took {} ms to set endpoint bytes", watch.stop().elapsed(TimeUnit.MILLISECONDS));
+ return endpointByteMap;
+ }
+
+ private DrillbitEndpoint getDrillBitEndpoint(String hostName) {
+ return endPointMap.get(hostName);
+ }
+
+ /**
+ * Builds a mapping of Drillbit endpoints to hostnames
+ */
+ private void buildEndpointMap() {
+ Stopwatch watch = new Stopwatch();
+ watch.start();
+ endPointMap = new HashMap<String, DrillbitEndpoint>();
+ for (DrillbitEndpoint d : endpoints) {
+ String hostName = d.getAddress();
+ endPointMap.put(hostName, d);
+ }
+ watch.stop();
+ logger.debug("Took {} ms to build endpoint map", watch.elapsed(TimeUnit.MILLISECONDS));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java
new file mode 100644
index 0000000..30b08f6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.schedule;
+
+import org.apache.drill.exec.store.dfs.easy.FileWork;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class CompleteFileWork implements FileWork, CompleteWork{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CompleteFileWork.class);
+
+ private long start;
+ private long length;
+ private String path;
+ private EndpointByteMap byteMap;
+
+ public CompleteFileWork(EndpointByteMap byteMap, long start, long length, String path) {
+ super();
+ this.start = start;
+ this.length = length;
+ this.path = path;
+ this.byteMap = byteMap;
+ }
+
+ @Override
+ public int compareTo(CompleteWork o) {
+ return Long.compare(getTotalBytes(), o.getTotalBytes());
+ }
+
+ @Override
+ public long getTotalBytes() {
+ return length;
+ }
+
+ @Override
+ public EndpointByteMap getByteMap() {
+ return byteMap;
+ }
+
+ @Override
+ public String getPath() {
+ return path;
+ }
+
+ @Override
+ public long getStart() {
+ return start;
+ }
+
+ @Override
+ public long getLength() {
+ return length;
+ }
+
+ public FileWorkImpl getAsFileWork(){
+ return new FileWorkImpl(start, length, path);
+ }
+
+ public static class FileWorkImpl implements FileWork{
+
+ @JsonCreator
+ public FileWorkImpl(@JsonProperty("start") long start, @JsonProperty("length") long length, @JsonProperty("path") String path) {
+ super();
+ this.start = start;
+ this.length = length;
+ this.path = path;
+ }
+
+ public long start;
+ public long length;
+ public String path;
+
+ @Override
+ public String getPath() {
+ return path;
+ }
+
+ @Override
+ public long getStart() {
+ return start;
+ }
+
+ @Override
+ public long getLength() {
+ return length;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java
new file mode 100644
index 0000000..44e27d4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.schedule;
+
+
+/**
+ * Container that holds a complete work unit. Can contain one or more partial units.
+ */
+public interface CompleteWork extends Comparable<CompleteWork>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CompleteWork.class);
+
+ public long getTotalBytes();
+ public EndpointByteMap getByteMap();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMap.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMap.java
new file mode 100644
index 0000000..f543d75
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMap.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.schedule;
+
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+
+import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+
+/**
+ * Presents an interface that describes the number of bytes for a particular work unit associated with a particular DrillbitEndpoint.
+ */
+public interface EndpointByteMap extends Iterable<ObjectLongCursor<DrillbitEndpoint>>{
+
+ public boolean isSet(DrillbitEndpoint endpoint);
+ public long get(DrillbitEndpoint endpoint);
+ public boolean isEmpty();
+ public long getMaxBytes();
+ public void add(DrillbitEndpoint endpoint, long bytes);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java
new file mode 100644
index 0000000..64e52eb
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.schedule;
+
+import java.util.Iterator;
+
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+
+import com.carrotsearch.hppc.ObjectLongOpenHashMap;
+import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+
+public class EndpointByteMapImpl implements EndpointByteMap{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EndpointByteMapImpl.class);
+
+ private final ObjectLongOpenHashMap<DrillbitEndpoint> map = new ObjectLongOpenHashMap<>();
+
+ private long maxBytes;
+
+ public boolean isSet(DrillbitEndpoint endpoint){
+ return map.containsKey(endpoint);
+ }
+
+ public long get(DrillbitEndpoint endpoint){
+ return map.get(endpoint);
+ }
+
+ public boolean isEmpty(){
+ return map.isEmpty();
+ }
+
+ public void add(DrillbitEndpoint endpoint, long bytes){
+ assert endpoint != null;
+ maxBytes = Math.max(maxBytes, map.putOrAdd(endpoint, bytes, bytes)+1);
+ }
+
+ public long getMaxBytes() {
+ return maxBytes;
+ }
+
+ @Override
+ public Iterator<ObjectLongCursor<DrillbitEndpoint>> iterator() {
+ return map.iterator();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java
new file mode 100644
index 0000000..bb8d950
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.schedule;
+
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+
+public class PartialWork {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PartialWork.class);
+
+ private final long length;
+ private final DrillbitEndpoint[] locations;
+
+ public PartialWork(long length, DrillbitEndpoint[] locations) {
+ super();
+ this.length = length;
+ this.locations = locations;
+ }
+
+ public long getLength() {
+ return length;
+ }
+ public DrillbitEndpoint[] getLocations() {
+ return locations;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 329815d..baecc3f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -346,7 +346,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
private void runSQL(String sql) {
try{
- DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getSchemaFactory(), context.getFunctionRegistry());
+ DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getFactory(), context.getFunctionRegistry());
LogicalPlan plan = sqlWorker.getPlan(sql);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/storage-engines.json b/exec/java-exec/src/main/resources/storage-engines.json
deleted file mode 100644
index d1d0413..0000000
--- a/exec/java-exec/src/main/resources/storage-engines.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
- "storage":{
- "parquet-local" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- },
- "parquet-cp" :
- {
- "type":"parquet",
- "dfsName" : "classpath:///"
- },
- "jsonl" :
- {
- "type":"json",
- "dfsName" : "file:///"
- },
- "json-cp" :
- {
- "type":"json",
- "dfsName" : "classpath:///"
- },
- "parquet" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- }
- }
-}
\ No newline at end of file
[34/38] git commit: Change FileSelection variable name to selection
(instead of file, which is not an accurate description)
Posted by ja...@apache.org.
Change FileSelection variable name to selection (instead of file, which is not an accurate description)
fix bug when selecting a parquet directory with _metadata file.
handle case where parquet directory does not have _metadata file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/cdfee72c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/cdfee72c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/cdfee72c
Branch: refs/heads/master
Commit: cdfee72c63fa4a09231070d18c65e8ae59436939
Parents: 68d2c38
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Feb 27 19:17:07 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:18 2014 -0800
----------------------------------------------------------------------
.../exec/store/dfs/BasicFormatMatcher.java | 6 ++--
.../drill/exec/store/dfs/FormatMatcher.java | 2 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 3 +-
.../exec/store/parquet/ParquetFormatPlugin.java | 35 ++++++++++++++++----
.../exec/store/ischema/TestTableProvider.java | 5 ++-
5 files changed, 36 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdfee72c/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
index 1c391de..50678a6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
@@ -60,9 +60,9 @@ public class BasicFormatMatcher extends FormatMatcher{
}
@Override
- public FormatSelection isReadable(FileSelection file) throws IOException {
- if(isReadable(file.getFirstPath(fs))){
- return new FormatSelection(plugin.getConfig(), file);
+ public FormatSelection isReadable(FileSelection selection) throws IOException {
+ if(isReadable(selection.getFirstPath(fs))){
+ return new FormatSelection(plugin.getConfig(), selection);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdfee72c/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
index e8521e4..92e3d0a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
@@ -23,6 +23,6 @@ public abstract class FormatMatcher {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatMatcher.class);
public abstract boolean supportDirectoryReads();
- public abstract FormatSelection isReadable(FileSelection file) throws IOException;
+ public abstract FormatSelection isReadable(FileSelection selection) throws IOException;
public abstract FormatPlugin getFormatPlugin();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdfee72c/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index 8a41575..d7949c3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -54,7 +54,8 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
private final FormatPluginConfig formatConfig;
private final String name;
- protected EasyFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, T formatConfig, boolean readable, boolean writable, boolean blockSplittable, String extension, String defaultName){
+ protected EasyFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig,
+ T formatConfig, boolean readable, boolean writable, boolean blockSplittable, String extension, String defaultName){
this.matcher = new BasicFormatMatcher(this, fs, extension);
this.readable = readable;
this.writable = writable;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdfee72c/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index 6d02046..bfaaa45 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.mapred.Utils;
import parquet.format.converter.ParquetMetadataConverter;
import parquet.hadoop.CodecFactoryExposer;
import parquet.hadoop.ParquetFileWriter;
@@ -151,20 +153,39 @@ public class ParquetFormatPlugin implements FormatPlugin{
}
@Override
- public FormatSelection isReadable(FileSelection file) throws IOException {
+ public FormatSelection isReadable(FileSelection selection) throws IOException {
// TODO: we only check the first file for directory reading. This is because
- if(file.containsDirectories(fs)){
- if(isDirReadable(file.getFirstPath(fs))){
- return new FormatSelection(plugin.getConfig(), file);
+ if(selection.containsDirectories(fs)){
+ if(isDirReadable(selection.getFirstPath(fs))){
+ return new FormatSelection(plugin.getConfig(), selection);
}
}
- return super.isReadable(file);
+ return super.isReadable(selection);
}
boolean isDirReadable(FileStatus dir) {
- Path p = new Path(dir.getPath(), "/" + ParquetFileWriter.PARQUET_METADATA_FILE);
+ Path p = new Path(dir.getPath(), ParquetFileWriter.PARQUET_METADATA_FILE);
try {
- return fs.getUnderlying().exists(p);
+ if (fs.getUnderlying().exists(p)) {
+ return true;
+ } else {
+
+ PathFilter filter = new Utils.OutputFileUtils.OutputFilesFilter() {
+ @Override
+ public boolean accept(Path path) {
+ if (path.toString().contains("_metadata")) {
+ return false;
+ }
+ return super.accept(path);
+ }
+ };
+
+ FileStatus[] files = fs.getUnderlying().listStatus(dir.getPath(), filter);
+ if (files.length == 0) {
+ return false;
+ }
+ return super.isReadable(files[0]);
+ }
} catch (IOException e) {
logger.info("Failure while attempting to check for Parquet metadata file.", e);
return false;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdfee72c/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java
index c4da32b..475d2ac 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java
@@ -31,11 +31,9 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.store.ischema.FixedTable;
-import org.apache.drill.exec.store.ischema.PipeProvider;
-import org.apache.drill.exec.store.ischema.RowRecordReader;
import org.apache.drill.exec.vector.ValueVector;
import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
/**
@@ -59,6 +57,7 @@ public class TestTableProvider {
}
@Test
+ @Ignore // due to out of heap space
public void largeRead() {
readTestTable(1024*1024);
}
[38/38] git commit: INSTALL.md and packaging fixes given recent
merges.
Posted by ja...@apache.org.
INSTALL.md and packaging fixes given recent merges.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4871fd0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4871fd0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4871fd0d
Branch: refs/heads/master
Commit: 4871fd0de300fb099152697239e4c9548b8ec5c0
Parents: cdfee72
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Mar 4 00:03:12 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 4 00:03:12 2014 -0800
----------------------------------------------------------------------
INSTALL.md | 18 ++++++++----
distribution/pom.xml | 6 +++-
distribution/src/assemble/bin.xml | 13 ++++++++
distribution/src/resources/storage-engines.json | 5 +++-
.../exec/store/ischema/InfoSchemaGroupScan.java | 2 +-
.../store/ischema/InfoSchemaStoragePlugin.java | 5 ++--
sqlline | 31 --------------------
7 files changed, 37 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4871fd0d/INSTALL.md
----------------------------------------------------------------------
diff --git a/INSTALL.md b/INSTALL.md
index e89da01..aeab9f4 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -23,17 +23,23 @@ Currently, the Apache Drill build process is known to work on Linux, Windows and
cd incubator-drill
mvn clean install
-
-## Start SQLLine
- ./sqlline -u jdbc:drill:zk=local -n admin -p admin
+## Explode tarball in installation directory
+
+ mkdir /opt/drill
+ tar xvzf distribution/target/*.tar.gz --strip=1 -C /opt/drill
+
+## Start SQLLine (which starts Drill in embedded mode)
+
+ cd /opt/drill
+ bin/sqlline -u jdbc:drill:zk=local -n admin -p admin
## Run a query
SELECT
- region_key,
- r_comment
- FROM "sample-data/region.parquet";
+ employee_id,
+ first_name
+ FROM cp.`employee.json`;
## More information
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4871fd0d/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 3ad42cb..a77e419 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -28,7 +28,11 @@
<groupId>sqlline</groupId>
<artifactId>sqlline</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>storage-hive</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.drill</groupId>
<artifactId>drill-protocol</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4871fd0d/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index 60aea67..fbc98b8 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -37,6 +37,7 @@
<include>org.apache.drill:drill-protocol:jar:rebuffed</include>
<include>org.apache.drill:drill-common:jar:rebuffed</include>
<include>org.apache.drill.exec:drill-java-exec:jar:rebuffed</include>
+ <include>org.apache.drill:storage-hive:jar</include>
</includes>
@@ -76,6 +77,18 @@
<files>
<file>
+ <source>../KEYS</source>
+ <outputDirectory></outputDirectory>
+ </file>
+ <file>
+ <source>../LICENSE</source>
+ <outputDirectory></outputDirectory>
+ </file>
+ <file>
+ <source>../NOTICE</source>
+ <outputDirectory></outputDirectory>
+ </file>
+ <file>
<source>src/resources/runbit</source>
<outputDirectory>bin</outputDirectory>
</file>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4871fd0d/distribution/src/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/distribution/src/resources/storage-engines.json b/distribution/src/resources/storage-engines.json
index a22e7d0..8b22858 100644
--- a/distribution/src/resources/storage-engines.json
+++ b/distribution/src/resources/storage-engines.json
@@ -7,7 +7,9 @@
cp: {
type: "file",
connection: "classpath:///"
- },
+ }
+
+ /*,
hive : {
type:"hive",
config :
@@ -19,5 +21,6 @@
"hive.metastore.sasl.enabled" : "false"
}
}
+ */
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4871fd0d/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
index ac625b7..b8b66dc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
@@ -33,7 +33,7 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.hive12.common.base.Preconditions;
+import com.google.common.base.Preconditions;
@JsonTypeName("info-schema")
public class InfoSchemaGroupScan extends AbstractGroupScan{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4871fd0d/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
index 04b027c..891419d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
@@ -24,7 +24,6 @@ import java.util.Set;
import net.hydromatic.optiq.Schema;
import net.hydromatic.optiq.SchemaPlus;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.data.Scan;
import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.server.DrillbitContext;
@@ -32,8 +31,8 @@ import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.AbstractStoragePlugin;
import org.apache.drill.exec.store.SchemaHolder;
-import com.google.hive12.common.collect.ImmutableMap;
-import com.google.hive12.common.collect.Maps;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
public class InfoSchemaStoragePlugin extends AbstractStoragePlugin{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaStoragePlugin.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4871fd0d/sqlline
----------------------------------------------------------------------
diff --git a/sqlline b/sqlline
deleted file mode 100755
index 8cc9531..0000000
--- a/sqlline
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# sqlline - Script to launch SQL shell
-#
-# Example:
-# $ ./sqlline
-# sqlline> !connect jdbc:optiq:model=common/target/test-classes/donuts-model.json admin admin
-
-# Build classpath on first call. (To force rebuild, remove .classpath.)
-if [ ! -f .classpath ]; then
- (cd contrib/sqlline; mvn dependency:build-classpath -Dmdep.outputFile=../../.classpath)
-fi
-
-#exec java -Drebel.log=true -javaagent:/src/jrebel/jrebel.jar -Dlogback.configurationFile=/src/drill/sandbox/prototype/sqlparser/src/test/resources/logback.xml -cp ./:"$(cat .classpath)" sqlline.SqlLine "$@"
-exec java -Xmx2G -Dlogback.configurationFile=./sqlparser/src/test/resources/logback.xml -cp ./:"$(cat .classpath)" sqlline.SqlLine --verbose=true "$@"
-
-
-# End sqlline
[27/38] git commit: DRILL-364: Fix VarChar casting
Posted by ja...@apache.org.
DRILL-364: Fix VarChar casting
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/153c727a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/153c727a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/153c727a
Branch: refs/heads/master
Commit: 153c727a6d393203501f3cfed2535a0fb0601061
Parents: ee9eaa1
Author: Jinfeng Ni <jn...@maprtech.com>
Authored: Sun Feb 23 16:02:26 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../common/expression/OutputTypeDeterminer.java | 20 +++++
.../common/expression/fn/CastFunctionDefs.java | 14 ++--
exec/java-exec/src/main/codegen/data/Casts.tdd | 4 +-
.../exec/physical/impl/TestCastFunctions.java | 80 +++++++++++++++++++-
.../functions/cast/testCastVarCharNull.json | 37 +++++++++
.../src/test/resources/jsoninput/input1.json | 6 ++
6 files changed, 148 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
index 66523c4..69acf12 100644
--- a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
+++ b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
@@ -74,4 +74,24 @@ public interface OutputTypeDeterminer {
}
}
+ public static class NullIfNullType implements OutputTypeDeterminer{
+ public MinorType outputMinorType;
+
+
+ public NullIfNullType(MinorType outputType) {
+ super();
+ this.outputMinorType = outputType;
+ }
+
+ @Override
+ public MajorType getOutputType(List<LogicalExpression> expressions) {
+ for(LogicalExpression e : expressions){
+ if(e.getMajorType().getMode() == DataMode.OPTIONAL){
+ return Types.optional(outputMinorType);
+ }
+ }
+ return Types.required(outputMinorType);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
index 6a98f94..4be3820 100644
--- a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
+++ b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
@@ -25,13 +25,13 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
public class CastFunctionDefs implements CallProvider{
- private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT", new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_BIGINT);
- private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT", new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_INT);
- private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4", new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_FLOAT4);
- private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8", new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_FLOAT8);
- private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR", new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VARCHAR);
- private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY", new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VARBINARY);
- private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR", new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VAR16CHAR);
+ private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT));
+ private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.INT));
+ private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT4));
+ private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT8));
+ private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR));
+ private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARBINARY));
+ private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VAR16CHAR));
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/main/codegen/data/Casts.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd
index ceb9cde..fde63cf 100644
--- a/exec/java-exec/src/main/codegen/data/Casts.tdd
+++ b/exec/java-exec/src/main/codegen/data/Casts.tdd
@@ -39,11 +39,11 @@
{from: "VarBinary", to: "Float8", major: "SrcVarlen", javaType:"Double", parse:"Double"},
{from: "BigInt", to: "VarChar", major: "TargetVarlen", javaType: "Long", bufferLength:"20"},
- {from: "Int", to: "VarChar", major: "TargetVarlen", javaType: "Integer", bufferLength:"10"},
+ {from: "Int", to: "VarChar", major: "TargetVarlen", javaType: "Integer", bufferLength:"11"},
{from: "Float4", to: "VarChar", major: "TargetVarlen", javaType: "Float", bufferLength:"100"},
{from: "Float8", to: "VarChar", major: "TargetVarlen", javaType: "Double", bufferLength:"100"},
{from: "BigInt", to: "VarBinary", major: "TargetVarlen", javaType: "Long", bufferLength:"20"},
- {from: "Int", to: "VarBinary", major: "TargetVarlen", javaType: "Integer", bufferLength:"10"},
+ {from: "Int", to: "VarBinary", major: "TargetVarlen", javaType: "Integer", bufferLength:"11"},
{from: "Float4", to: "VarBinary", major: "TargetVarlen", javaType: "Float", bufferLength:"100"},
{from: "Float8", to: "VarBinary", major: "TargetVarlen", javaType: "Double", bufferLength:"100"},
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
index 3010c37..4d18815 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
@@ -19,6 +19,9 @@ package org.apache.drill.exec.physical.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
import mockit.Injectable;
import mockit.NonStrictExpectations;
@@ -26,6 +29,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.expr.holders.Float4Holder;
@@ -38,34 +42,44 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.rpc.user.UserServer;
+import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.Float4Vector;
import org.apache.drill.exec.vector.Float8Vector;
import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
import org.junit.After;
+import org.junit.Ignore;
import org.junit.Test;
import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-public class TestCastFunctions {
+public class TestCastFunctions extends PopUnitTestBase{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class);
DrillConfig c = DrillConfig.create();
+
@Test
// cast to bigint.
public void testCastBigInt(@Injectable final DrillbitContext bitContext,
@Injectable UserServer.UserClientConnection connection) throws Throwable{
-
new NonStrictExpectations(){{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = new TopLevelAllocator();
@@ -342,7 +356,6 @@ public class TestCastFunctions {
}
@Test(expected = NumberFormatException.class)
- //nested: cast is nested in another cast, or another function.
public void testCastNumException(@Injectable final DrillbitContext bitContext,
@Injectable UserServer.UserClientConnection connection) throws Throwable{
@@ -370,6 +383,65 @@ public class TestCastFunctions {
}
+ @Test
+ public void testCastFromNullablCol() throws Throwable {
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ try(Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
+ bit.run();
+
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
+ Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"), Charsets.UTF_8).replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/jsoninput/input1.json").toURI().toString()));
+
+ QueryResultBatch batch = results.get(0);
+
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
+ batchLoader.load(batch.getHeader().getDef(), batch.getData());
+
+ Object [][] result = getRunResult(batchLoader);
+
+ Object [][] expected = new Object[2][2];
+
+ expected[0][0] = new String("2001");
+ expected[0][1] = new String("1.2");
+
+ expected[1][0] = new String("-2002");
+ expected[1][1] = new String("-1.2");
+
+ assertEquals(result.length, expected.length);
+ assertEquals(result[0].length, expected[0].length);
+
+ for (int i = 0; i<result.length; i++ ) {
+ for (int j = 0; j<result[0].length; j++) {
+ assertEquals(String.format("Column %s at row %s have wrong result", j, i), result[i][j], expected[i][j]);
+ }
+ }
+ }
+ }
+
+ private Object[][] getRunResult(VectorAccessible va) {
+ int size = 0;
+ for (VectorWrapper v : va) {
+ size++;
+ }
+
+ Object[][] res = new Object [va.getRecordCount()][size];
+ for (int j = 0; j < va.getRecordCount(); j++) {
+ int i = 0;
+ for (VectorWrapper v : va) {
+ Object o = v.getValueVector().getAccessor().getObject(j);
+ if (o instanceof byte[]) {
+ res[j][i++] = new String((byte[]) o);
+ } else {
+ res[j][i++] = o;
+ }
+ }
+ }
+ return res;
+ }
+
@After
public void tearDown() throws Exception{
// pause to get logger to catch up.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
new file mode 100644
index 0000000..030b696
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
@@ -0,0 +1,37 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"json-scan",
+ entries: [
+ {
+ path : "#{TEST_FILE}"
+ }
+ ],
+ storageengine: {
+ "type": "json",
+ "dfsName": "file:///"
+ }
+ }, {
+ @id:2,
+ child: 1,
+ pop:"project",
+ exprs: [
+ { ref: "int2varchar", expr:"cast(integer as varchar(20))" },
+ { ref: "float2varchar", expr:"cast(float as varchar(20))" }
+ ]
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/resources/jsoninput/input1.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/jsoninput/input1.json b/exec/java-exec/src/test/resources/jsoninput/input1.json
new file mode 100644
index 0000000..e9bde7e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/jsoninput/input1.json
@@ -0,0 +1,6 @@
+{ "integer" : 2001,
+ "float" : 1.2
+}
+{ "integer" : -2002,
+ "float" : -1.2
+}
[35/38] git commit: Update hive to use generated data instead of
packaged data. Remove unused test.
Posted by ja...@apache.org.
Update hive to use generated data instead of packaged data. Remove unused test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/68d2c383
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/68d2c383
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/68d2c383
Branch: refs/heads/master
Commit: 68d2c38391deb35fc8022b1c6f9914e2bbca74a1
Parents: 0ca797a
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Mar 2 18:29:27 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:18 2014 -0800
----------------------------------------------------------------------
.../store/ischema/InfoSchemaBatchCreator.java | 17 ++++
.../exec/store/ischema/InfoSchemaConfig.java | 17 ++++
.../store/ischema/InfoSchemaDrillTable.java | 17 ++++
.../exec/store/ischema/InfoSchemaGroupScan.java | 17 ++++
.../store/ischema/InfoSchemaStoragePlugin.java | 17 ++++
.../exec/store/ischema/InfoSchemaSubScan.java | 17 ++++
.../drill/exec/store/ischema/SelectedTable.java | 17 ++++
.../drill/exec/store/TestOrphanSchema.java | 17 ++++
.../exec/store/hive/HiveTestDataGenerator.java | 97 ++++++++++++++++++++
.../drill/exec/store/hive/TestHiveScan.java | 67 --------------
.../drill/exec/store/ischema/OrphanSchema.java | 17 ++++
.../src/test/resources/storage-engines.json | 16 +---
sqlparser/pom.xml | 20 ----
.../apache/drill/jdbc/test/TestJdbcQuery.java | 15 ++-
.../src/test/resources/storage-engines.json | 2 +-
15 files changed, 264 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java
index 68844e1..a400245 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store.ischema;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java
index 1447eb3..d0ae77a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store.ischema;
import org.apache.drill.common.logical.StoragePluginConfig;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java
index 7b6cb32..20609b8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store.ischema;
import org.apache.drill.common.logical.StoragePluginConfig;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
index 1985d6a..ac625b7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store.ischema;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
index e5ab158..04b027c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store.ischema;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java
index 20375ca..dce4d3b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store.ischema;
import org.apache.drill.exec.physical.base.AbstractSubScan;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java
index 3e354f8..1dd7c8a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store.ischema;
import net.hydromatic.optiq.SchemaPlus;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
index 2087dc2..429d5fe 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store;
import mockit.NonStrictExpectations;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
new file mode 100644
index 0000000..7155a52
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.store.hive;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+public class HiveTestDataGenerator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveTestDataGenerator.class);
+
+ static int RETRIES = 5;
+ private Driver hiveDriver = null;
+
+ public static void main(String[] args) throws Exception {
+ HiveTestDataGenerator htd = new HiveTestDataGenerator();
+ htd.generateTestData();
+ }
+
+ public void generateTestData() throws Exception {
+ HiveConf conf = new HiveConf();
+
+ conf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=/tmp/drill_hive_db;create=true");
+ conf.set("fs.default.name", "file:///");
+ conf.set("hive.metastore.warehouse.dir", "/tmp/drill_hive_wh");
+
+ String tableName = "kv";
+
+ SessionState ss = new SessionState(new HiveConf(SessionState.class));
+ SessionState.start(ss);
+ hiveDriver = new Driver(conf);
+ executeQuery(String.format("CREATE TABLE IF NOT EXISTS default.kv(key INT, value STRING) "+
+ "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE", tableName));
+ executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE %s", generateTestDataFile(), tableName));
+ ss.close();
+ }
+
+ private String generateTestDataFile() throws Exception {
+ File file = null;
+ while (true) {
+ file = File.createTempFile("drill-hive-test", ".txt");
+ if (file.exists()) {
+ boolean success = file.delete();
+ if (success) {
+ break;
+ }
+ }
+ logger.debug("retry creating tmp file");
+ }
+
+ PrintWriter printWriter = new PrintWriter(file);
+ for (int i=1; i<=5; i++)
+ printWriter.println (String.format("%d, key_%d", i, i));
+ printWriter.close();
+
+ return file.getPath();
+ }
+
+ private void executeQuery(String query) {
+ CommandProcessorResponse response = null;
+ boolean failed = false;
+ int retryCount = RETRIES;
+
+ try {
+ response = hiveDriver.run(query);
+ } catch(CommandNeedRetryException ex) {
+ if (--retryCount == 0)
+ failed = true;
+ }
+
+ if (failed || response.getResponseCode() != 0 )
+ throw new RuntimeException(String.format("Failed to execute command '%s', errorMsg = '%s'",
+ query, (response != null ? response.getErrorMessage() : "")));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
deleted file mode 100644
index c6edc20..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.hive;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.pop.PopUnitTestBase;
-import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Created with IntelliJ IDEA.
- * User: sphillips
- * Date: 1/23/14
- * Time: 5:22 AM
- * To change this template use File | Settings | File Templates.
- */
-public class TestHiveScan extends PopUnitTestBase {
- @Ignore
- @Test
- public void twoBitTwoExchangeTwoEntryRun() throws Exception {
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-
- try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
- Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
-
- bit1.run();
- bit2.run();
- client.connect();
- List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/hive/test.json"),
- Charsets.UTF_8));
- int count = 0;
- for(QueryResultBatch b : results) {
- if (b.getHeader().getRowCount() != 0)
- count += b.getHeader().getRowCount();
- }
- assertEquals(100, count);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
index 268b844..ee36493 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.drill.exec.store.ischema;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/exec/java-exec/src/test/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/storage-engines.json b/exec/java-exec/src/test/resources/storage-engines.json
index 7d3dc38..58f091e 100644
--- a/exec/java-exec/src/test/resources/storage-engines.json
+++ b/exec/java-exec/src/test/resources/storage-engines.json
@@ -3,18 +3,6 @@
dfs: {
type: "file",
connection: "file:///"
- },
- hive :
- {
- type:"hive",
- config :
- {
- "hive.metastore.uris" : "",
- "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true",
- "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
- "fs.default.name" : "file:///",
- "hive.metastore.sasl.enabled" : "false"
- }
- }
- }
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/sqlparser/pom.xml
----------------------------------------------------------------------
diff --git a/sqlparser/pom.xml b/sqlparser/pom.xml
index 90e044f..f0606f1 100644
--- a/sqlparser/pom.xml
+++ b/sqlparser/pom.xml
@@ -91,26 +91,6 @@
</excludes>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>test</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <tasks>
- <copy todir="/tmp/drill_hive_wh">
- <fileset dir="${basedir}/../sample-data/drill_hive_wh"/>
- </copy>
- </tasks>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
<profiles>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index f5081b3..1d678cc 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -26,7 +26,9 @@ import java.sql.Statement;
import java.util.concurrent.TimeUnit;
import org.apache.drill.common.util.TestTools;
+import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
import org.apache.drill.jdbc.Driver;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
@@ -37,8 +39,7 @@ import com.google.common.base.Stopwatch;
public class TestJdbcQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class);
- static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
-
+
// Set a timeout unless we're debugging.
@Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(20000);
@@ -46,17 +47,23 @@ public class TestJdbcQuery {
static{
Driver.load();
WORKING_PATH = Paths.get("").toAbsolutePath().toString();
+
+ }
+
+ @BeforeClass
+ public static void generateHive() throws Exception{
+ new HiveTestDataGenerator().generateTestData();
}
@Test
public void testHiveRead() throws Exception{
- testQuery("select * from hive.kv_text");
+ testQuery("select * from hive.kv");
}
@Test
@Ignore // something not working here.
public void testHiveReadWithDb() throws Exception{
- testQuery("select * from hive.`default`.kv_text");
+ testQuery("select * from hive.`default`.kv");
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68d2c383/sqlparser/src/test/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/resources/storage-engines.json b/sqlparser/src/test/resources/storage-engines.json
index fca6b68..c16a971 100644
--- a/sqlparser/src/test/resources/storage-engines.json
+++ b/sqlparser/src/test/resources/storage-engines.json
@@ -17,7 +17,7 @@
config :
{
"hive.metastore.uris" : "",
- "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../sample-data/drill_hive_db;create=true",
+ "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=/tmp/drill_hive_db;create=true",
"hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
"fs.default.name" : "file:///",
"hive.metastore.sasl.enabled" : "false"
[07/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillColumnMetaDataList.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillColumnMetaDataList.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillColumnMetaDataList.java
new file mode 100644
index 0000000..5119212
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillColumnMetaDataList.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.sql.ResultSetMetaData;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.MaterializedField;
+
+import net.hydromatic.avatica.ColumnMetaData;
+
+public class DrillColumnMetaDataList extends BasicList<ColumnMetaData>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillColumnMetaDataList.class);
+
+ private ColumnMetaData[] columns = new ColumnMetaData[0];
+
+ @Override
+ public int size() {
+ return columns.length;
+ }
+
+ @Override
+ public ColumnMetaData get(int index) {
+ return columns[index];
+ }
+
+ public void updateColumnMetaData(String catalogName, String schemaName, String tableName, BatchSchema schema){
+
+ columns = new ColumnMetaData[schema.getFieldCount()];
+ for(int i = 0; i < schema.getFieldCount(); i++){
+ MaterializedField f = schema.getColumn(i);
+ MajorType t = f.getType();
+ ColumnMetaData col = new ColumnMetaData( //
+ i, // ordinal
+ false, // autoIncrement
+ true, // caseSensitive
+ false, // searchable
+ false, // currency
+ f.getDataMode() == DataMode.OPTIONAL ? ResultSetMetaData.columnNullable : ResultSetMetaData.columnNoNulls, //nullability
+ !Types.isUnSigned(t), // signed
+ 10, // display size.
+ f.getName(), // label
+ f.getName(), // columnname
+ schemaName, // schemaname
+ t.hasPrecision() ? t.getPrecision() : 0, // precision
+ t.hasScale() ? t.getScale() : 0, // scale
+ null, // tablename is null so sqlline doesn't try to retrieve primary keys.
+ catalogName, // catalogname
+ Types.getSqlType(t), // sql type
+ t.getMinorType().name()+":" + t.getMode().name(), // typename
+ true, // readonly
+ false, // writable
+ false, // definitely writable
+ "none", // column class name
+ ColumnMetaData.Rep.BOOLEAN // Dummy value for representation as it doesn't apply in drill's case.
+ );
+ columns[i] =col;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java
new file mode 100644
index 0000000..ab1259a
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.jdbc;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+
+import org.apache.drill.exec.client.DrillClient;
+
+
+public interface DrillConnection extends Connection{
+
+ // in java.sql.Connection from JDK 1.7, but declare here to allow other JDKs
+ void setSchema(String schema) throws SQLException;
+
+ // in java.sql.Connection from JDK 1.7, but declare here to allow other JDKs
+ String getSchema() throws SQLException;
+
+ /** Returns a view onto this connection's configuration properties. Code
+ * within Optiq should use this view rather than calling
+ * {@link java.util.Properties#getProperty(String)}. */
+ ConnectionConfig config();
+
+ public DrillClient getClient();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java
new file mode 100644
index 0000000..11a1157
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.io.IOException;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Properties;
+import java.util.TimeZone;
+
+import net.hydromatic.avatica.AvaticaConnection;
+import net.hydromatic.avatica.AvaticaFactory;
+import net.hydromatic.avatica.Helper;
+import net.hydromatic.avatica.Meta;
+import net.hydromatic.avatica.UnregisteredDriver;
+
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.RemoteServiceSet;
+
+/**
+ * Implementation of JDBC connection in Drill.
+ *
+ * <p>
+ * Abstract to allow newer versions of JDBC to add methods.
+ * </p>
+ */
+abstract class DrillConnectionImpl extends AvaticaConnection implements org.apache.drill.jdbc.DrillConnection {
+ public final DrillStatementRegistry registry = new DrillStatementRegistry();
+ final ConnectionConfig config;
+
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConnection.class);
+
+ private final DrillClient client;
+ private final BufferAllocator allocator;
+ private Drillbit bit;
+ private RemoteServiceSet serviceSet;
+
+ protected DrillConnectionImpl(Driver driver, AvaticaFactory factory, String url, Properties info) throws SQLException{
+ super(driver, factory, url, info);
+ this.config = new ConnectionConfig(info);
+
+ this.allocator = new TopLevelAllocator();
+
+ try{
+ if(config.isLocal()){
+ RemoteServiceSet set = GlobalServiceSetReference.SETS.get();
+ if(set == null){
+ // we're embedded, start a local drill bit.
+ serviceSet = RemoteServiceSet.getLocalServiceSet();
+ set = serviceSet;
+ try{
+ bit = new Drillbit(driver.getConfig(), serviceSet);
+ bit.run();
+ }catch(Exception e){
+ throw new SQLException("Failure while attempting to start Drillbit in embedded mode.", e);
+ }
+ }else{
+ serviceSet = null;
+ bit = null;
+ }
+ this.client = new DrillClient(driver.getConfig(), set.getCoordinator());
+ this.client.connect(null);
+ }else{
+ this.client = new DrillClient();
+ this.client.connect(config.getZookeeperConnectionString());
+ }
+ }catch(RpcException e){
+ throw new SQLException("Failure while attempting to connect to Drill.", e);
+ }
+ }
+
+
+ public ConnectionConfig config(){
+ return config;
+ }
+
+ @Override
+ protected Meta createMeta() {
+ return new MetaImpl(this);
+ }
+
+ MetaImpl meta() {
+ return (MetaImpl) meta;
+ }
+
+ BufferAllocator getAllocator(){
+ return allocator;
+ }
+
+ public DrillClient getClient(){
+ return client;
+ }
+
+ @Override
+ public DrillStatement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability)
+ throws SQLException {
+ DrillStatement statement = (DrillStatement) super.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability);
+ registry.addStatement(statement);
+ return statement;
+ }
+
+ @Override
+ public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency,
+ int resultSetHoldability) throws SQLException {
+ try {
+ DrillPrepareResult prepareResult = new DrillPrepareResult(sql);
+ DrillPreparedStatement statement = (DrillPreparedStatement) factory.newPreparedStatement(this, prepareResult,
+ resultSetType, resultSetConcurrency, resultSetHoldability);
+ registry.addStatement(statement);
+ return statement;
+ } catch (RuntimeException e) {
+ throw Helper.INSTANCE.createException("Error while preparing statement [" + sql + "]", e);
+ } catch (Exception e) {
+ throw Helper.INSTANCE.createException("Error while preparing statement [" + sql + "]", e);
+ }
+ }
+
+ @Override
+ public TimeZone getTimeZone() {
+ return config.getTimeZone();
+ }
+
+ // do not make public
+ UnregisteredDriver getDriver() {
+ return driver;
+ }
+
+ // do not make public
+ AvaticaFactory getFactory() {
+ return factory;
+ }
+
+ void cleanup(){
+ client.close();
+ allocator.close();
+ if(bit != null) bit.close();
+
+ if(serviceSet != null){
+ try{
+ serviceSet.close();
+ }catch(IOException e){
+ logger.warn("Exception while closing service set.", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillCursor.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillCursor.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillCursor.java
new file mode 100644
index 0000000..1145b84
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillCursor.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.sql.SQLException;
+import java.util.Calendar;
+import java.util.List;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import net.hydromatic.avatica.ColumnMetaData;
+import net.hydromatic.avatica.Cursor;
+
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.jdbc.DrillResultSet.Listener;
+
+public class DrillCursor implements Cursor{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillCursor.class);
+
+ private static final String UNKNOWN = "--UNKNOWN--";
+
+ private boolean started = false;
+ private boolean finished = false;
+ private final RecordBatchLoader currentBatch;
+ private final DrillResultSet.Listener listener;
+ private boolean redoFirstNext = false;
+
+ private DrillColumnMetaDataList columnMetaDataList;
+
+ final DrillResultSet results;
+ int currentRecord = 0;
+ private long recordBatchCount;
+ private final DrillAccessorList accessors = new DrillAccessorList();
+
+
+ public DrillCursor(DrillResultSet results) {
+ super();
+ this.results = results;
+ currentBatch = results.currentBatch;
+ this.listener = results.listener;
+ }
+
+ @Override
+ public List<Accessor> createAccessors(List<ColumnMetaData> types, Calendar localCalendar) {
+ columnMetaDataList = (DrillColumnMetaDataList) types;
+ return accessors;
+ }
+
+ @Override
+ public boolean next() throws SQLException {
+ if(!started){
+ started = true;
+ redoFirstNext = true;
+ }else if(redoFirstNext && !finished){
+ redoFirstNext = false;
+ return true;
+ }
+
+ if(finished) return false;
+
+ if(currentRecord+1 < currentBatch.getRecordCount()){
+ currentRecord++;
+ return true;
+ }else{
+ try {
+ QueryResultBatch qrb = listener.getNext();
+ recordBatchCount++;
+ while(qrb != null && qrb.getHeader().getRowCount() == 0 ){
+ qrb.release();
+ qrb = listener.getNext();
+ recordBatchCount++;
+ }
+
+ if(qrb == null){
+ finished = true;
+ return false;
+ }else{
+ currentRecord = 0;
+ boolean changed = currentBatch.load(qrb.getHeader().getDef(), qrb.getData());
+ if(changed) updateColumns();
+ return true;
+ }
+ } catch (RpcException | InterruptedException | SchemaChangeException e) {
+ throw new SQLException("Failure while trying to get next result batch.", e);
+ }
+
+ }
+ }
+
+ void updateColumns(){
+ accessors.generateAccessors(this, currentBatch);
+ columnMetaDataList.updateColumnMetaData(UNKNOWN, UNKNOWN, UNKNOWN, currentBatch.getSchema());
+ if(results.changeListener != null) results.changeListener.schemaChanged(currentBatch.getSchema());
+ }
+
+ public long getRecordBatchCount(){
+ return recordBatchCount;
+ }
+
+ @Override
+ public void close() {
+ results.cleanup();
+ }
+
+ @Override
+ public boolean wasNull() throws SQLException {
+ return accessors.wasNull();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java
deleted file mode 100644
index e395515..0000000
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.jdbc;
-
-import net.hydromatic.optiq.jdbc.DriverVersion;
-
-/**
- * Version information for Drill JDBC Driver.
- */
-class DrillDriverVersion extends DriverVersion {
- /** Creates a DrillDriverVersion. */
- DrillDriverVersion() {
- super(
- "Apache Drill JDBC Driver",
- "1.0",
- "Drill",
- "1.0",
- true,
- 1,
- 0,
- 1,
- 0);
- }
-}
-
-
-// End DrillDriverVersion.java
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillFactory.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillFactory.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillFactory.java
new file mode 100644
index 0000000..9b9eb7b
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillFactory.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.jdbc;
+
+import java.sql.SQLException;
+import java.util.Properties;
+
+import net.hydromatic.avatica.AvaticaConnection;
+import net.hydromatic.avatica.AvaticaFactory;
+import net.hydromatic.avatica.UnregisteredDriver;
+
+/**
+ * Extension of {@link net.hydromatic.avatica.AvaticaFactory} for Drill.
+ */
+public abstract class DrillFactory implements AvaticaFactory {
+ protected final int major;
+ protected final int minor;
+
+ /** Creates a JDBC factory with given major/minor version number. */
+ protected DrillFactory(int major, int minor) {
+ this.major = major;
+ this.minor = minor;
+ }
+
+ public int getJdbcMajorVersion() {
+ return major;
+ }
+
+ public int getJdbcMinorVersion() {
+ return minor;
+ }
+
+ public final AvaticaConnection newConnection(UnregisteredDriver driver, AvaticaFactory factory, String url,
+ Properties info) throws SQLException{
+ return newDrillConnection((Driver) driver, (DrillFactory) factory, url, info);
+ }
+
+ /** Creates a connection with a root schema. */
+ public abstract DrillConnectionImpl newDrillConnection(Driver driver, DrillFactory factory, String url,
+ Properties info) throws SQLException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
index 4712302..592bff3 100644
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
@@ -17,193 +17,31 @@
*/
package org.apache.drill.jdbc;
-import java.io.IOException;
import java.sql.SQLException;
-import java.util.Map;
-import java.util.Properties;
-import net.hydromatic.linq4j.QueryProvider;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.optiq.MutableSchema;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.impl.java.JavaTypeFactory;
-import net.hydromatic.optiq.impl.java.MapSchema;
-import net.hydromatic.optiq.jdbc.HandlerImpl;
-import net.hydromatic.optiq.jdbc.OptiqConnection;
-import net.hydromatic.optiq.model.ModelHandler;
+import net.hydromatic.avatica.AvaticaConnection;
+import net.hydromatic.avatica.AvaticaStatement;
+import net.hydromatic.avatica.Handler;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.coord.ClusterCoordinator;
-import org.apache.drill.exec.coord.ZKClusterCoordinator;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.drill.exec.store.SchemaProviderRegistry;
-import org.apache.drill.exec.store.hive.HiveStorageEngine.HiveSchemaProvider;
-import org.apache.drill.exec.store.json.JsonSchemaProvider;
-import org.apache.drill.exec.store.parquet.ParquetSchemaProvider;
-import org.apache.drill.sql.client.full.FileSystemSchema;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.io.Resources;
-import org.apache.drill.sql.client.full.HiveSchema;
-
-public class DrillHandler extends HandlerImpl {
+public class DrillHandler implements Handler{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHandler.class);
- private ClusterCoordinator coordinator;
- private volatile DrillClient client;
- private Drillbit bit;
- private DrillConfig config = DrillConfig.create();
- private SchemaProviderRegistry registry;
- private final boolean ref;
-
- public DrillHandler(boolean ref){
- this.ref = ref;
- }
-
- public void onConnectionInit(OptiqConnection connection) throws SQLException {
- super.onConnectionInit(connection);
-
- final Properties p = connection.getProperties();
-
- if (ref) {
- final String model = p.getProperty("model");
- if (model != null) {
- if (model != null) {
- try {
- new ModelHandler(connection, model);
- } catch (IOException e) {
- throw new SQLException(e);
- }
- }
- }
- } else {
-
- registry = new SchemaProviderRegistry(config);
-
- Preconditions.checkArgument(bit == null);
- Preconditions.checkArgument(client == null);
- Preconditions.checkArgument(coordinator == null);
-
- final String zk = connection.getProperties().getProperty("zk");
-
- try {
- String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
-
- StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class);
-
- if (zk != null) {
- coordinator = new ZKClusterCoordinator(config, zk);
- coordinator.start(10000);
- DrillClient cl = new DrillClient(config, coordinator);
- cl.connect();
- client = cl;
- } else {
-
- RemoteServiceSet local = RemoteServiceSet.getLocalServiceSet();
- this.coordinator = local.getCoordinator();
- bit = new Drillbit(config, local);
- bit.run();
-
- DrillClient cl = new DrillClient(config, coordinator);
- cl.connect();
- client = cl;
- }
-
- MutableSchema rootSchema = connection.getRootSchema();
-
- for (Map.Entry<String, StorageEngineConfig> entry : engines) {
- SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
- Schema schema = getSchema(provider, client, entry.getKey(), entry.getValue(), rootSchema);
- rootSchema.addSchema(entry.getKey(), schema);
- }
-
- rootSchema.addSchema(
- "--FAKE--",
- new FakeSchema(rootSchema, rootSchema.getQueryProvider(), rootSchema.getTypeFactory(), "fake", rootSchema
- .getExpression()));
-
- } catch (Exception ex) {
- System.out.println(ex);
- logger.error("Failure while setting up jdbc handler", ex);
- throw new SQLException("Failure trying to connect to Drill.", ex);
- }
- }
-
- // The "schema" parameter currently gives a name to the schema. In future
- // it will choose a schema that (presumably) already exists.
- final String schemaName = connection.getProperties().getProperty("schema");
- if (schemaName != null) {
- connection.setSchema(schemaName);
- }
-
- final String catalogName = connection.getProperties().getProperty("catalog");
- if (catalogName != null) {
- connection.setCatalog(catalogName);
- }
- }
-
- private Schema getSchema(SchemaProvider provider, DrillClient client, String name, StorageEngineConfig config, Schema rootSchema)
- throws SQLException {
- if (provider instanceof ParquetSchemaProvider || provider instanceof JsonSchemaProvider) {
- return new FileSystemSchema(client, config, provider,
- rootSchema.getTypeFactory(), rootSchema, name, rootSchema.getExpression(),
- rootSchema.getQueryProvider());
- } else if (provider instanceof HiveSchemaProvider) {
- return new HiveSchema(client, config, provider,
- rootSchema.getTypeFactory(), rootSchema, name, rootSchema.getExpression(),
- rootSchema.getQueryProvider());
- }
-
- throw new SQLException("Unknown schema provider");
+ @Override
+ public void onConnectionInit(AvaticaConnection c) throws SQLException {
}
- public class FakeSchema extends MapSchema {
-
- public FakeSchema(Schema parentSchema, QueryProvider queryProvider, JavaTypeFactory typeFactory, String name,
- Expression expression) {
- super(parentSchema, queryProvider, typeFactory, name, expression);
-
- }
-
- public DrillClient getClient() {
- return client;
- }
+ @Override
+ public void onConnectionClose(AvaticaConnection c) throws RuntimeException {
+ DrillConnectionImpl connection = (DrillConnectionImpl) c;
+ connection.cleanup();
}
- public DrillClient getClient() {
- return client;
+ @Override
+ public void onStatementExecute(AvaticaStatement statement, ResultSink resultSink) throws RuntimeException {
}
@Override
- public void onConnectionClose(OptiqConnection connection) throws RuntimeException {
- super.onConnectionClose(connection);
- if (client != null)
- client.close();
- if (bit != null)
- bit.close();
- if (coordinator != null)
- try {
- coordinator.close();
- } catch (IOException e) {
- throw new RuntimeException("Failure closing coordinator.", e);
- }
- bit = null;
- client = null;
- coordinator = null;
+ public void onStatementClose(AvaticaStatement statement) throws RuntimeException {
+ ((DrillRemoteStatement) statement).cleanup();
}
-
}
-/*
- *
- * optiq work ==========
- *
- * 1. todo: test can cast(<any> as varchar) (or indeed to any type)
- *
- * 2. We declare a variant record by adding a '_MAP any' field. It's nice that there can be some declared fields, and
- * some undeclared. todo: Better syntactic sugar.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillJdbc40Factory.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillJdbc40Factory.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillJdbc40Factory.java
new file mode 100644
index 0000000..b542eb2
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillJdbc40Factory.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+/**
+ * Implementation of {@link net.hydromatic.avatica.AvaticaFactory}
+ * for Drill and JDBC 4.0 (corresponds to JDK 1.6).
+ */
+public class DrillJdbc40Factory extends DrillJdbc41Factory {
+ /** Creates a factory for JDBC version 4.1. */
+ public DrillJdbc40Factory() {
+ super(4, 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillJdbc41Factory.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillJdbc41Factory.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillJdbc41Factory.java
new file mode 100644
index 0000000..e6e533e
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillJdbc41Factory.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.jdbc;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.sql.NClob;
+import java.sql.ResultSetMetaData;
+import java.sql.RowId;
+import java.sql.SQLException;
+import java.sql.SQLXML;
+import java.util.List;
+import java.util.Properties;
+import java.util.TimeZone;
+
+import net.hydromatic.avatica.AvaticaConnection;
+import net.hydromatic.avatica.AvaticaDatabaseMetaData;
+import net.hydromatic.avatica.AvaticaPrepareResult;
+import net.hydromatic.avatica.AvaticaPreparedStatement;
+import net.hydromatic.avatica.AvaticaResultSetMetaData;
+import net.hydromatic.avatica.AvaticaStatement;
+import net.hydromatic.avatica.ColumnMetaData;
+
+/**
+ * Implementation of {@link net.hydromatic.avatica.AvaticaFactory} for Drill and JDBC 4.1 (corresponds to JDK 1.7).
+ */
+@SuppressWarnings("UnusedDeclaration")
+public class DrillJdbc41Factory extends DrillFactory {
+ /** Creates a factory for JDBC version 4.1. */
+ public DrillJdbc41Factory() {
+ this(4, 1);
+ }
+
+ /** Creates a JDBC factory with given major/minor version number. */
+ protected DrillJdbc41Factory(int major, int minor) {
+ super(major, minor);
+ }
+
+ public DrillJdbc41Connection newDrillConnection(Driver driver, DrillFactory factory, String url, Properties info) throws SQLException{
+ return new DrillJdbc41Connection((Driver) driver, factory, url, info);
+ }
+
+ public DrillJdbc41DatabaseMetaData newDatabaseMetaData(AvaticaConnection connection) {
+ return new DrillJdbc41DatabaseMetaData((DrillConnectionImpl) connection);
+ }
+
+ public DrillJdbc41Statement newStatement(AvaticaConnection connection, int resultSetType, int resultSetConcurrency,
+ int resultSetHoldability) {
+ return new DrillJdbc41Statement((DrillConnectionImpl) connection, resultSetType, resultSetConcurrency,
+ resultSetHoldability);
+ }
+
+ public AvaticaPreparedStatement newPreparedStatement(AvaticaConnection connection,
+ AvaticaPrepareResult prepareResult, int resultSetType, int resultSetConcurrency, int resultSetHoldability)
+ throws SQLException {
+ return new DrillJdbc41PreparedStatement((DrillConnectionImpl) connection, (DrillPrepareResult) prepareResult,
+ resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ public DrillResultSet newResultSet(AvaticaStatement statement, AvaticaPrepareResult prepareResult, TimeZone timeZone) {
+ final ResultSetMetaData metaData = newResultSetMetaData(statement, prepareResult.getColumnList());
+ return new DrillResultSet(statement, (DrillPrepareResult) prepareResult, metaData, timeZone);
+ }
+
+ public ResultSetMetaData newResultSetMetaData(AvaticaStatement statement, List<ColumnMetaData> columnMetaDataList) {
+ return new AvaticaResultSetMetaData(statement, null, columnMetaDataList);
+ }
+
+ private static class DrillJdbc41Connection extends DrillConnectionImpl {
+ DrillJdbc41Connection(Driver driver, DrillFactory factory, String url, Properties info) throws SQLException {
+ super(driver, factory, url, info);
+ }
+
+ }
+
+ private static class DrillJdbc41Statement extends DrillStatement {
+ public DrillJdbc41Statement(DrillConnectionImpl connection, int resultSetType, int resultSetConcurrency,
+ int resultSetHoldability) {
+ super(connection, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+ }
+
+ private static class DrillJdbc41PreparedStatement extends DrillPreparedStatement {
+ DrillJdbc41PreparedStatement(DrillConnectionImpl connection, DrillPrepareResult prepareResult, int resultSetType,
+ int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+ super(connection, prepareResult, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ public void setRowId(int parameterIndex, RowId x) throws SQLException {
+ getParameter(parameterIndex).setRowId(x);
+ }
+
+ public void setNString(int parameterIndex, String value) throws SQLException {
+ getParameter(parameterIndex).setNString(value);
+ }
+
+ public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException {
+ getParameter(parameterIndex).setNCharacterStream(value, length);
+ }
+
+ public void setNClob(int parameterIndex, NClob value) throws SQLException {
+ getParameter(parameterIndex).setNClob(value);
+ }
+
+ public void setClob(int parameterIndex, Reader reader, long length) throws SQLException {
+ getParameter(parameterIndex).setClob(reader, length);
+ }
+
+ public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException {
+ getParameter(parameterIndex).setBlob(inputStream, length);
+ }
+
+ public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException {
+ getParameter(parameterIndex).setNClob(reader, length);
+ }
+
+ public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException {
+ getParameter(parameterIndex).setSQLXML(xmlObject);
+ }
+
+ public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException {
+ getParameter(parameterIndex).setAsciiStream(x, length);
+ }
+
+ public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException {
+ getParameter(parameterIndex).setBinaryStream(x, length);
+ }
+
+ public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException {
+ getParameter(parameterIndex).setCharacterStream(reader, length);
+ }
+
+ public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException {
+ getParameter(parameterIndex).setAsciiStream(x);
+ }
+
+ public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException {
+ getParameter(parameterIndex).setBinaryStream(x);
+ }
+
+ public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException {
+ getParameter(parameterIndex).setCharacterStream(reader);
+ }
+
+ public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException {
+ getParameter(parameterIndex).setNCharacterStream(value);
+ }
+
+ public void setClob(int parameterIndex, Reader reader) throws SQLException {
+ getParameter(parameterIndex).setClob(reader);
+ }
+
+ public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException {
+ getParameter(parameterIndex).setBlob(inputStream);
+ }
+
+ public void setNClob(int parameterIndex, Reader reader) throws SQLException {
+ getParameter(parameterIndex).setNClob(reader);
+ }
+ }
+
+ private static class DrillJdbc41DatabaseMetaData extends AvaticaDatabaseMetaData {
+ DrillJdbc41DatabaseMetaData(DrillConnectionImpl connection) {
+ super(connection);
+ }
+ }
+
+}
+
+// End DrillJdbc41Factory.java
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillPrepareResult.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillPrepareResult.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillPrepareResult.java
new file mode 100644
index 0000000..836f346
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillPrepareResult.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.util.Collections;
+import java.util.List;
+
+import net.hydromatic.avatica.AvaticaParameter;
+import net.hydromatic.avatica.AvaticaPrepareResult;
+import net.hydromatic.avatica.ColumnMetaData;
+
+public class DrillPrepareResult implements AvaticaPrepareResult{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillPrepareResult.class);
+
+ final String sql;
+ final DrillColumnMetaDataList columns = new DrillColumnMetaDataList();
+
+ public DrillPrepareResult(String sql) {
+ super();
+ this.sql = sql;
+ }
+
+ @Override
+ public List<ColumnMetaData> getColumnList() {
+ return columns;
+ }
+
+ @Override
+ public String getSql() {
+ return sql;
+ }
+
+ @Override
+ public List<AvaticaParameter> getParameterList() {
+ return Collections.emptyList();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillPreparedStatement.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillPreparedStatement.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillPreparedStatement.java
new file mode 100644
index 0000000..6d4ee95
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillPreparedStatement.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.sql.SQLException;
+
+import net.hydromatic.avatica.AvaticaPrepareResult;
+import net.hydromatic.avatica.AvaticaPreparedStatement;
+
+/**
+ * Implementation of {@link java.sql.PreparedStatement} for Drill.
+ *
+ * <p>
+ * This class has sub-classes which implement JDBC 3.0 and JDBC 4.0 APIs; it is instantiated using
+ * {@link net.hydromatic.avatica.AvaticaFactory#newPreparedStatement}.
+ * </p>
+ */
+abstract class DrillPreparedStatement extends AvaticaPreparedStatement implements DrillRemoteStatement {
+
+ protected DrillPreparedStatement(DrillConnectionImpl connection, AvaticaPrepareResult prepareResult,
+ int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+ super(connection, prepareResult, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ @Override
+ public DrillConnectionImpl getConnection() {
+ return (DrillConnectionImpl) super.getConnection();
+ }
+
+ @Override
+ public void cleanup() {
+ final DrillConnectionImpl connection1 = (DrillConnectionImpl) connection;
+ connection1.registry.removeStatement(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillRemoteStatement.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillRemoteStatement.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillRemoteStatement.java
new file mode 100644
index 0000000..e19b49d
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillRemoteStatement.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+public interface DrillRemoteStatement {
+ public void cleanup();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillResultSet.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillResultSet.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillResultSet.java
new file mode 100644
index 0000000..be56b96
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillResultSet.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.TimeZone;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import net.hydromatic.avatica.AvaticaPrepareResult;
+import net.hydromatic.avatica.AvaticaResultSet;
+import net.hydromatic.avatica.AvaticaStatement;
+
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserProtos.QueryType;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+
+import com.google.common.collect.Queues;
+
+public class DrillResultSet extends AvaticaResultSet {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillResultSet.class);
+
+ SchemaChangeListener changeListener;
+ final Listener listener = new Listener();
+ private volatile QueryId queryId;
+ private final DrillClient client;
+ final RecordBatchLoader currentBatch;
+ final DrillCursor cursor;
+
+ public DrillResultSet(AvaticaStatement statement, AvaticaPrepareResult prepareResult,
+ ResultSetMetaData resultSetMetaData, TimeZone timeZone) {
+ super(statement, prepareResult, resultSetMetaData, timeZone);
+ DrillConnection c = (DrillConnection) statement.getConnection();
+ DrillClient client = c.getClient();
+ // DrillClient client, DrillStatement statement) {
+ currentBatch = new RecordBatchLoader(client.getAllocator());
+ this.client = client;
+ cursor = new DrillCursor(this);
+ }
+
+ @Override
+ protected void cancel() {
+ cleanup();
+ close();
+ }
+
+ synchronized void cleanup(){
+ if (queryId != null && !listener.completed){
+ client.cancelQuery(queryId);
+ }
+ listener.close();
+ }
+
+ @Override protected DrillResultSet execute() throws SQLException{
+ // Call driver's callback. It is permitted to throw a RuntimeException.
+ DrillConnectionImpl connection = (DrillConnectionImpl) statement.getConnection();
+
+ connection.getClient().runQuery(QueryType.SQL, this.prepareResult.getSql(), listener);
+ connection.getDriver().handler.onStatementExecute(statement, null);
+
+ super.execute();
+
+ // don't return with metadata until we've achieved at least one return message.
+ try {
+ listener.latch.await();
+ cursor.next();
+ } catch (InterruptedException e) {
+ }
+
+ return this;
+ }
+
+ class Listener implements UserResultsListener {
+ private static final int MAX = 100;
+ private volatile RpcException ex;
+ volatile boolean completed = false;
+ private volatile boolean autoread = true;
+ private volatile ConnectionThrottle throttle;
+ private volatile boolean closed = false;
+ private CountDownLatch latch = new CountDownLatch(1);
+ private AtomicBoolean receivedMessage = new AtomicBoolean(false);
+
+
+
+ final LinkedBlockingDeque<QueryResultBatch> queue = Queues.newLinkedBlockingDeque();
+
+ private boolean releaseIfFirst(){
+ if(receivedMessage.compareAndSet(false, true)){
+ latch.countDown();
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public void submissionFailed(RpcException ex) {
+ releaseIfFirst();
+ this.ex = ex;
+ completed = true;
+ close();
+ System.out.println("Query failed: " + ex);
+ }
+
+ @Override
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
+ logger.debug("Result arrived {}", result);
+
+ // if we're in a closed state, just release the message.
+ if (closed) {
+ result.release();
+ completed = true;
+ return;
+ }
+
+ // we're active, let's add to the queue.
+ queue.add(result);
+ if (queue.size() >= MAX - 1) {
+ throttle.setAutoRead(false);
+ this.throttle = throttle;
+ autoread = false;
+ }
+
+ if (result.getHeader().getIsLastChunk()) {
+ completed = true;
+ }
+
+ if (result.getHeader().getErrorCount() > 0) {
+ submissionFailed(new RpcException(String.format("%s", result.getHeader().getErrorList())));
+ }
+
+ releaseIfFirst();
+
+ }
+
+ public QueryResultBatch getNext() throws RpcException, InterruptedException {
+ while (true) {
+ if (ex != null)
+ throw ex;
+ if (completed && queue.isEmpty()) {
+ return null;
+ } else {
+ QueryResultBatch q = queue.poll(50, TimeUnit.MILLISECONDS);
+ if (q != null) {
+ if (!autoread && queue.size() < MAX / 2) {
+ autoread = true;
+ throttle.setAutoRead(true);
+ throttle = null;
+ }
+ return q;
+ }
+
+ }
+
+ }
+ }
+
+ void close() {
+ closed = true;
+ while (!queue.isEmpty()) {
+ QueryResultBatch qrb = queue.poll();
+ if(qrb != null && qrb.getData() != null) qrb.getData().release();
+ }
+ completed = true;
+ }
+
+ @Override
+ public void queryIdArrived(QueryId queryId) {
+ DrillResultSet.this.queryId = queryId;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillStatement.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillStatement.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillStatement.java
new file mode 100644
index 0000000..fec126e
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillStatement.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import net.hydromatic.avatica.AvaticaStatement;
+
+public abstract class DrillStatement extends AvaticaStatement implements DrillRemoteStatement {
+
+ DrillStatement(DrillConnectionImpl connection, int resultSetType, int resultSetConcurrency, int resultSetHoldability) {
+ super(connection, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ @Override
+ public DrillConnectionImpl getConnection() {
+ return (DrillConnectionImpl) connection;
+ }
+
+ @Override
+ public void cleanup() {
+ final DrillConnectionImpl connection1 = (DrillConnectionImpl) connection;
+ connection1.registry.removeStatement(this);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillStatementRegistry.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillStatementRegistry.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillStatementRegistry.java
new file mode 100644
index 0000000..480caca
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillStatementRegistry.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+class DrillStatementRegistry {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillStatementRegistry.class);
+
+
+ public void addStatement(DrillRemoteStatement statement){}
+ public void removeStatement(DrillRemoteStatement statement){}
+
+ public void close(){}
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
deleted file mode 100644
index f51021e..0000000
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.jdbc;
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import net.hydromatic.linq4j.BaseQueryable;
-import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.linq4j.Linq4j;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.linq4j.expressions.Expressions;
-import net.hydromatic.linq4j.expressions.MethodCallExpression;
-import net.hydromatic.optiq.BuiltinMethod;
-import net.hydromatic.optiq.DataContext;
-import net.hydromatic.optiq.MutableSchema;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.Statistic;
-import net.hydromatic.optiq.Statistics;
-import net.hydromatic.optiq.TableFactory;
-import net.hydromatic.optiq.TranslatableTable;
-
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.ref.rops.DataWriter;
-import org.apache.drill.exec.ref.rse.ClasspathRSE;
-import org.apache.drill.exec.ref.rse.ClasspathRSE.ClasspathInputConfig;
-import org.apache.drill.optiq.DrillRel;
-import org.apache.drill.optiq.DrillScan;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptTable;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeFactory;
-import org.eigenbase.sql.type.SqlTypeName;
-
-/** Optiq Table used by Drill. */
-public class DrillTable extends BaseQueryable<Object> implements TranslatableTable<Object>{
-
- private final Schema schema;
- private final String name;
- private final String storageEngineName;
- private final RelDataType rowType;
- public final StorageEngineConfig storageEngineConfig;
- private Object selection;
- public DrillClient client;
-
-
- public boolean isRefEngine(){
- return client == null;
- }
-
- /** Creates a DrillTable. */
- public DrillTable(DrillClient client, Schema schema,
- Type elementType,
- Expression expression,
- RelDataType rowType,
- String name,
- String storageEngineName,
- Object selection,
- StorageEngineConfig storageEngineConfig
- ) {
- super(schema.getQueryProvider(), elementType, expression);
- this.client = client;
- this.schema = schema;
- this.name = name;
- this.rowType = rowType;
- this.selection = selection;
- this.storageEngineConfig = storageEngineConfig;
- this.storageEngineName = client == null ? storageEngineName : schema.getName();
- }
-
- public String getName() {
- return name;
- }
-
- public StorageEngineConfig getStorageEngineConfig(){
- return storageEngineConfig;
- }
-
- public Object getSelection() {
- return selection;
- }
-
- public static DrillTable createTable(
- DrillClient client,
- RelDataTypeFactory typeFactory,
- Schema schema,
- String name,
- String storageEngineName,
- StorageEngineConfig storageEngineConfig,
- Object selection
- ) {
- final MethodCallExpression call = Expressions.call(schema.getExpression(), //
- BuiltinMethod.DATA_CONTEXT_GET_TABLE.method, //
- Expressions.constant(name), //
- Expressions.constant(Object.class));
-
- final RelDataType rowType =
- typeFactory.createStructType(
- Collections.singletonList(
- typeFactory.createMapType(
- typeFactory.createSqlType(SqlTypeName.VARCHAR),
- typeFactory.createSqlType(SqlTypeName.ANY))),
- Collections.singletonList("_MAP"));
- return new DrillTable(client, schema, Object.class, call, rowType, name, storageEngineName, selection, storageEngineConfig);
- }
-
-
-
- @Override
- public DataContext getDataContext() {
- return schema;
- }
-
-
- public String getStorageEngineName() {
- return storageEngineName;
- }
-
- public boolean useReferenceInterpreter() {
- return this.client == null;
- }
-
- @Override
- public RelDataType getRowType() {
- return rowType;
- }
-
- @Override
- public Statistic getStatistic() {
- return Statistics.UNKNOWN;
- }
-
- @Override
- public Enumerator<Object> enumerator() {
- return Linq4j.emptyEnumerator();
- }
-
- public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable table) {
- return new DrillScan(context.getCluster(),
- context.getCluster().traitSetOf(DrillRel.CONVENTION),
- table);
- }
-
-
-
- /** Factory for custom tables in Optiq schema. */
- @SuppressWarnings("UnusedDeclaration")
- public static class Factory implements TableFactory<DrillTable> {
-
- private static final List<String> DONUTS_TABLES = Arrays.asList(
- "DONUTS");
-
- private static final List<String> HR_TABLES = Arrays.asList(
- "EMPLOYEES", "DEPARTMENTS");
-
- private static final List<String> FOODMART_TABLES = Arrays.asList(
- "ACCOUNT", "CATEGORY", "CURRENCY", "CUSTOMER", "DAYS", "DEPARTMENT",
- "EMPLOYEE_CLOSURE", "EMPLOYEE", "EXPENSE_FACT", "INVENTORY_FACT_1997",
- "INVENTORY_FACT_1998", "POSITION", "PRODUCT_CLASS", "PRODUCT",
- "PROMOTION", "REGION", "RESERVE_EMPLOYEE", "SALARY", "SALES_FACT_1997",
- "SALES_FACT_1998", "SALES_FACT_DEC_1998", "STORE", "STORE_RAGGED",
- "TIME_BY_DAY", "WAREHOUSE", "WAREHOUSE_CLASS");
-
-// public DrillTable create(
-// JavaTypeFactory typeFactory,
-// Schema schema,
-// String name,
-// DistributedMap<String, Object> operand,
-// RelDataType rowType) {
-// final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
-// final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
-// assert DONUTS_TABLES.contains(name)
-// || HR_TABLES.contains(name)
-// || FOODMART_TABLES.contains(name)
-// : name;
-// inputConfig.path = "/" + name.toLowerCase() + ".json";
-// inputConfig.type = DataWriter.ConverterType.JSON;
-// boolean useReferenceInterpreter;
-// if (operand.get("useReferenceInterpreter") != null){
-// useReferenceInterpreter = operand.get("useReferenceInterpreter").equals("true") ? true : false;
-// }
-// else{
-// useReferenceInterpreter = false;
-// }
-// return createTable(typeFactory, (MutableSchema) schema, name, rseConfig,
-// inputConfig, "donuts-json", useReferenceInterpreter);
-// }
-//
- @Override
- public DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) {
-
- final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
- final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
- assert DONUTS_TABLES.contains(name)
- || HR_TABLES.contains(name)
- || FOODMART_TABLES.contains(name)
- : name;
- inputConfig.path = "/" + name.toLowerCase() + ".json";
- inputConfig.type = DataWriter.ConverterType.JSON;
- return createTable(null, schema.getTypeFactory(), (MutableSchema) schema, name, "donuts-json", rseConfig, inputConfig);
- }
- }
-
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java b/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
index d7878c9..9ac913b 100644
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
@@ -17,55 +17,71 @@
*/
package org.apache.drill.jdbc;
-import net.hydromatic.linq4j.function.Function0;
-import net.hydromatic.optiq.jdbc.DriverVersion;
-import net.hydromatic.optiq.jdbc.Handler;
-import net.hydromatic.optiq.jdbc.OptiqPrepare;
-import net.hydromatic.optiq.jdbc.UnregisteredDriver;
+import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.optiq.DrillPrepareImpl;
+import net.hydromatic.avatica.DriverVersion;
+import net.hydromatic.avatica.Handler;
+import net.hydromatic.avatica.HandlerImpl;
+import net.hydromatic.avatica.UnregisteredDriver;
/**
- * JDBC driver for Apache Drill.
+ * Optiq JDBC driver.
*/
public class Driver extends UnregisteredDriver {
public static final String CONNECT_STRING_PREFIX = "jdbc:drill:";
- private volatile DrillHandler handler;
+
+
+ final DrillConfig config;
- static {
- new Driver().register();
+ public Driver() {
+ super();
+ config = DrillConfig.create();
}
+
+ public static boolean load(){
+ return true;
+ }
+
+ @Override
protected String getConnectStringPrefix() {
return CONNECT_STRING_PREFIX;
}
- protected DriverVersion createDriverVersion() {
- return new DrillDriverVersion();
+ @Override
+ protected String getFactoryClassName(JdbcVersion jdbcVersion) {
+ switch (jdbcVersion) {
+ case JDBC_30:
+ return "org.apache.drill.jdbc.DrillJdbc3Factory";
+ case JDBC_40:
+ return "org.apache.drill.jdbc.DrillJdbc40Factory";
+ case JDBC_41:
+ default:
+ return "org.apache.drill.jdbc.DrillJdbc41Factory";
+ }
}
- @Override
- protected Function0<OptiqPrepare> createPrepareFactory() {
- return new Function0<OptiqPrepare>() {
- @Override
- public OptiqPrepare apply() {
- return new DrillPrepareImpl(Driver.this);
- }
- };
+ protected DriverVersion createDriverVersion() {
+ return DriverVersion.load(
+ Driver.class,
+ "apache-drill-jdbc.properties",
+ "Drill JDBC Driver",
+ "unknown version",
+ "Optiq",
+ "unknown version");
}
- public DrillClient getClient(){
- return handler.getClient();
+ DrillConfig getConfig(){
+ return config;
}
@Override
protected Handler createHandler() {
- this.handler = new DrillHandler(false);
- return handler;
+ return new HandlerImpl();
}
-
-}
-// End Driver.java
+ static {
+ new Driver().register();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/GlobalServiceSetReference.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/GlobalServiceSetReference.java b/sqlparser/src/main/java/org/apache/drill/jdbc/GlobalServiceSetReference.java
new file mode 100644
index 0000000..9326283
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/GlobalServiceSetReference.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import org.apache.drill.exec.server.RemoteServiceSet;
+
+public class GlobalServiceSetReference {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GlobalServiceSetReference.class);
+
+ public static final ThreadLocal<RemoteServiceSet> SETS = new ThreadLocal<RemoteServiceSet>();
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/MetaImpl.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/MetaImpl.java b/sqlparser/src/main/java/org/apache/drill/jdbc/MetaImpl.java
new file mode 100644
index 0000000..5d8b6a7
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/MetaImpl.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.sql.ResultSet;
+import java.util.List;
+
+import net.hydromatic.avatica.AvaticaPrepareResult;
+import net.hydromatic.avatica.AvaticaResultSet;
+import net.hydromatic.avatica.AvaticaStatement;
+import net.hydromatic.avatica.Cursor;
+import net.hydromatic.avatica.Meta;
+import net.hydromatic.linq4j.Linq4j;
+
+public class MetaImpl implements Meta {
+
+ static final Driver DRIVER = new Driver();
+
+ final DrillConnectionImpl connection;
+
+ public MetaImpl(DrillConnectionImpl connection) {
+ this.connection = connection;
+ }
+
+ public String getSqlKeywords() {
+ return "";
+ }
+
+ public String getNumericFunctions() {
+ return "";
+ }
+
+ public String getStringFunctions() {
+ return "";
+ }
+
+ public String getSystemFunctions() {
+ return "";
+ }
+
+ public String getTimeDateFunctions() {
+ return "";
+ }
+
+ public static ResultSet getEmptyResultSet() {
+ return null;
+ }
+
+ public ResultSet getTables(String catalog, final Pat schemaPattern, final Pat tableNamePattern,
+ final List<String> typeList) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getSchemas(String catalog, Pat schemaPattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getCatalogs() {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getTableTypes() {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getProcedures(String catalog, Pat schemaPattern, Pat procedureNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getProcedureColumns(String catalog, Pat schemaPattern, Pat procedureNamePattern,
+ Pat columnNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getColumnPrivileges(String catalog, String schema, String table, Pat columnNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getTablePrivileges(String catalog, Pat schemaPattern, Pat tableNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getBestRowIdentifier(String catalog, String schema, String table, int scope, boolean nullable) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getVersionColumns(String catalog, String schema, String table) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getPrimaryKeys(String catalog, String schema, String table) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getImportedKeys(String catalog, String schema, String table) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getExportedKeys(String catalog, String schema, String table) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getCrossReference(String parentCatalog, String parentSchema, String parentTable,
+ String foreignCatalog, String foreignSchema, String foreignTable) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getTypeInfo() {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getUDTs(String catalog, Pat schemaPattern, Pat typeNamePattern, int[] types) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getSuperTypes(String catalog, Pat schemaPattern, Pat typeNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getSuperTables(String catalog, Pat schemaPattern, Pat tableNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getAttributes(String catalog, Pat schemaPattern, Pat typeNamePattern, Pat attributeNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getClientInfoProperties() {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getFunctions(String catalog, Pat schemaPattern, Pat functionNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getFunctionColumns(String catalog, Pat schemaPattern, Pat functionNamePattern, Pat columnNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public ResultSet getPseudoColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
+ return getEmptyResultSet();
+ }
+
+ public Cursor createCursor(AvaticaResultSet resultSet_) {
+ return ((DrillResultSet) resultSet_).cursor;
+ }
+
+ public AvaticaPrepareResult prepare(AvaticaStatement statement_, String sql) {
+ //DrillStatement statement = (DrillStatement) statement_;
+ return new DrillPrepareResult(sql);
+ }
+
+ interface Named {
+ String getName();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/RefDriver.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/RefDriver.java b/sqlparser/src/main/java/org/apache/drill/jdbc/RefDriver.java
deleted file mode 100644
index 85c7f5d..0000000
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/RefDriver.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.jdbc;
-
-import net.hydromatic.linq4j.function.Function0;
-import net.hydromatic.optiq.jdbc.DriverVersion;
-import net.hydromatic.optiq.jdbc.Handler;
-import net.hydromatic.optiq.jdbc.OptiqPrepare;
-import net.hydromatic.optiq.jdbc.UnregisteredDriver;
-
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.optiq.DrillPrepareImpl;
-
-/**
- * JDBC driver for Apache Drill.
- */
-public class RefDriver extends UnregisteredDriver {
- public static final String CONNECT_STRING_PREFIX = "jdbc:drillref:";
-
- private volatile DrillHandler handler;
-
- static {
- new RefDriver().register();
- }
-
- protected String getConnectStringPrefix() {
- return CONNECT_STRING_PREFIX;
- }
-
- protected DriverVersion createDriverVersion() {
- return new DrillDriverVersion();
- }
-
- @Override
- protected Function0<OptiqPrepare> createPrepareFactory() {
- return new Function0<OptiqPrepare>() {
- @Override
- public OptiqPrepare apply() {
- return new DrillPrepareImpl(null);
- }
- };
- }
-
- public DrillClient getClient(){
- return handler.getClient();
- }
-
- @Override
- protected Handler createHandler() {
- this.handler = new DrillHandler(true);
- return handler;
- }
-
-}
-
-// End Driver.java
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/SchemaChangeListener.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/SchemaChangeListener.java b/sqlparser/src/main/java/org/apache/drill/jdbc/SchemaChangeListener.java
new file mode 100644
index 0000000..38a39da
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/SchemaChangeListener.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import org.apache.drill.exec.record.BatchSchema;
+
+public interface SchemaChangeListener {
+ public void schemaChanged(BatchSchema newSchema);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/SqlTimeoutException.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/SqlTimeoutException.java b/sqlparser/src/main/java/org/apache/drill/jdbc/SqlTimeoutException.java
new file mode 100644
index 0000000..c24858e
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/SqlTimeoutException.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.sql.SQLException;
+
+/**
+ * Indicates that an operation timed out. This is not an error; you can
+ * retry the operation.
+ */
+public class SqlTimeoutException
+ extends SQLException
+{
+ SqlTimeoutException() {
+ // SQLException(reason, SQLState, vendorCode)
+ // REVIEW mb 19-Jul-05 Is there a standard SQLState?
+ super("timeout", null, 0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/StorageEngines.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/StorageEngines.java b/sqlparser/src/main/java/org/apache/drill/jdbc/StorageEngines.java
deleted file mode 100644
index d037508..0000000
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/StorageEngines.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.jdbc;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Charsets;
-import com.google.common.io.Resources;
-
-public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineConfig>>{
-
- private Map<String, StorageEngineConfig> storage;
-
- @JsonCreator
- public StorageEngines(@JsonProperty("storage") Map<String, StorageEngineConfig> storage){
- this.storage = storage;
- }
-
- public static void main(String[] args) throws Exception{
- DrillConfig config = DrillConfig.create();
- String data = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
- StorageEngines se = config.getMapper().readValue(data, StorageEngines.class);
- System.out.println(se);
- }
-
- @Override
- public String toString() {
- final int maxLen = 10;
- return "StorageEngines [storage=" + (storage != null ? toString(storage.entrySet(), maxLen) : null) + "]";
- }
-
- @Override
- public Iterator<Entry<String, StorageEngineConfig>> iterator() {
- return storage.entrySet().iterator();
- }
-
- private String toString(Collection<?> collection, int maxLen) {
- StringBuilder builder = new StringBuilder();
- builder.append("[");
- int i = 0;
- for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
- if (i > 0)
- builder.append(", ");
- builder.append(iterator.next());
- }
- builder.append("]");
- return builder.toString();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/package-info.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/package-info.java b/sqlparser/src/main/java/org/apache/drill/jdbc/package-info.java
new file mode 100644
index 0000000..f465718
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * JDBC driver for Drill.
+ */
+package org.apache.drill.jdbc;
[10/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/LimitROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/LimitROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/LimitROP.java
deleted file mode 100644
index d26bcd9..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/LimitROP.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.logical.data.Limit;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-
-public class LimitROP extends SingleInputROPBase<Limit>{
-
- private LimitIterator iter;
- private Integer first;
- private Integer last;
-
- public LimitROP(Limit config) {
- super(config);
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) {
- first = config.getFirst();
- last = config.getLast();
- }
-
- @Override
- public void setInput(RecordIterator incoming) {
- iter = new LimitIterator(incoming);
- }
-
- @Override
- public RecordIterator getIteratorInternal() {
- return iter;
- }
-
- private class LimitIterator implements RecordIterator{
- RecordIterator incoming;
- int currentIndex = 0;
-
- public LimitIterator(RecordIterator incoming) {
- this.incoming = incoming;
- }
-
- @Override
- public NextOutcome next() {
- NextOutcome r;
- while(true){
- r = incoming.next();
- currentIndex++;
- if (currentIndex > first && (last == null || currentIndex <= last))
- return r;
-
- if (last != null && currentIndex > last)
- return NextOutcome.NONE_LEFT;
- }
- }
-
- @Override
- public ROP getParent() {
- return LimitROP.this;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return record;
- }
-
-
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/OrderROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/OrderROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/OrderROP.java
deleted file mode 100644
index 9f25641..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/OrderROP.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.logical.data.Order;
-import org.apache.drill.common.logical.data.Order.Direction;
-import org.apache.drill.common.logical.data.Order.NullCollation;
-import org.apache.drill.common.logical.data.Order.Ordering;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.ComparableValue;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues;
-import org.apache.hadoop.util.IndexedSortable;
-import org.apache.hadoop.util.QuickSort;
-
-public class OrderROP extends AbstractBlockingOperator<Order> {
-
- private List<CompoundValue> records = new ArrayList<CompoundValue>();
- private SortDefinition[] defs;
- private int withinExtra = 0;
- private boolean withinConstrained;
- private BasicEvaluator within;
- private DataValue previous;
- private long withinMarkerValue;
-
- public OrderROP(Order config) {
- super(config);
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) {
- Ordering[] orderings = config.getOrderings();
- withinConstrained = config.getWithin() != null;
- if (withinConstrained) {
- withinExtra = 1;
- within = builder.getBasicEvaluator(record, config.getWithin());
- }
-
- defs = new SortDefinition[orderings.length];
-
- for (int i = 0; i < orderings.length; i++) {
- defs[i] = new SortDefinition(builder.getBasicEvaluator(inputRecord, orderings[i].getExpr()),
- orderings[i].getDirection() == Direction.ASC, orderings[i].getNullCollation() == NullCollation.NULLS_LAST);
- }
- }
-
- @Override
- protected void consumeRecord() {
- DataValue[] values = new DataValue[defs.length + withinExtra];
-
- RecordPointer r = inputRecord.copy();
-
- /**
- * Rather than use a sort on the within value, we need to make sure that this operator correctly implements the
- * reference implementation. Ordering only operates with sequential segment key values. If a segment key value
- * repeats elsewhere in the stream, this operator should bring those together. together.
- **/
- if (withinConstrained) {
- DataValue current = within.eval();
- if (!current.equals(previous)) {
- withinMarkerValue++;
- }
- values[0] = new ScalarValues.LongScalar(withinMarkerValue);
- }
-
- for (int i = 0; i < defs.length; i++) {
- values[i + withinExtra] = defs[i].evaluator.eval();
- }
- CompoundValue v = new CompoundValue(r, values);
- records.add(v);
- }
-
- @Override
- protected RecordIterator doWork() {
- StackedComparator r = new StackedComparator(defs);
- QuickSort qs = new QuickSort();
- qs.sort(r, 0, records.size());
- return new OrderIterator();
- }
-
- public class SortDefinition {
- boolean forward;
- boolean nullsLast;
- BasicEvaluator evaluator;
-
- public SortDefinition(BasicEvaluator evaluator, boolean forward, boolean nullsLast) {
- this.evaluator = evaluator;
- this.forward = forward;
- this.nullsLast = nullsLast;
- }
- }
-
- private class CompoundValue {
- DataValue[] values;
- RecordPointer record;
-
- public CompoundValue(RecordPointer record, DataValue[] values) {
- super();
- this.record = record;
- this.values = values;
- }
-
- }
-
- private class StackedComparator implements IndexedSortable {
-
- public StackedComparator(SortDefinition[] defs) {
- }
-
- @Override
- public void swap(int index0, int index1) {
- CompoundValue v = records.get(index0);
- records.set(index0, records.get(index1));
- records.set(index1, v);
- }
-
- @Override
- public int compare(int index0, int index1) {
- int result = 0;
- CompoundValue v1 = records.get(index0);
- CompoundValue v2 = records.get(index1);
-
- for (int i = 0; i < defs.length; i++) {
- boolean nullLast = defs[i].nullsLast;
- boolean asc = defs[i].forward;
- DataValue dv1 = v1.values[i];
- DataValue dv2 = v2.values[i];
- if (dv1 == DataValue.NULL_VALUE) {
- if (dv2 == DataValue.NULL_VALUE) {
- result = 0;
- } else {
- result = nullLast ? 1 : -1;
- }
- } else if (dv2 == DataValue.NULL_VALUE) {
- result = nullLast ? -1 : 1;
- } else {
- if (dv1 instanceof ComparableValue && ((ComparableValue) dv1).supportsCompare(dv2)) {
- result = ((ComparableValue) dv1).compareTo(dv2);
- if (!asc) result = -result;
- } else {
- return 0; // we break even though there may be more evaluators because we should always return the same
- // ordering for non-comparable values no matter the compare order.
- }
- }
- if (result != 0) return result;
- }
- return result;
- }
-
- }
-
- public class OrderIterator implements RecordIterator {
- final Iterator<CompoundValue> iter;
-
- public OrderIterator() {
- this.iter = records.iterator();
- }
-
- @Override
- public NextOutcome next() {
- if (iter.hasNext()) {
- outputRecord.setRecord(iter.next().record);
- return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
- }
-
- return NextOutcome.NONE_LEFT;
- }
-
- @Override
- public ROP getParent() {
- return OrderROP.this;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return inputRecord;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ProjectROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ProjectROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ProjectROP.java
deleted file mode 100644
index 3426aa6..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ProjectROP.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.common.logical.data.Project;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-
-public class ProjectROP extends SingleInputROPBase<Project> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectROP.class);
-
- private final UnbackedRecord outputRecord = new UnbackedRecord();
- private final PathSegment[] paths;
- private final BasicEvaluator[] evaluators;
- private final NamedExpression[] selections;
- private RecordIterator incoming;
-
- public ProjectROP(Project config) {
- super(config);
- this.selections = config.getSelections();
- this.paths = new PathSegment[selections.length];
- this.evaluators = new BasicEvaluator[selections.length];
- for(int i=0; i < selections.length; i++){
- paths[i] = selections[i].getRef().getRootSegment().getChild();
- }
- }
-
- @Override
- protected void setInput(RecordIterator incoming) {
- this.incoming = incoming;
- }
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return new ProjectionIterator();
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) {
- for(int i =0; i < evaluators.length; i++){
- evaluators[i] = builder.getBasicEvaluator(record, selections[i].getExpr());
- }
- }
-
-
- private class ProjectionIterator implements RecordIterator{
-
- @Override
- public RecordPointer getRecordPointer() {
- return outputRecord;
- }
-
- @Override
- public NextOutcome next() {
- outputRecord.clear();
- NextOutcome n = incoming.next();
- if(n != NextOutcome.NONE_LEFT){
- for(int i =0; i < evaluators.length; i++){
- if(paths[i] == null){
- outputRecord.merge(evaluators[i].eval());
- }else{
- outputRecord.addField(paths[i], evaluators[i].eval());
- }
- }
- }
- return n;
- }
-
- @Override
- public ROP getParent() {
- return ProjectROP.this;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ProxySimpleRecord.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ProxySimpleRecord.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ProxySimpleRecord.java
deleted file mode 100644
index 79499b2..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ProxySimpleRecord.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.io.IOException;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.values.DataValue;
-
-public class ProxySimpleRecord implements RecordPointer{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProxySimpleRecord.class);
-
- private RecordPointer record;
-
- @Override
- public DataValue getField(SchemaPath field) {
- return record.getField(field);
- }
-
- @Override
- public void addField(SchemaPath field, DataValue value) {
- record.addField(field, value);
- }
-
-
- @Override
- public void addField(PathSegment segment, DataValue value) {
- record.addField(segment, value);
- }
-
- @Override
- public void removeField(SchemaPath field) {
- record.removeField(field);
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- record.write(writer);
- }
-
- @Override
- public void copyFrom(RecordPointer r) {
- record.copyFrom(r);
- }
-
- @Override
- public RecordPointer copy() {
- return record.copy();
- }
-
- public RecordPointer getRecord() {
- return record;
- }
-
- public void setRecord(RecordPointer record) {
- this.record = record;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ROP.java
deleted file mode 100644
index 42b532f..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ROP.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.exec.ref.*;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-
-public interface ROP {
- public void init(IteratorRegistry registry, EvaluatorFactory builder) throws SetupException;
- public RecordIterator getOutput();
- public void cleanup(RunOutcome.OutcomeType outcome);
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ROPBase.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ROPBase.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ROPBase.java
deleted file mode 100644
index 22a2bfa..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ROPBase.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.exec.ref.*;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-
-public abstract class ROPBase<T extends LogicalOperator> implements ROP{
-
- private boolean alreadyUsed = false;
- protected final T config;
-
-
- public ROPBase(T config) {
- if(config == null) throw new IllegalArgumentException("Config must be defined.");
- this.config = config;
- }
-
- protected void setupEvals(EvaluatorFactory builder) throws SetupException{};
- protected void setupIterators(IteratorRegistry registry) throws SetupException{};
- protected abstract RecordIterator getIteratorInternal();
-
- @Override
- public void init(IteratorRegistry registry, EvaluatorFactory builder) throws SetupException {
- if(config != null) registry.register(config, this);
- setupIterators(registry);
- setupEvals(builder);
- }
-
- @Override
- public final RecordIterator getOutput() {
- if(alreadyUsed) throw new IllegalStateException("You can only request the ouput of a reference operator once.");
- alreadyUsed = true;
- return getIteratorInternal();
- }
-
- @Override
- public void cleanup(RunOutcome.OutcomeType outcome) {
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ReferenceOperator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ReferenceOperator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ReferenceOperator.java
deleted file mode 100644
index 2921d9c..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ReferenceOperator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.exec.ref.RecordIterator;
-
-public interface ReferenceOperator {
-
- public void setInput(RecordIterator incoming);
- public RecordIterator getOutput();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ReferenceOperatorBase.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ReferenceOperatorBase.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ReferenceOperatorBase.java
deleted file mode 100644
index 06b3907..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ReferenceOperatorBase.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.exec.ref.RecordIterator;
-
-public abstract class ReferenceOperatorBase implements ReferenceOperator{
-
- private boolean alreadyUsed = false;
-
- @Override
- public void setInput(RecordIterator incoming) {
- }
-
- protected abstract RecordIterator getIteratorInternal();
-
- @Override
- public final RecordIterator getOutput() {
- if(alreadyUsed) throw new IllegalStateException("You can only request the ouput of a reference operator once.");
- alreadyUsed = true;
- return getIteratorInternal();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ScanROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ScanROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ScanROP.java
deleted file mode 100644
index 5c815d0..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/ScanROP.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.io.IOException;
-
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.apache.drill.exec.ref.rse.RecordReader;
-import org.apache.drill.exec.ref.rse.ReferenceStorageEngine;
-import org.apache.drill.exec.ref.rse.ReferenceStorageEngine.ReadEntry;
-
-public class ScanROP extends ROPBase<Scan>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanROP.class);
-
- private ReadEntry entry;
- private ReferenceStorageEngine engine;
- private RecordReader reader;
-
- public ScanROP(Scan config, ReadEntry entry, ReferenceStorageEngine engine) {
- super(config);
- this.entry = entry;
- this.engine = engine;
-
- }
-
-
- @Override
- protected void setupIterators(IteratorRegistry registry) throws SetupException {
- try{
- super.setupIterators(registry);
- reader = engine.getReader(entry, this);
- reader.setup();
- }catch(IOException e){
- throw new SetupException(String.format("Failure while setting up reader for Entry %s.", entry), e);
- }
- }
-
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return reader.getIterator();
- }
-
-
- @Override
- public void cleanup(OutcomeType outcome) {
- reader.cleanup();
- super.cleanup(outcome);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SegmentROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SegmentROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SegmentROP.java
deleted file mode 100644
index 0770860..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SegmentROP.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.Segment;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.DataValueSet;
-import org.apache.drill.exec.ref.values.ScalarValues;
-
-import com.google.common.collect.ArrayListMultimap;
-
-public class SegmentROP extends AbstractBlockingOperator<Segment> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SegmentROP.class);
-
- private ArrayListMultimap<DataValueSet, RecordPointer> map;
- private DataValueSet staging;
-
- FieldReference outputSegmentKey;
-
- public SegmentROP(Segment g){
- super(g);
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) {
- LogicalExpression[] groupings = config.getExprs();
- outputSegmentKey = config.getName();
- map = ArrayListMultimap.create();
-
- BasicEvaluator[] evals = new BasicEvaluator[groupings.length];
-
- for(int i = 0; i < groupings.length; i++){
- evals[i] = builder.getBasicEvaluator(record, groupings[i]);
- }
-
- staging = new DataValueSet(evals);
-
- }
-
-
- private class BatchIterator implements RecordIterator{
- private DataValueSet previousKeys;
- private final Iterator<Entry<DataValueSet, RecordPointer>> entryIterator;
- private long segmentKey;
-
- public BatchIterator(){
- entryIterator = map.entries().iterator();
- }
-
-
- @Override
- public RecordPointer getRecordPointer() {
- return outputRecord;
- }
-
- @Override
- public NextOutcome next() {
- if(!entryIterator.hasNext()) return NextOutcome.NONE_LEFT;
-
- Entry<DataValueSet, RecordPointer> e = entryIterator.next();
- RecordPointer r = e.getValue();
- if(r == null) throw new UnsupportedOperationException();
- outputRecord.setRecord(r);
-
- if(!e.getKey().equals(previousKeys)){
- previousKeys = e.getKey();
- segmentKey++;
- }
-
- r.addField(outputSegmentKey, new ScalarValues.LongScalar(segmentKey));
- return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
- }
-
- @Override
- public ROP getParent() {
- return SegmentROP.this;
- }
- }
-
-
- @Override
- protected void consumeRecord() {
- staging.grabValues();
- map.put(staging.cloneValuesOnly(), record.copy());
- }
-
-
- @Override
- protected RecordIterator doWork() {
- return new BatchIterator();
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SingleInputROPBase.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SingleInputROPBase.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SingleInputROPBase.java
deleted file mode 100644
index 541db2e..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SingleInputROPBase.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.util.List;
-
-import org.apache.drill.common.logical.data.SingleInputOperator;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-
-public abstract class SingleInputROPBase<T extends SingleInputOperator> extends ROPBase<T>{
-
- protected RecordPointer record;
-
- public SingleInputROPBase(T config) {
- super(config);
- }
-
- @Override
- protected void setupIterators(IteratorRegistry registry) {
- List<RecordIterator> iters = registry.getOperator(config.getInput());
- if(iters.size() != 1) throw new IllegalArgumentException(String.format("Expected one input iterator for class %s. Received %d", this.getClass().getCanonicalName(), iters.size()));
- RecordIterator i = iters.get(0);
- this.record = i.getRecordPointer();
- if(record == null) throw new SetupException(String.format("The %s op iterator return a null record pointer.", i.getParent()));
- setInput(i);
- }
-
- protected abstract void setInput(RecordIterator incoming);
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SinkROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SinkROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SinkROP.java
deleted file mode 100644
index 1795b95..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/SinkROP.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.exec.ref.RunOutcome;
-
-public interface SinkROP {
-
- public RunOutcome run(StatusHandle handle);
-
- public interface StatusHandle{
- public void progress(long bytes, long records);
- public boolean okToContinue();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/StoreROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/StoreROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/StoreROP.java
deleted file mode 100644
index 9c5c876..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/StoreROP.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.io.IOException;
-
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
-import org.apache.drill.exec.ref.rse.RecordRecorder;
-import org.apache.drill.exec.ref.rse.ReferenceStorageEngine;
-
-public class StoreROP extends BaseSinkROP<Store> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoreROP.class);
-
- private ReferenceStorageEngine engine;
- private RecordRecorder writer;
-
- public StoreROP(Store config, ReferenceStorageEngine engine) {
- super(config);
- this.engine = engine;
- }
-
- @Override
- public long sinkRecord(RecordPointer r) throws IOException {
- return writer.recordRecord(r);
- }
-
- @Override
- protected void setupSink() throws IOException {
- writer = engine.getWriter(config);
- writer.setup();
- }
-
- @Override
- public void cleanup(OutcomeType outcome) {
-
- try {
- writer.finish(outcome);
- } catch (IOException e) {
- logger.warn("Failure while cleaning up storage ROP", e);
- }
- super.cleanup(outcome);
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/TransformROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/TransformROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/TransformROP.java
deleted file mode 100644
index 6ed43d8..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/TransformROP.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.common.logical.data.Transform;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.ConnectedEvaluator;
-
-public class TransformROP extends SingleInputROPBase<Transform>{
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TransformROP.class);
-
- private ConnectedEvaluator[] transformers;
- private RecordIterator iter;
-
- public TransformROP(Transform transform){
- super(transform);
- }
-
- @Override
- protected void setupEvals(EvaluatorFactory builder) {
- int i =0;
- transformers = new ConnectedEvaluator[config.getTransforms().length];
- for(NamedExpression ne : config.getTransforms()){
- transformers[i] = builder.getConnectedEvaluator(record, ne);
- i++;
- }
- }
-
- @Override
- public void setInput(RecordIterator incoming) {
- this.iter = new TransformIterator(incoming);
- }
-
- private class TransformIterator implements RecordIterator{
- private final RecordIterator incoming;
-
- public TransformIterator(RecordIterator incoming){
- this.incoming = incoming;
- }
-
- @Override
- public NextOutcome next() {
- NextOutcome outcome = incoming.next();
- if(outcome == NextOutcome.NONE_LEFT){
- return outcome;
- }else{
- for(int i = 0; i < transformers.length; i++){
- transformers[i].eval();
- }
- return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
- }
- }
-
- @Override
- public ROP getParent() {
- return TransformROP.this;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return record;
- }
-
- }
-
- @Override
- public RecordIterator getIteratorInternal() {
- return iter;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/UnionROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/UnionROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/UnionROP.java
deleted file mode 100644
index 717a1dc..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/UnionROP.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.common.logical.data.Union;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordIterator.NextOutcome;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class UnionROP extends ROPBase<Union> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnionROP.class);
-
- private Collection<UnbackedRecord> records;
- private List<RecordIterator> incoming = Lists.newArrayList();
- private Iterator<UnbackedRecord> iterator;
-
- public UnionROP(Union config) {
- super(config);
- // to make things simple, we'll just always make this a blocking operator.
- if(config.isDistinct()){
- records = Sets.newHashSet();
- }else{
- records = Lists.newArrayList();
- }
- }
-
- @Override
- protected void setupIterators(IteratorRegistry registry) throws SetupException {
- for(LogicalOperator op : config.getInputs()){
- List<RecordIterator> more = registry.getOperator(op);
- if(more.size() != 1) throw new SetupException("Iterator list was incorrect size.");
- incoming.addAll(more);
- }
- }
-
-
- protected void doWork() {
- for(RecordIterator ri : incoming){
- RecordPointer rp = ri.getRecordPointer();
- while(ri.next() != NextOutcome.NONE_LEFT){
- UnbackedRecord r = new UnbackedRecord();
- r.copyFrom(rp);
- records.add(r);
- }
- }
- this.iterator = records.iterator();
- }
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return new ProxyIterator();
- }
-
- private class ProxyIterator implements RecordIterator{
- private ProxySimpleRecord proxyRecord = new ProxySimpleRecord();
-
- @Override
- public RecordPointer getRecordPointer() {
- return proxyRecord;
- }
-
- @Override
- public NextOutcome next() {
- if(iterator == null) doWork();
-
- if(iterator.hasNext()){
- proxyRecord.setRecord(iterator.next());
- return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
- }else{
- return NextOutcome.NONE_LEFT;
- }
-
- }
-
- @Override
- public ROP getParent() {
- return UnionROP.this;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/WindowFrameROP.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/WindowFrameROP.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/WindowFrameROP.java
deleted file mode 100644
index 30e472b..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/WindowFrameROP.java
+++ /dev/null
@@ -1,321 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Queues;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.logical.data.WindowFrame;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues;
-
-import java.util.List;
-import java.util.Queue;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-public class WindowFrameROP extends SingleInputROPBase<WindowFrame> {
-
- private RecordIterator incoming;
- private FieldReference segmentRef;
- private FieldReference positionRef;
- private FieldReference withinRef;
- private boolean withinConstrained;
-
- public WindowFrameROP(WindowFrame config) {
- super(config);
- WindowFrame.FrameRef frameRef = config.getFrame();
- if (frameRef != null) {
- positionRef = frameRef.getPosition();
- segmentRef = frameRef.getSegment();
- }
-
- if (positionRef == null) {
- positionRef = new FieldReference("ref.position", ExpressionPosition.UNKNOWN);
- }
-
- if (segmentRef == null) {
- segmentRef = new FieldReference("ref.segment", ExpressionPosition.UNKNOWN);
- }
-
- withinRef = config.getWithin();
- withinConstrained = withinRef != null;
- }
-
- @Override
- protected void setInput(RecordIterator incoming) {
- this.incoming = incoming;
- }
-
- @Override
- protected RecordIterator getIteratorInternal() {
- return new WindowIterator(config.getStart(), config.getEnd());
- }
-
- private class Window {
- List<WindowObjectHolder> holders;
- int windowId;
- int nextRecordIndex;
- int lastIndex;
- int nextPosition;
-
- private Window(long start, long end, Window lastWindow) {
- this.holders = Lists.newArrayList();
- if (lastWindow != null) {
- this.windowId = lastWindow.getWindowId() + 1;
- lastIndex = (int) (windowId + end);
- final int lastMinIndex = (int) Math.max(windowId + start, 0);
- Iterable<WindowObjectHolder> lastHolders = Iterables.filter(lastWindow.getHolders(), new Predicate<WindowObjectHolder>() {
- @Override
- public boolean apply(WindowObjectHolder windowObjectHolder) {
- return windowObjectHolder.getIndex() >= lastMinIndex;
- }
- });
- for (WindowObjectHolder holder : lastHolders) {
- holder.setPosition(nextPosition());
- addRecord(holder);
- }
- } else {
- this.windowId = 0;
- lastIndex = (int) (windowId + end);
- }
-
- }
-
- private int getWindowId() {
- return windowId;
- }
-
- private boolean isFull() {
- if (holders.isEmpty()) {
- return false;
- }
-
- WindowObjectHolder lastHolder = holders.get(holders.size() - 1);
- return lastHolder.getIndex() >= lastIndex;
- }
-
- private void addRecord(RecordPointer pointer, int index, boolean schemaChanged) {
- addRecord(new WindowObjectHolder(pointer, nextPosition(), index, schemaChanged));
- }
-
- private void addRecord(WindowObjectHolder holder) {
- if (!isFull()) {
- holders.add(holder);
- } else {
- throw new DrillRuntimeException("Adding more records into windows then configured.");
- }
- }
-
- private int nextPosition() {
- int position = nextPosition;
- if (nextPosition == 0) {
- nextPosition = 1;
- } else if (nextPosition > 0) {
- nextPosition = -nextPosition;
- } else {
- nextPosition = -nextPosition + 1;
- }
- return position;
- }
-
- public List<WindowObjectHolder> getHolders() {
- return holders;
- }
-
- public WindowObjectHolder nextRecord() {
- if (nextRecordIndex >= holders.size()) {
- return null;
- } else {
- return holders.get(nextRecordIndex++);
- }
- }
-
- public boolean isCrossedWithinBoundary(RecordPointer nextRecord) {
- if (withinConstrained && nextRecord != null && !holders.isEmpty()) {
- DataValue lastWithinVal = holders.get(holders.size() - 1).getPointer().getField(withinRef);
- DataValue nextWithinVal = nextRecord.getField(withinRef);
- boolean lastIsNull = lastWithinVal == null;
- boolean nextIsNull = nextWithinVal == null;
- return lastIsNull != nextIsNull || (!nextIsNull && !nextWithinVal.equals(lastWithinVal));
- }
-
- return false;
- }
-
- public void removeHoldersBeforeIndex(final int index) {
- Iterables.removeIf(holders, new Predicate<WindowObjectHolder>() {
- @Override
- public boolean apply(WindowObjectHolder windowObjectHolder) {
- return windowObjectHolder.getIndex() <= index;
- }
- });
- }
- }
-
- private class WindowObjectHolder {
- private final RecordPointer pointer;
- private int position;
- private final int index;
- private final boolean schemaChanged;
- private int windowId;
-
- private WindowObjectHolder(RecordPointer pointer, int position, int index, boolean schemaChanged) {
- this.pointer = pointer;
- this.position = position;
- this.index = index;
- this.schemaChanged = schemaChanged;
- }
-
- public WindowObjectHolder setWindowId(int windowId) {
- this.windowId = windowId;
- return this;
- }
-
- public RecordPointer getPointer() {
- return pointer;
- }
-
- public int getPosition() {
- return position;
- }
-
- public int getIndex() {
- return index;
- }
-
- public boolean isSchemaChanged() {
- return schemaChanged;
- }
-
- public int getWindowId() {
- return windowId;
- }
-
- public void setPosition(int position) {
- this.position = position;
- }
- }
-
- private class WindowManager {
- Queue<WindowObjectHolder> holderBuffer;
- Window curWindow;
- long start;
- long end;
-
- WindowManager(long start, long end) {
- holderBuffer = Queues.newLinkedBlockingDeque();
- this.start = start;
- this.end = end;
- }
-
- private WindowObjectHolder nextRecord() {
- if (curWindow != null) {
- WindowObjectHolder holder = curWindow.nextRecord();
- if (holder != null) {
- return holder.setWindowId(curWindow.getWindowId());
- } else if (!holderBuffer.isEmpty()) {
- WindowObjectHolder obj = holderBuffer.poll();
- addRecord(obj.getPointer(), obj.getIndex(), obj.isSchemaChanged());
- return nextRecord();
- }
- }
-
- return null;
- }
-
- public void addRecord(RecordPointer recordPointer, int index, boolean schemaChanged) {
- if (curWindow == null || curWindow.isFull()) {
- curWindow = new Window(start, end, curWindow);
- } else if (curWindow.isCrossedWithinBoundary(recordPointer)) {
- if (index - 1 == curWindow.getWindowId()) {
- curWindow.removeHoldersBeforeIndex(curWindow.getWindowId());
- }
- curWindow = new Window(start, end, curWindow);
- }
-
- if (curWindow.isCrossedWithinBoundary(recordPointer)) {
- holderBuffer.add(new WindowObjectHolder(recordPointer, 0, index, schemaChanged));
- } else {
- curWindow.addRecord(recordPointer, index, schemaChanged);
- }
- }
-
- public boolean hasMoreWindows(int curIndex) {
- return curWindow != null && curWindow.getWindowId() < curIndex;
- }
-
- public WindowObjectHolder nextWindowOnEmpty() {
- curWindow = new Window(start, end, curWindow);
- return nextRecord();
- }
- }
-
-
- private class WindowIterator implements RecordIterator {
- private int curIndex = -1;
- private WindowManager windowManager;
- private ProxySimpleRecord windowPointer;
-
- public WindowIterator(long start, long end) {
- checkArgument(end - start >= 0, "Invalid end and start. end: %s, start: %s", end, start);
- windowManager = new WindowManager(start, end);
- windowPointer = new ProxySimpleRecord();
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return windowPointer;
- }
-
- @Override
- public NextOutcome next() {
- WindowObjectHolder holder = windowManager.nextRecord();
- if (holder == null) {
- NextOutcome outcome = incoming.next();
- if (outcome != NextOutcome.NONE_LEFT) {
- windowManager.addRecord(incoming.getRecordPointer().copy(), ++curIndex, outcome == NextOutcome.INCREMENTED_SCHEMA_CHANGED);
- holder = windowManager.nextRecord();
- } else if (windowManager.hasMoreWindows(curIndex)) {
- holder = windowManager.nextWindowOnEmpty();
- }
-
- if (holder == null) {
- return NextOutcome.NONE_LEFT;
- }
- }
-
- RecordPointer newPointer = holder.getPointer().copy();
- newPointer.addField(positionRef, new ScalarValues.IntegerScalar(holder.getPosition()));
- newPointer.addField(segmentRef, new ScalarValues.IntegerScalar(holder.getWindowId()));
- windowPointer.setRecord(newPointer);
- return holder.isSchemaChanged() ? NextOutcome.INCREMENTED_SCHEMA_CHANGED : NextOutcome.INCREMENTED_SCHEMA_UNCHANGED;
- }
-
- @Override
- public ROP getParent() {
- return WindowFrameROP.this;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java
deleted file mode 100644
index 8e68643..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.Collection;
-import java.util.Collections;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.apache.drill.exec.ref.rops.DataWriter.ConverterType;
-import org.apache.drill.exec.ref.rops.ROP;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-public class ClasspathRSE extends RSEBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClasspathRSE.class);
-
- private DrillConfig dConfig;
- private SchemaPath rootPath;
-
- public ClasspathRSE(ClasspathRSEConfig engineConfig, DrillConfig dConfig) throws SetupException{
- this.dConfig = dConfig;
- }
-
-
- @JsonTypeName("classpath")
- public static class ClasspathRSEConfig extends StorageEngineConfigBase {
- @Override
- public boolean equals(Object o) {
- // if fields are added to this class this method should be changed
- return true;
- }
- }
-
- public static class ClasspathInputConfig implements ReadEntry{
- public String path;
- public ConverterType type;
- @JsonIgnore public SchemaPath rootPath;
- }
-
- public boolean supportsRead() {
- return true;
- }
-
- @Override
- public Collection<ReadEntry> getReadEntries(Scan scan) throws IOException {
- ClasspathInputConfig c = scan.getSelection().getWith(dConfig, ClasspathInputConfig.class);
- c.rootPath = scan.getOutputReference();
- return Collections.singleton((ReadEntry) c);
- }
-
- @Override
- public RecordReader getReader(ReadEntry readEntry, ROP parentROP) throws IOException {
- ClasspathInputConfig e = getReadEntry(ClasspathInputConfig.class, readEntry);
- URL u = RecordReader.class.getResource(e.path);
- if(u == null){
- throw new IOException(String.format("Failure finding classpath resource %s.", e.path));
- }
- switch(e.type){
- case JSON:
- return new JSONRecordReader(e.rootPath, dConfig, u.openStream(), parentROP);
- default:
- throw new UnsupportedOperationException();
- }
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java
deleted file mode 100644
index 901a35c..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.OutputStream;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.exec.ref.rops.DataWriter.ConverterType;
-
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-public class ConsoleRSE extends RSEBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ConsoleRSE.class);
-
- private final DrillConfig dConfig;
-
- public static enum Pipe {
- STD_OUT, STD_ERR
- };
-
- public ConsoleRSE(ConsoleRSEConfig engineConfig, DrillConfig dConfig){
- this.dConfig = dConfig;
- }
-
- public static class ConsoleOutputConfig {
- public Pipe pipe = Pipe.STD_OUT;
- public ConverterType type = ConverterType.JSON;
- }
-
- @JsonTypeName("console") public static class ConsoleRSEConfig extends StorageEngineConfigBase {
- @Override
- public boolean equals(Object o) {
- // if fields are added to this class this method needs to be updated
- return true;
- }
- }
-
- public boolean supportsWrite() {
- return true;
- }
-
- @Override
- public RecordRecorder getWriter(Store store) {
- ConsoleOutputConfig config = store.getTarget().getWith(dConfig, ConsoleOutputConfig.class);
- OutputStream out = config.pipe == Pipe.STD_OUT ? System.out : System.err;
- return new OutputStreamWriter(out, config.type, false);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java
deleted file mode 100644
index 80ec9cd..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.apache.drill.exec.ref.rops.DataWriter.ConverterType;
-import org.apache.drill.exec.ref.rops.ROP;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-public class FileSystemRSE extends RSEBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemRSE.class);
-
- private FileSystem fs;
- private Path basePath;
- private DrillConfig dConfig;
-
- public FileSystemRSE(FileSystemRSEConfig engineConfig, DrillConfig dConfig) throws SetupException{
- this.dConfig = dConfig;
-
- try {
- URI u = new URI(engineConfig.root);
- String path = u.getPath();
-
- if(path.charAt(path.length()-1) != '/') throw new SetupException(String.format("The file root provided of %s included a file '%s'. This must be a base path.", engineConfig.root, u.getPath()));
- fs = FileSystem.get(u, new Configuration());
- basePath = new Path(u.getPath());
- } catch (URISyntaxException | IOException e) {
- throw new SetupException("Failure while reading setting up file system root path.", e);
- }
- }
-
-
- @JsonTypeName("fs")
- public static class FileSystemRSEConfig extends StorageEngineConfigBase {
- private String root;
-
- @JsonCreator
- public FileSystemRSEConfig(@JsonProperty("root") String root) {
- this.root = root;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- FileSystemRSEConfig that = (FileSystemRSEConfig) o;
-
- if (root != null ? !root.equals(that.root) : that.root != null) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return root != null ? root.hashCode() : 0;
- }
- }
-
- public static class FileSystemInputConfig {
- public FileSpec[] files;
- }
-
- public static class FileSpec{
- public String path;
- public ConverterType type;
- }
-
-
- public class FSEntry implements ReadEntry{
- Path path;
- ConverterType type;
- SchemaPath rootPath;
-
- public FSEntry(FileSpec spec, SchemaPath rootPath){
- this.path = new Path(basePath, spec.path);
- this.type = spec.type;
- this.rootPath = rootPath;
- }
-
- }
-
- public class FileSystemOutputConfig {
- public String file;
- public ConverterType type;
- }
-
- public boolean supportsRead() {
- return true;
- }
-
- public boolean supportsWrite() {
- return true;
- }
-
- @Override
- public RecordRecorder getWriter(Store store) throws IOException {
- FileSystemOutputConfig config = store.getTarget().getWith(dConfig, FileSystemOutputConfig.class);
- OutputStream out = fs.create(new Path(basePath, config.file));
- return new OutputStreamWriter(out, config.type, true);
- }
-
- @Override
- public Collection<ReadEntry> getReadEntries(Scan scan) throws IOException {
- Set<ReadEntry> s = new HashSet<ReadEntry>();
- for(FileSpec f : scan.getSelection().getWith(dConfig, FileSystemInputConfig.class).files){
- s.add(new FSEntry(f, scan.getOutputReference()));
- }
- return s;
- }
-
- @Override
- public RecordReader getReader(ReadEntry readEntry, ROP parentROP) throws IOException {
- FSEntry e = getReadEntry(FSEntry.class, readEntry);
-
- switch(e.type){
- case JSON:
- return new JSONRecordReader(e.rootPath, dConfig, fs.open(e.path), parentROP);
- default:
- throw new UnsupportedOperationException();
- }
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/JSONDataWriter.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/JSONDataWriter.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/JSONDataWriter.java
deleted file mode 100644
index 34fa427..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/JSONDataWriter.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.apache.drill.exec.ref.rops.DataWriter;
-
-import com.fasterxml.jackson.core.JsonEncoding;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
-
-public class JSONDataWriter implements DataWriter{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONDataWriter.class);
-
- private final JsonGenerator g;
-// private CharSequence transientName;
-
- public JSONDataWriter(OutputStream out) throws IOException{
- JsonFactory f = new JsonFactory();
-
- this.g = f.createJsonGenerator(out, JsonEncoding.UTF8);
- this.g.useDefaultPrettyPrinter();
- }
-
- private String s(CharSequence seq) {
- String s = (seq instanceof String) ? (String) seq : seq.toString();
- return s;
- }
-
- @Override
- public void startRecord() throws IOException {
-
- }
-
- @Override
- public void writeArrayStart(int length) throws IOException {
- g.writeStartArray();
- }
-
- @Override
- public void writeArrayElementStart() throws IOException {
- }
-
- @Override
- public void writeArrayElementEnd() throws IOException {
- }
-
- @Override
- public void writeArrayEnd() throws IOException {
- g.writeEndArray();
- }
-
- @Override
- public void writeMapStart() throws IOException {
- g.writeStartObject();
- }
-
- @Override
- public void writeMapKey(CharSequence seq) throws IOException {
- g.writeFieldName(s(seq));
- }
-
- @Override
- public void writeMapValueStart() throws IOException {
- }
-
- @Override
- public void writeMapValueEnd() throws IOException {
- }
-
- @Override
- public void writeMapEnd() throws IOException {
- g.writeEndObject();
- }
-
- @Override
- public void writeBoolean(boolean b) throws IOException {
- g.writeBoolean(b);
- }
-
- @Override
- public void writeSInt32(int value) throws IOException {
- g.writeNumber(value);
- }
-
- @Override
- public void writeSInt64(long value) throws IOException {
- g.writeNumber(value);
- }
-
- @Override
- public void writeBytes(byte[] bytes) throws IOException {
- g.writeBinary(bytes);
- }
-
- @Override
- public void writeSFloat64(double value) throws IOException {
- g.writeNumber(value);
- }
-
- @Override
- public void writeSFloat32(float value) throws IOException {
- g.writeNumber(value);
- }
-
- @Override
- public void writeNullValue() throws IOException {
- g.writeNull();
- }
-
- @Override
- public void writeCharSequence(CharSequence value) throws IOException {
- g.writeString(s(value));
- }
-
- @Override
- public void endRecord() throws IOException {
- g.writeRawValue("\n");
- }
-
- public void finish() throws IOException{
- g.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/JSONRecordReader.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/JSONRecordReader.java
deleted file mode 100644
index f73bee5..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/JSONRecordReader.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.Iterator;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.RunOutcome;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.exceptions.RecordException;
-import org.apache.drill.exec.ref.rops.ROP;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues.BooleanScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.BytesScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.DoubleScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.IntegerScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.LongScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.StringScalar;
-import org.apache.drill.exec.ref.values.SimpleArrayValue;
-import org.apache.drill.exec.ref.values.SimpleMapValue;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Charsets;
-
-public class JSONRecordReader implements RecordReader {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader.class);
-
- private InputStreamReader input;
- private String file;
- private SchemaPath rootPath;
- private JsonParser parser;
- private UnbackedRecord record = new UnbackedRecord();
- private ObjectMapper mapper;
- private ROP parent;
-
- public JSONRecordReader(SchemaPath rootPath, DrillConfig dConfig, InputStream stream, ROP parent) throws IOException {
- this.input = new InputStreamReader(stream, Charsets.UTF_8);
- this.mapper = dConfig.getMapper();
- this.parser = mapper.getFactory().createJsonParser(input);
- this.parent = parent;
- this.rootPath = rootPath;
- }
-
- private class NodeIter implements RecordIterator {
-
- @Override
- public NextOutcome next() {
-// logger.debug("Next Record Called");
- try {
- if (parser.nextToken() == null) {
-// logger.debug("No current token, returning.");
- return NextOutcome.NONE_LEFT;
- }
- JsonNode n = mapper.readTree(parser);
- if (n == null) {
-// logger.debug("Nothing was returned for read tree, returning.");
- return NextOutcome.NONE_LEFT;
- }
-// logger.debug("Record found, returning new json record.");
- record.setClearAndSetRoot(rootPath, convert(n));
- // todo, add schema checking here.
- return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
- } catch (IOException e) {
- throw new RecordException("Failure while reading record", null, e);
- }
- }
-
-
- @Override
- public RecordPointer getRecordPointer() {
- return record;
- }
-
-
- @Override
- public ROP getParent() {
- return parent;
- }
-
- }
-
- public static DataValue convert(JsonNode node) {
- if (node == null || node.isNull() || node.isMissingNode()) {
- return DataValue.NULL_VALUE;
- } else if (node.isArray()) {
- SimpleArrayValue arr = new SimpleArrayValue(node.size());
- for (int i = 0; i < node.size(); i++) {
- arr.addToArray(i, convert(node.get(i)));
- }
- return arr;
- } else if (node.isObject()) {
- SimpleMapValue map = new SimpleMapValue();
- String name;
- for (Iterator<String> iter = node.fieldNames(); iter.hasNext();) {
- name = iter.next();
- map.setByName(name, convert(node.get(name)));
- }
- return map;
- } else if (node.isBinary()) {
- try {
- return new BytesScalar(node.binaryValue());
- } catch (IOException e) {
- throw new RuntimeException("Failure converting binary value.", e);
- }
- } else if (node.isBigDecimal()) {
- throw new UnsupportedOperationException();
-// return new BigDecimalScalar(node.decimalValue());
- } else if (node.isBigInteger()) {
- throw new UnsupportedOperationException();
-// return new BigIntegerScalar(node.bigIntegerValue());
- } else if (node.isBoolean()) {
- return new BooleanScalar(node.asBoolean());
- } else if (node.isFloatingPointNumber()) {
- if (node.isBigDecimal()) {
- throw new UnsupportedOperationException();
-// return new BigDecimalScalar(node.decimalValue());
- } else {
- return new DoubleScalar(node.asDouble());
- }
- } else if (node.isInt()) {
- return new IntegerScalar(node.asInt());
- } else if (node.isLong()) {
- return new LongScalar(node.asLong());
- } else if (node.isTextual()) {
- return new StringScalar(node.asText());
- } else {
- throw new UnsupportedOperationException(String.format("Don't know how to convert value of type %s.", node
- .getClass().getCanonicalName()));
- }
-
- }
-
-
- /* (non-Javadoc)
- * @see org.apache.drill.exec.ref.rse.DataReader#getIterator()
- */
- @Override
- public RecordIterator getIterator() {
- return new NodeIter();
- }
-
- /* (non-Javadoc)
- * @see org.apache.drill.exec.ref.rse.DataReader#cleanup()
- */
- @Override
- public void cleanup() {
- try {
- parser.close();
- this.input.close();
- } catch (IOException e) {
- logger.warn("Error while closing InputStream for file {}", file, e);
- }
-
- }
-
-
- @Override
- public void setup() {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/OutputStreamWriter.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/OutputStreamWriter.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/OutputStreamWriter.java
deleted file mode 100644
index baae164..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/OutputStreamWriter.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
-import org.apache.drill.exec.ref.rops.DataWriter;
-import org.apache.drill.exec.ref.rops.DataWriter.ConverterType;
-import org.apache.hadoop.fs.FSDataOutputStream;
-
-public class OutputStreamWriter implements RecordRecorder{
-
- private OutputStream stream;
- private FSDataOutputStream posStream;
- private DataWriter writer;
- private ConverterType type;
- private boolean closeStream;
-
- public OutputStreamWriter(OutputStream stream, ConverterType type, boolean closeStream){
- this.stream = stream;
- this.closeStream = closeStream;
- this.type = type;
- if(stream instanceof FSDataOutputStream) posStream = (FSDataOutputStream) stream;
- }
-
- @Override
- public void setup() throws IOException {
- DataWriter w = null;
- switch(type){
- case JSON:
- w = new JSONDataWriter(stream);
- break;
- default:
- throw new UnsupportedOperationException();
- }
- this.writer = w;
- }
-
- private long getPos() throws IOException{
- if(posStream == null) return 0;
- return posStream.getPos();
- }
-
- @Override
- public long recordRecord(RecordPointer pointer) throws IOException {
- pointer.write(writer);
- return getPos();
- }
-
- @Override
- public void finish(OutcomeType outcome) throws IOException {
- writer.finish();
- if(closeStream){
- stream.close();
- }else{
- stream.flush();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/QueueRSE.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/QueueRSE.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/QueueRSE.java
deleted file mode 100644
index 96aef77..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/QueueRSE.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ArrayBlockingQueue;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-import javax.validation.constraints.NotNull;
-
-public class QueueRSE extends RSEBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueueRSE.class);
-
- private DrillConfig dConfig;
- private final List<Queue<Object>> sinkQueues;
- private final QueueRSEConfig engineConfig;
-
- public QueueRSE(QueueRSEConfig engineConfig, DrillConfig dConfig) throws SetupException{
- this.dConfig = dConfig;
- this.engineConfig = engineConfig;
- sinkQueues = Collections.singletonList( (Queue<Object>) (new ArrayBlockingQueue<Object>(100)));
- }
-
- public Queue<Object> getQueue(int number){
- return sinkQueues.get(number);
- }
-
- @JsonTypeName("queue")
- public static class QueueRSEConfig extends StorageEngineConfigBase {
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- QueueRSEConfig that = (QueueRSEConfig) o;
-
- if (encoding != that.encoding) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return encoding != null ? encoding.hashCode() : 0;
- }
-
- public static enum Encoding {JSON, RECORD};
-
- private final Encoding encoding;
-
- @JsonCreator
- public QueueRSEConfig(@JsonProperty("encoding") Encoding encoding) {
- this.encoding = encoding == null ? Encoding.JSON : encoding;
- }
-
- public Encoding getEncoding() {
- return encoding;
- }
-
-
- }
-
- public static class QueueOutputInfo{
- public int number;
- }
-
-
- public boolean supportsWrite() {
- return true;
- }
-
-
- @Override
- public RecordRecorder getWriter(Store store) throws IOException {
- QueueOutputInfo config = store.getTarget().getWith(dConfig, QueueOutputInfo.class);
- Queue<Object> q = dConfig.getQueue(config.number);
- return new QueueRecordRecorder(q);
- }
-
-
- private class QueueRecordRecorder implements RecordRecorder{
-
- private final Queue<Object> queue;
-
- public QueueRecordRecorder(Queue<Object> queue) {
- this.queue = Objects.requireNonNull(queue);
- }
-
- @Override
- public void setup() throws IOException {
- }
-
- @Override
- public long recordRecord(RecordPointer r) throws IOException {
- switch(engineConfig.encoding){
- case RECORD:
- queue.add(r.copy());
- return 0;
- default:
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final JSONDataWriter writer = new JSONDataWriter(baos);
- r.write(writer);
- writer.finish();
- queue.add(baos.toByteArray());
- return 0;
- }
-
- }
-
- @Override
- public void finish(OutcomeType type) throws IOException {
- queue.add(type);
- }
-
- }
-
-
-}
[13/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java
new file mode 100644
index 0000000..fae18ca
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.ValuesRel;
+import org.eigenbase.relopt.*;
+
+/**
+ * Rule that converts a {@link ValuesRel} to a Drill "values" operation.
+ */
+public class DrillValuesRule extends RelOptRule {
+ public static final RelOptRule INSTANCE = new DrillValuesRule();
+
+ private DrillValuesRule() {
+ super(RelOptHelper.any(ValuesRel.class, Convention.NONE), "DrillValuesRule");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final ValuesRel values = (ValuesRel) call.rel(0);
+ final RelTraitSet traits = values.getTraitSet().plus(DrillRel.CONVENTION);
+ call.transformTo(new DrillValuesRel(values.getCluster(), values.getRowType(), values.getTuples(), traits));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/EnumerableDrillRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/EnumerableDrillRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/EnumerableDrillRule.java
new file mode 100644
index 0000000..2fcf660
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/EnumerableDrillRule.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import net.hydromatic.optiq.rules.java.EnumerableConvention;
+
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.convert.ConverterRule;
+
+/**
+ * Rule that converts any Drill relational expression to enumerable format by adding a {@link DrillScreenRel}.
+ */
+public class EnumerableDrillRule extends ConverterRule {
+
+ public static EnumerableDrillRule INSTANCE = new EnumerableDrillRule(EnumerableConvention.INSTANCE);
+
+
+ private EnumerableDrillRule(EnumerableConvention outConvention) {
+ super(RelNode.class, DrillRel.CONVENTION, outConvention, "EnumerableDrillRule." + outConvention);
+ }
+
+ @Override
+ public boolean isGuaranteed() {
+ return true;
+ }
+
+ @Override
+ public RelNode convert(RelNode rel) {
+ assert rel.getTraitSet().contains(DrillRel.CONVENTION);
+ return new DrillScreenRel(rel.getCluster(), rel.getTraitSet().replace(getOutConvention()), rel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ExprHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ExprHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ExprHelper.java
new file mode 100644
index 0000000..9456a81
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ExprHelper.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.List;
+
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+
+public class ExprHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExprHelper.class);
+
+ private final static String COMPOUND_FAIL_MESSAGE = "The current Optiq based logical plan interpreter does not complicated expressions. For Order By and Filter";
+
+ public static String getAggregateFieldName(FunctionCall c){
+ List<LogicalExpression> exprs = c.args;
+ if(exprs.size() != 1) throw new UnsupportedOperationException(COMPOUND_FAIL_MESSAGE);
+ return getFieldName(exprs.iterator().next());
+ }
+
+ public static String getFieldName(LogicalExpression e){
+ if(e instanceof SchemaPath) return ((SchemaPath) e).getPath().toString();
+ throw new UnsupportedOperationException(COMPOUND_FAIL_MESSAGE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java
new file mode 100644
index 0000000..835e4a5
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+
+
+
+import org.eigenbase.reltype.*;
+import org.eigenbase.sql.SqlCollation;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlIntervalQualifier;
+import org.eigenbase.sql.type.SqlTypeName;
+
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+/* We use an instance of this class as the row type for
+ * Drill table. Since we don't know the schema before hand
+ * whenever optiq requires us to validate that a field exists
+ * we always return true and indicate that the type of that
+ * field is 'ANY'
+ */
+public class RelDataTypeDrillImpl extends RelDataTypeImpl {
+
+ private RelDataTypeField defaultField;
+ RelDataTypeFactory typeFactory;
+ List<RelDataTypeField> drillFieldList = new LinkedList<>();
+ List<String> drillfieldNames = new LinkedList<>();
+
+
+ public RelDataTypeDrillImpl(RelDataTypeFactory typeFactory) {
+ this.typeFactory = typeFactory;
+ computeDigest();
+ }
+
+ @Override
+ public List<RelDataTypeField> getFieldList() {
+
+ if (drillFieldList.size() == 0)
+ {
+ /* By default we only have a single row in drill of 'ANY' type
+ * (mainly for select * type queries)
+ */
+ defaultField = new RelDataTypeFieldImpl("*", 0, typeFactory.createSqlType(SqlTypeName.ANY));
+
+ drillFieldList.add(defaultField);
+ drillfieldNames.add("*");
+ }
+ return drillFieldList;
+ }
+
+
+ @Override
+ public int getFieldCount() {
+ return drillFieldList.size();
+ }
+
+// @Override
+// public int getFieldOrdinal(String fieldName, boolean caseSensitive) {
+//
+// /* Get the list of fields and return the
+// * index if the field exists
+// */
+// for (RelDataTypeField field : drillFieldList) {
+// if (field.getName().equals(fieldName))
+// return field.getIndex();
+// }
+//
+// /* Couldn't find the field in our list, return -1
+// * Unsure if I should add it to our list of fields
+// */
+// return -1;
+// }
+
+ @Override
+ /**
+ *
+ */
+ public RelDataTypeField getField(String fieldName, boolean caseSensitive) {
+
+ /* First check if this field name exists in our field list */
+ for (RelDataTypeField field : drillFieldList)
+ {
+ if (field.getName().equals(fieldName))
+ return field;
+ }
+
+ /* This field does not exist in our field list add it */
+ RelDataTypeField newField = new RelDataTypeFieldImpl(fieldName, drillFieldList.size(), typeFactory.createSqlType(SqlTypeName.ANY));
+
+ /* Add it to the list of fields */
+ drillFieldList.add(newField);
+
+ /* Add the name to our list of field names */
+ drillfieldNames.add(fieldName);
+
+ return newField;
+ }
+
+
+ @Override
+ public List<String> getFieldNames() {
+
+ if (drillfieldNames.size() == 0) {
+ drillfieldNames.add("*");
+ }
+
+ return drillfieldNames;
+ }
+
+ @Override
+ public SqlTypeName getSqlTypeName() {
+ return null;
+ }
+
+ @Override
+ protected void generateTypeString(StringBuilder sb, boolean withDetail) {
+ sb.append("DrillRecordRow");
+ }
+
+ @Override
+ public boolean isStruct() {
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelOptHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelOptHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelOptHelper.java
new file mode 100644
index 0000000..92272f8
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelOptHelper.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleOperand;
+import org.eigenbase.relopt.RelTrait;
+
+public class RelOptHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RelOptHelper.class);
+
+ public static RelOptRuleOperand any(Class<? extends RelNode> first, RelTrait trait){
+ return RelOptRule.operand(first, trait, RelOptRule.any());
+ }
+
+ public static RelOptRuleOperand any(Class<? extends RelNode> first){
+ return RelOptRule.operand(first, RelOptRule.any());
+ }
+
+ public static RelOptRuleOperand some(Class<? extends RelNode> rel, RelOptRuleOperand first, RelOptRuleOperand... rest){
+ return RelOptRule.operand(rel, RelOptRule.some(first, rest));
+ }
+
+ public static RelOptRuleOperand some(Class<? extends RelNode> rel, RelTrait trait, RelOptRuleOperand first, RelOptRuleOperand... rest){
+ return RelOptRule.operand(rel, trait, RelOptRule.some(first, rest));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ScanFieldDeterminer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ScanFieldDeterminer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ScanFieldDeterminer.java
new file mode 100644
index 0000000..9809770
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ScanFieldDeterminer.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.logical.data.Constant;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.GroupingAggregate;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.JoinCondition;
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Order.Ordering;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.SinkOperator;
+import org.apache.drill.common.logical.data.Store;
+import org.apache.drill.common.logical.data.Union;
+import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+/**
+ * This visitor will walk a logical plan and record in a map the list of field references associated to each scan. These
+ * can then be used to update scan object to appear to be explicitly fielded for optimization purposes.
+ */
+public class ScanFieldDeterminer extends AbstractLogicalVisitor<Void, ScanFieldDeterminer.FieldList, RuntimeException> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanFieldDeterminer.class);
+
+ private FieldReferenceFinder finder = new FieldReferenceFinder();
+ private Map<Scan, FieldList> scanFields = Maps.newHashMap();
+
+
+ public static Map<Scan, FieldList> getFieldLists(LogicalPlan plan){
+ Collection<SinkOperator> ops = plan.getGraph().getRoots();
+ Preconditions.checkArgument(ops.size() == 1, "Scan Field determiner currently only works with plans that have a single root.");
+ ScanFieldDeterminer sfd = new ScanFieldDeterminer();
+ ops.iterator().next().accept(sfd, new FieldList());
+ return sfd.scanFields;
+ }
+
+ private ScanFieldDeterminer(){
+ }
+
+ public static class FieldList {
+ private Set<SchemaPath> projected = Sets.newHashSet();
+ private Set<SchemaPath> referenced = Sets.newHashSet();
+
+ public void addProjected(SchemaPath path) {
+ projected.add(path);
+ }
+
+ public void addReferenced(SchemaPath path) {
+ referenced.add(path);
+ }
+
+ public void addReferenced(Collection<SchemaPath> paths) {
+ referenced.addAll(paths);
+ }
+
+ public void addProjected(Collection<SchemaPath> paths) {
+ projected.addAll(paths);
+ }
+
+ public FieldList clone() {
+ FieldList newList = new FieldList();
+ for (SchemaPath p : projected) {
+ newList.addProjected(p);
+ }
+ for (SchemaPath p : referenced) {
+ newList.addReferenced(p);
+ }
+ return newList;
+ }
+ }
+
+ @Override
+ public Void visitScan(Scan scan, FieldList value) {
+ if (value == null) {
+ scanFields.put(scan, new FieldList());
+ } else {
+ scanFields.put(scan, value);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitStore(Store store, FieldList value) {
+ store.getInput().accept(this, value);
+ return null;
+ }
+
+ @Override
+ public Void visitGroupingAggregate(GroupingAggregate groupBy, FieldList value) {
+ FieldList list = new FieldList();
+ for (NamedExpression e : groupBy.getExprs()) {
+ list.addProjected(e.getExpr().accept(finder, null));
+ }
+ for (NamedExpression e : groupBy.getKeys()) {
+ list.addProjected(e.getExpr().accept(finder, null));
+ }
+ groupBy.getInput().accept(this, list);
+ return null;
+ }
+
+ @Override
+ public Void visitFilter(Filter filter, FieldList value) {
+ value.addReferenced(filter.getExpr().accept(finder, null));
+ return null;
+ }
+
+ @Override
+ public Void visitProject(Project project, FieldList value) {
+ FieldList fl = new FieldList();
+ for (NamedExpression e : project.getSelections()) {
+ fl.addProjected(e.getExpr().accept(finder, null));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitConstant(Constant constant, FieldList value) {
+ return null;
+ }
+
+ @Override
+ public Void visitOrder(Order order, FieldList fl) {
+ for (Ordering o : order.getOrderings()) {
+ fl.addReferenced(o.getExpr().accept(finder, null));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitJoin(Join join, FieldList fl) {
+ {
+ FieldList leftList = fl.clone();
+ for (JoinCondition c : join.getConditions()) {
+ leftList.addReferenced(c.getLeft().accept(finder, null));
+ }
+ join.getLeft().accept(this, leftList);
+ }
+
+ {
+ FieldList rightList = fl.clone();
+ for (JoinCondition c : join.getConditions()) {
+ rightList.addReferenced(c.getRight().accept(finder, null));
+ }
+ join.getLeft().accept(this, rightList);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLimit(Limit limit, FieldList value) {
+ limit.getInput().accept(this, value);
+ return null;
+ }
+
+ @Override
+ public Void visitUnion(Union union, FieldList value) {
+ for (LogicalOperator o : union.getInputs()) {
+ o.accept(this, value.clone());
+ }
+ return null;
+ }
+
+
+ /**
+ * Search through a LogicalExpression, finding all internal schema path references and returning them in a set.
+ */
+ private class FieldReferenceFinder extends AbstractExprVisitor<Set<SchemaPath>, Void, RuntimeException> {
+
+ @Override
+ public Set<SchemaPath> visitSchemaPath(SchemaPath path, Void value) {
+ Set<SchemaPath> set = Sets.newHashSet();
+ set.add(path);
+ return set;
+ }
+
+ @Override
+ public Set<SchemaPath> visitUnknown(LogicalExpression e, Void value) {
+ Set<SchemaPath> paths = Sets.newHashSet();
+ for (LogicalExpression ex : e) {
+ paths.addAll(ex.accept(this, null));
+ }
+ return paths;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
new file mode 100644
index 0000000..a020ab3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.StorageEngineConfig;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
+
+public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineConfig>>{
+
+ private Map<String, StorageEngineConfig> storage;
+
+ @JsonCreator
+ public StorageEngines(@JsonProperty("storage") Map<String, StorageEngineConfig> storage){
+ this.storage = storage;
+ }
+
+ public static void main(String[] args) throws Exception{
+ DrillConfig config = DrillConfig.create();
+ String data = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
+ StorageEngines se = config.getMapper().readValue(data, StorageEngines.class);
+ System.out.println(se);
+ }
+
+ @Override
+ public String toString() {
+ final int maxLen = 10;
+ return "StorageEngines [storage=" + (storage != null ? toString(storage.entrySet(), maxLen) : null) + "]";
+ }
+
+ @Override
+ public Iterator<Entry<String, StorageEngineConfig>> iterator() {
+ return storage.entrySet().iterator();
+ }
+
+ private String toString(Collection<?> collection, int maxLen) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("[");
+ int i = 0;
+ for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
+ if (i > 0)
+ builder.append(", ");
+ builder.append(iterator.next());
+ }
+ builder.append("]");
+ return builder.toString();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
new file mode 100644
index 0000000..23d03b8
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import net.hydromatic.linq4j.function.Function1;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.exec.exception.SetupException;
+import org.apache.drill.exec.planner.logical.StorageEngines;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.exec.store.SchemaProviderRegistry;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class DrillSchemaFactory implements Function1<SchemaPlus, Schema>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSchemaFactory.class);
+
+ private final SchemaProviderRegistry registry;
+ private final Map<String, StorageEngineEntry> preEntries = Maps.newHashMap();
+
+ public static DrillSchemaFactory createEmpty(){
+ return new DrillSchemaFactory();
+ }
+
+ private DrillSchemaFactory(){
+ this.registry = null;
+ }
+
+ public DrillSchemaFactory(StorageEngines engines, DrillConfig config) throws SetupException {
+ super();
+ this.registry = new SchemaProviderRegistry(config);
+
+ for (Map.Entry<String, StorageEngineConfig> entry : engines) {
+ SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
+ preEntries.put(entry.getKey(), new StorageEngineEntry(entry.getValue(), provider));
+ }
+
+ }
+
+ public Schema apply(SchemaPlus root) {
+ List<String> schemaNames = Lists.newArrayList();
+ Schema defaultSchema = null;
+ for(Entry<String, StorageEngineEntry> e : preEntries.entrySet()){
+ FileSystemSchema schema = new FileSystemSchema(e.getValue().getConfig(), e.getValue().getProvider(), root, e.getKey());
+ if(defaultSchema == null) defaultSchema = schema;
+ root.add(schema);
+ schemaNames.add(e.getKey());
+ }
+ logger.debug("Registered schemas for {}", schemaNames);
+ return defaultSchema;
+ }
+
+
+ private class StorageEngineEntry{
+ StorageEngineConfig config;
+ SchemaProvider provider;
+
+
+ public StorageEngineEntry(StorageEngineConfig config, SchemaProvider provider) {
+ super();
+ this.config = config;
+ this.provider = provider;
+ }
+
+ public StorageEngineConfig getConfig() {
+ return config;
+ }
+ public SchemaProvider getProvider() {
+ return provider;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
new file mode 100644
index 0000000..818e892
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql;
+
+import net.hydromatic.optiq.jdbc.ConnectionConfig;
+import net.hydromatic.optiq.tools.Frameworks;
+import net.hydromatic.optiq.tools.Planner;
+import net.hydromatic.optiq.tools.RelConversionException;
+import net.hydromatic.optiq.tools.RuleSet;
+import net.hydromatic.optiq.tools.ValidationException;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.FunctionRegistry;
+import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
+import org.apache.drill.exec.planner.logical.DrillImplementor;
+import org.apache.drill.exec.planner.logical.DrillParseContext;
+import org.apache.drill.exec.planner.logical.DrillRel;
+import org.apache.drill.exec.planner.logical.DrillRuleSets;
+import org.apache.drill.exec.planner.logical.DrillScreenRel;
+import org.apache.drill.exec.planner.logical.DrillStoreRel;
+import org.apache.drill.exec.planner.logical.StorageEngines;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.sql.SqlExplain;
+import org.eigenbase.sql.SqlKind;
+import org.eigenbase.sql.SqlLiteral;
+import org.eigenbase.sql.SqlNode;
+import org.eigenbase.sql.fun.SqlStdOperatorTable;
+import org.eigenbase.sql.parser.SqlParseException;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
+
+public class DrillSqlWorker {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSqlWorker.class);
+
+ private final FunctionRegistry registry;
+ private final Planner planner;
+
+ public DrillSqlWorker(DrillSchemaFactory schemaFactory, FunctionRegistry functionRegistry) throws Exception {
+ this.registry = functionRegistry;
+ this.planner = Frameworks.getPlanner(ConnectionConfig.Lex.MYSQL, schemaFactory, SqlStdOperatorTable.instance(), new RuleSet[]{DrillRuleSets.DRILL_BASIC_RULES});
+
+ }
+
+
+ public LogicalPlan getPlan(String sql) throws SqlParseException, ValidationException, RelConversionException{
+ SqlNode sqlNode = planner.parse(sql);
+
+ ResultMode resultMode = ResultMode.EXEC;
+ if(sqlNode.getKind() == SqlKind.EXPLAIN){
+ SqlExplain explain = (SqlExplain) sqlNode;
+ sqlNode = explain.operands[0];
+ SqlLiteral op = (SqlLiteral) explain.operands[2];
+ SqlExplain.Depth depth = (SqlExplain.Depth) op.getValue();
+ switch(depth){
+ case Logical:
+ resultMode = ResultMode.LOGICAL;
+ break;
+ case Physical:
+ resultMode = ResultMode.PHYSICAL;
+ break;
+ default:
+ }
+
+
+ }
+
+ SqlNode validatedNode = planner.validate(sqlNode);
+ RelNode relNode = planner.convert(validatedNode);
+ RelNode convertedRelNode = planner.transform(0, planner.getEmptyTraitSet().plus(DrillRel.CONVENTION), relNode);
+ if(convertedRelNode instanceof DrillStoreRel){
+ throw new UnsupportedOperationException();
+ }else{
+ convertedRelNode = new DrillScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode);
+ }
+ DrillImplementor implementor = new DrillImplementor(new DrillParseContext(registry), resultMode);
+ implementor.go( (DrillRel) convertedRelNode);
+ planner.close();
+ planner.reset();
+ return implementor.getPlan();
+
+ }
+ private void x() throws Exception {
+ String sqlAgg = "select a, count(1) from parquet.`/Users/jnadeau/region.parquet` group by a";
+ String sql = "select * from parquet.`/Users/jnadeau/region.parquet`";
+
+
+ System.out.println(sql);
+ System.out.println(getPlan(sql).toJsonString(DrillConfig.create()));
+ System.out.println("///////////");
+ System.out.println(sqlAgg);
+ System.out.println(getPlan(sqlAgg).toJsonString(DrillConfig.create()));
+ }
+
+ public static void main(String[] args) throws Exception {
+ DrillConfig config = DrillConfig.create();
+
+ String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
+ StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class);
+ FunctionRegistry fr = new FunctionRegistry(config);
+ DrillSchemaFactory schemaFactory = new DrillSchemaFactory(engines, config);
+ DrillSqlWorker worker = new DrillSqlWorker(schemaFactory, fr);
+ worker.x();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/ExpandingConcurrentMap.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/ExpandingConcurrentMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/ExpandingConcurrentMap.java
new file mode 100644
index 0000000..e348bc3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/ExpandingConcurrentMap.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.collect.Maps;
+
+/**
+ * A special type of concurrent map which attempts to create an object before returning that it does not exist. It will also provide the same functionality on it's keyset.
+ * @param <KEY> The key in the map.
+ * @param <VALUE> The value in the map.
+ */
+public class ExpandingConcurrentMap<KEY, VALUE> implements ConcurrentMap<KEY, VALUE> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpandingConcurrentMap.class);
+
+ private final ConcurrentMap<KEY, VALUE> internalMap = Maps.newConcurrentMap();
+ private final DelegatingKeySet keySet = new DelegatingKeySet();
+ private final MapValueFactory<KEY, VALUE> fac;
+
+ /**
+ * Create a new ExpandingConcurrentMap.
+ * @param fac The object factory responsible for attempting to generate object instances.
+ */
+ public ExpandingConcurrentMap(MapValueFactory<KEY, VALUE> fac) {
+ super();
+ this.fac = fac;
+ }
+
+ @Override
+ public int size() {
+ return internalMap.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return internalMap.isEmpty();
+ }
+
+ @Override
+ public boolean containsKey(Object k) {
+ @SuppressWarnings("unchecked") KEY key = (KEY) k;
+
+ if(internalMap.containsKey(key)) return true;
+ VALUE v = getNewEntry(k);
+ return v != null;
+ }
+
+ @Override
+ public boolean containsValue(Object value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public VALUE get(Object key) {
+ VALUE out = internalMap.get(key);
+ if(out != null) return out;
+ return getNewEntry(key);
+ }
+
+ private VALUE getNewEntry(Object k){
+ @SuppressWarnings("unchecked")
+ KEY key = (KEY) k;
+ VALUE v = this.fac.create(key);
+ if(v == null) return null;
+ VALUE old = internalMap.putIfAbsent(key, v);
+ if(old == null) return v;
+ fac.destroy(v);
+ return old;
+ }
+
+ @Override
+ public VALUE put(KEY key, VALUE value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public VALUE remove(Object key) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void putAll(Map<? extends KEY, ? extends VALUE> m) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Set<KEY> keySet() {
+ return this.keySet;
+ }
+
+ @Override
+ public Collection<VALUE> values() {
+ return internalMap.values();
+ }
+
+ @Override
+ public Set<java.util.Map.Entry<KEY, VALUE>> entrySet() {
+ return internalMap.entrySet();
+ }
+
+ @Override
+ public VALUE putIfAbsent(KEY key, VALUE value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean remove(Object key, Object value) {
+ return false;
+ }
+
+ @Override
+ public boolean replace(KEY key, VALUE oldValue, VALUE newValue) {
+ return false;
+ }
+
+ @Override
+ public VALUE replace(KEY key, VALUE value) {
+ return null;
+ }
+
+ private class DelegatingKeySet implements Set<KEY>{
+
+ @Override
+ public int size() {
+ return ExpandingConcurrentMap.this.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return ExpandingConcurrentMap.this.isEmpty();
+ }
+
+ @Override
+ public boolean contains(Object o) {
+ return ExpandingConcurrentMap.this.containsKey(o);
+ }
+
+ @Override
+ public Iterator<KEY> iterator() {
+ return internalMap.keySet().iterator();
+ }
+
+ @Override
+ public Object[] toArray() {
+ return internalMap.keySet().toArray();
+ }
+
+ @Override
+ public <T> T[] toArray(T[] a) {
+ return internalMap.keySet().toArray(a);
+ }
+
+ @Override
+ public boolean add(KEY e) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean containsAll(Collection<?> c) {
+ for(Object o : c){
+ if(this.contains(o)) return false;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends KEY> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ public interface MapValueFactory<KEY, VALUE> {
+
+ public VALUE create(KEY key);
+ public void destroy(VALUE value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
new file mode 100644
index 0000000..ac29fc3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+
+import net.hydromatic.linq4j.expressions.DefaultExpression;
+import net.hydromatic.linq4j.expressions.Expression;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+import net.hydromatic.optiq.TableFunction;
+
+import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.store.SchemaProvider;
+
+public class FileSystemSchema implements Schema, ExpandingConcurrentMap.MapValueFactory<String, DrillTable>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchema.class);
+
+ private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<String, DrillTable>(this);
+
+ private final SchemaPlus parentSchema;
+ private final String name;
+ private final Expression expression = new DefaultExpression(Object.class);
+ private final SchemaProvider schemaProvider;
+ private final StorageEngineConfig config;
+
+ public FileSystemSchema(StorageEngineConfig config, SchemaProvider schemaProvider, SchemaPlus parentSchema, String name) {
+ super();
+ this.parentSchema = parentSchema;
+ this.name = name;
+ this.schemaProvider = schemaProvider;
+ this.config = config;
+ }
+
+ @Override
+ public Schema getSubSchema(String name) {
+ return null;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Expression getExpression() {
+ return expression;
+ }
+
+ @Override
+ public Collection<TableFunction> getTableFunctions(String name) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public SchemaPlus getParentSchema() {
+ return parentSchema;
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return tables.keySet();
+ }
+
+ @Override
+ public Set<String> getTableFunctionNames() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public boolean isMutable() {
+ return true;
+ }
+
+ @Override
+ public DrillTable getTable(String name) {
+ return tables.get(name);
+ }
+
+ @Override
+ public DrillTable create(String key) {
+ Object selection = schemaProvider.getSelectionBaseOnName(key);
+ if(selection == null) return null;
+
+ return new DrillTable(name, this.name, selection, config);
+ }
+
+ @Override
+ public void destroy(DrillTable value) {
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java
new file mode 100644
index 0000000..d4aabb4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.torel;
+
+import java.util.List;
+import java.util.Map;
+
+import net.hydromatic.optiq.prepare.Prepare;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.GroupingAggregate;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Union;
+import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
+import org.apache.drill.exec.planner.logical.DrillAggregateRel;
+import org.apache.drill.exec.planner.logical.DrillFilterRel;
+import org.apache.drill.exec.planner.logical.DrillJoinRel;
+import org.apache.drill.exec.planner.logical.DrillLimitRel;
+import org.apache.drill.exec.planner.logical.DrillProjectRel;
+import org.apache.drill.exec.planner.logical.DrillRel;
+import org.apache.drill.exec.planner.logical.DrillScanRel;
+import org.apache.drill.exec.planner.logical.DrillSortRel;
+import org.apache.drill.exec.planner.logical.DrillUnionRel;
+import org.apache.drill.exec.planner.logical.ScanFieldDeterminer;
+import org.apache.drill.exec.planner.logical.ScanFieldDeterminer.FieldList;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptTable;
+import org.eigenbase.relopt.RelOptTable.ToRelContext;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.rex.RexBuilder;
+import org.eigenbase.rex.RexNode;
+
+public class ConversionContext implements ToRelContext {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ConversionContext.class);
+
+ private static final ConverterVisitor VISITOR = new ConverterVisitor();
+
+ private final Map<Scan, FieldList> scanFieldLists;
+ private final RelOptCluster cluster;
+ private final Prepare prepare;
+
+ public ConversionContext(RelOptCluster cluster, LogicalPlan plan) {
+ super();
+ scanFieldLists = ScanFieldDeterminer.getFieldLists(plan);
+ this.cluster = cluster;
+ this.prepare = null;
+ }
+
+ @Override
+ public RelOptCluster getCluster() {
+ return cluster;
+ }
+
+
+ private FieldList getFieldList(Scan scan) {
+ assert scanFieldLists.containsKey(scan);
+ return scanFieldLists.get(scan);
+ }
+
+
+ public RexBuilder getRexBuilder(){
+ return cluster.getRexBuilder();
+ }
+
+ public RelTraitSet getLogicalTraits(){
+ RelTraitSet set = RelTraitSet.createEmpty();
+ set.add(DrillRel.CONVENTION);
+ return set;
+ }
+
+ public RelNode toRel(LogicalOperator operator) throws InvalidRelException{
+ return operator.accept(VISITOR, this);
+ }
+
+ public RexNode toRex(LogicalExpression e){
+ return null;
+ }
+
+ public RelDataTypeFactory getTypeFactory(){
+ return cluster.getTypeFactory();
+ }
+
+ public RelOptTable getTable(Scan scan){
+ FieldList list = getFieldList(scan);
+
+ return null;
+ }
+
+ @Override
+ public RelNode expandView(RelDataType rowType, String queryString, List<String> schemaPath) {
+ throw new UnsupportedOperationException();
+ }
+
+ private static class ConverterVisitor extends AbstractLogicalVisitor<RelNode, ConversionContext, InvalidRelException>{
+
+ @Override
+ public RelNode visitScan(Scan scan, ConversionContext context){
+ return DrillScanRel.convert(scan, context);
+ }
+
+ @Override
+ public RelNode visitFilter(Filter filter, ConversionContext context) throws InvalidRelException{
+ return DrillFilterRel.convert(filter, context);
+ }
+
+ @Override
+ public RelNode visitProject(Project project, ConversionContext context) throws InvalidRelException{
+ return DrillProjectRel.convert(project, context);
+ }
+
+ @Override
+ public RelNode visitOrder(Order order, ConversionContext context) throws InvalidRelException{
+ return DrillSortRel.convert(order, context);
+ }
+
+ @Override
+ public RelNode visitJoin(Join join, ConversionContext context) throws InvalidRelException{
+ return DrillJoinRel.convert(join, context);
+ }
+
+ @Override
+ public RelNode visitLimit(Limit limit, ConversionContext context) throws InvalidRelException{
+ return DrillLimitRel.convert(limit, context);
+ }
+
+ @Override
+ public RelNode visitUnion(Union union, ConversionContext context) throws InvalidRelException{
+ return DrillUnionRel.convert(union, context);
+ }
+
+ @Override
+ public RelNode visitGroupingAggregate(GroupingAggregate groupBy, ConversionContext context)
+ throws InvalidRelException {
+ return DrillAggregateRel.convert(groupBy, context);
+ }
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
index 991ade6..ae21a3c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
@@ -36,6 +36,15 @@ public class BatchSchema implements Iterable<MaterializedField> {
return new SchemaBuilder();
}
+ public int getFieldCount(){
+ return fields.size();
+ }
+
+ public MaterializedField getColumn(int index){
+ if(index < 0 || index >= fields.size()) return null;
+ return fields.get(index);
+ }
+
@Override
public Iterator<MaterializedField> iterator() {
return fields.iterator();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index 8d952e3..6e764a8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -138,17 +138,18 @@ public class VectorContainer implements Iterable<VectorWrapper<?>>, VectorAccess
@Override
- @SuppressWarnings("unchecked")
public VectorWrapper<?> getValueAccessorById(int fieldId, Class<?> clazz) {
VectorWrapper<?> va = wrappers.get(fieldId);
- assert va != null;
- if (va.getVectorClass() != clazz) {
+ if(va!= null && clazz == null){
+ return (VectorWrapper<?>) va;
+ }
+ if (va != null && va.getVectorClass() != clazz) {
logger.warn(String.format(
"Failure while reading vector. Expected vector class of %s but was holding vector class %s.",
clazz.getCanonicalName(), va.getVectorClass().getCanonicalName()));
return null;
}
- return (VectorWrapper) va;
+ return (VectorWrapper<?>) va;
}
public BatchSchema getSchema() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
index 4a43a99..1d9fecf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
@@ -104,7 +104,7 @@ public abstract class BasicClient<T extends EnumLite, R extends RemoteConnection
}
public <SEND extends MessageLite, RECEIVE extends MessageLite> DrillRpcFuture<RECEIVE> send(T rpcType,
- SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) throws RpcException {
+ SEND protobufBody, Class<RECEIVE> clazz, ByteBuf... dataBodies) {
return super.send(connection, rpcType, protobufBody, clazz, dataBodies);
}
@@ -201,6 +201,10 @@ public abstract class BasicClient<T extends EnumLite, R extends RemoteConnection
}
+ public void setAutoRead(boolean enableAutoRead){
+ connection.setAutoRead(enableAutoRead);
+ }
+
public void close() {
logger.debug("Closing client");
connection.getChannel().close();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
index a0b5cfd..c665949 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
@@ -25,6 +25,7 @@ import io.netty.util.concurrent.GenericFutureListener;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.rpc.BasicClientWithConnection.ServerConnection;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import com.google.protobuf.Internal.EnumLite;
import com.google.protobuf.MessageLite;
@@ -48,10 +49,10 @@ public abstract class BasicClientWithConnection<T extends EnumLite, HANDSHAKE_SE
@Override
protected Response handle(ServerConnection connection, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException {
- return handle(rpcType, pBody, dBody);
+ return handleReponse( (ConnectionThrottle) connection, rpcType, pBody, dBody);
}
- protected abstract Response handle(int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException ;
+ protected abstract Response handleReponse(ConnectionThrottle throttle, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException ;
@Override
public ServerConnection initRemoteConnection(Channel channel) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
index 0eaade8..a19f8d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
@@ -22,8 +22,9 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-public abstract class RemoteConnection{
+public abstract class RemoteConnection implements ConnectionThrottle{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemoteConnection.class);
private final Channel channel;
private final WriteManager writeManager;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/ConnectionThrottle.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/ConnectionThrottle.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/ConnectionThrottle.java
new file mode 100644
index 0000000..e2ffcc0
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/ConnectionThrottle.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc.user;
+
+public interface ConnectionThrottle {
+ public void setAutoRead(boolean enableAutoRead);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
index 5b4a504..5345b31 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/QueryResultHandler.java
@@ -51,7 +51,7 @@ public class QueryResultHandler {
return new SubmissionListener(listener);
}
- public void batchArrived(ByteBuf pBody, ByteBuf dBody) throws RpcException {
+ public void batchArrived(ConnectionThrottle throttle, ByteBuf pBody, ByteBuf dBody) throws RpcException {
final QueryResult result = RpcBus.get(pBody, QueryResult.PARSER);
final QueryResultBatch batch = new QueryResultBatch(result, dBody);
UserResultsListener l = resultsListener.get(result.getQueryId());
@@ -72,7 +72,7 @@ public class QueryResultHandler {
l.submissionFailed(new RpcException("Remote failure while running query." + batch.getHeader().getErrorList()));
resultsListener.remove(result.getQueryId(), l);
}else{
- l.resultArrived(batch);
+ l.resultArrived(batch, throttle);
}
if (
@@ -100,13 +100,13 @@ public class QueryResultHandler {
private volatile boolean finished = false;
private volatile RpcException ex;
private volatile UserResultsListener output;
-
+ private volatile ConnectionThrottle throttle;
public boolean transferTo(UserResultsListener l) {
synchronized (this) {
output = l;
boolean last = false;
for (QueryResultBatch r : results) {
- l.resultArrived(r);
+ l.resultArrived(r, throttle);
last = r.getHeader().getIsLastChunk();
}
if(ex != null){
@@ -119,14 +119,15 @@ public class QueryResultHandler {
@Override
- public void resultArrived(QueryResultBatch result) {
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
+ this.throttle = throttle;
if(result.getHeader().getIsLastChunk()) finished = true;
synchronized (this) {
if (output == null) {
this.results.add(result);
} else {
- output.resultArrived(result);
+ output.resultArrived(result, throttle);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
index f28ff4b..8b1bdec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
@@ -70,10 +70,10 @@ public class UserClient extends BasicClientWithConnection<RpcType, UserToBitHand
throw new RpcException(String.format("Unable to deal with RpcType of %d", rpcType));
}
- protected Response handle(int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException {
+ protected Response handleReponse(ConnectionThrottle throttle, int rpcType, ByteBuf pBody, ByteBuf dBody) throws RpcException {
switch (rpcType) {
case RpcType.QUERY_RESULT_VALUE:
- queryResultHandler.batchArrived(pBody, dBody);
+ queryResultHandler.batchArrived(throttle, pBody, dBody);
return new Response(RpcType.ACK, Ack.getDefaultInstance());
default:
throw new RpcException(String.format("Unknown Rpc Type %d. ", rpcType));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java
index 3bcd0cf..71f5994 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserResultsListener.java
@@ -24,6 +24,6 @@ public interface UserResultsListener {
public abstract void queryIdArrived(QueryId queryId);
public abstract void submissionFailed(RpcException ex);
- public abstract void resultArrived(QueryResultBatch result);
+ public abstract void resultArrived(QueryResultBatch result, ConnectionThrottle throttle);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
index 0f45dbe..aba9ab7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
@@ -94,7 +94,7 @@ public class UserServer extends BasicServer<RpcType, UserServer.UserClientConnec
}
default:
- throw new UnsupportedOperationException();
+ throw new UnsupportedOperationException(String.format("UserServer received rpc of unknown type. Type was %d.", rpcType));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 39821e3..60a7d5c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -23,12 +23,16 @@ import java.util.Collection;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FunctionRegistry;
import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.coord.ClusterCoordinator;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.planner.logical.StorageEngines;
+import org.apache.drill.exec.planner.sql.DrillSchemaFactory;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.rpc.control.Controller;
import org.apache.drill.exec.rpc.control.WorkEventBus;
@@ -37,7 +41,9 @@ import org.apache.drill.exec.store.StorageEngine;
import org.apache.drill.exec.store.StorageEngineRegistry;
import com.codahale.metrics.MetricRegistry;
+import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
+import com.google.common.io.Resources;
public class DrillbitContext {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillbitContext.class);
@@ -53,6 +59,9 @@ public class DrillbitContext {
private final OperatorCreatorRegistry operatorCreatorRegistry;
private final Controller controller;
private final WorkEventBus workBus;
+ private final FunctionImplementationRegistry functionRegistry;
+ private final FunctionRegistry functionRegistryX;
+ private final DrillSchemaFactory factory;
public DrillbitContext(DrillbitEndpoint endpoint, BootStrapContext context, ClusterCoordinator coord, Controller controller, DataConnectionCreator connectionsPool, DistributedCache cache, WorkEventBus workBus) {
super();
@@ -70,8 +79,34 @@ public class DrillbitContext {
this.storageEngineRegistry = new StorageEngineRegistry(this);
this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storageEngineRegistry);
this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig());
+ this.functionRegistry = new FunctionImplementationRegistry(context.getConfig());
+
+ DrillSchemaFactory factory = null;
+ try{
+ String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
+ StorageEngines engines = context.getConfig().getMapper().readValue(enginesData, StorageEngines.class);
+ factory = new DrillSchemaFactory(engines, context.getConfig());
+ }catch(Exception e){
+ logger.error("Failure reading storage engines data. Creating empty list of schemas.", e);
+ factory = DrillSchemaFactory.createEmpty();
+ }
+ this.factory = factory;
+ this.functionRegistryX = new FunctionRegistry(context.getConfig());
+
}
+ public DrillSchemaFactory getSchemaFactory(){
+ return factory;
+ }
+
+ public FunctionRegistry getFunctionRegistry(){
+ return functionRegistryX;
+ }
+
+ public FunctionImplementationRegistry getFunctionImplementationRegistry() {
+ return functionRegistry;
+ }
+
public WorkEventBus getWorkBus(){
return workBus;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 1756d96..cb05273 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -116,7 +116,7 @@ public class ParquetGroupScan extends AbstractGroupScan {
calculateEndpointBytes();
}
- public ParquetGroupScan(ArrayList<ReadEntryWithPath> entries,
+ public ParquetGroupScan(List<ReadEntryWithPath> entries,
ParquetStorageEngine storageEngine, FieldReference ref) throws IOException {
this.storageEngine = storageEngine;
this.engineConfig = storageEngine.getEngineConfig();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
index 86be49e..c17a9e3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
@@ -52,7 +52,7 @@ public class ParquetSchemaProvider implements SchemaProvider{
@Override
public Object getSelectionBaseOnName(String tableName) {
try{
-// if(!fs.exists(new Path(tableName))) return null;
+ if(!fs.exists(new Path(tableName))) return null;
ReadEntryWithPath re = new ReadEntryWithPath(tableName);
return Lists.newArrayList(re);
}catch(Exception e){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
index e04d3e9..45b9cc1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
@@ -99,7 +99,7 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
ArrayList<ReadEntryWithPath> readEntries = scan.getSelection().getListWith(new ObjectMapper(),
new TypeReference<ArrayList<ReadEntryWithPath>>() {});
- return new ParquetGroupScan(readEntries, this, scan.getOutputReference());
+ return new ParquetGroupScan( Lists.newArrayList(readEntries), this, scan.getOutputReference());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index cbc8b86..4f8cd33 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -167,6 +167,10 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
return ((data.getByte((int) Math.floor(index / 8)) & (int) Math.pow(2, (index % 8))) == 0) ? 0 : 1;
}
+ public boolean isNull(int index){
+ return false;
+ }
+
@Override
public final Object getObject(int index) {
return new Boolean(get(index) != 0);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 4e5364a..a3d3a8a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -125,6 +125,8 @@ public interface ValueVector extends Closeable {
public abstract Object getObject(int index);
public int getValueCount();
+
+ public boolean isNull(int index);
public void reset();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java
new file mode 100644
index 0000000..93089e7
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.SQLException;
+import java.sql.Time;
+import java.sql.Timestamp;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+abstract class AbstractSqlAccessor implements SqlAccessor {
+
+ @Override
+ public abstract boolean isNull(int index);
+
+ @Override
+ public BigDecimal getBigDecimal(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("BigDecimal");
+ }
+
+ @Override
+ public boolean getBoolean(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("boolean");
+ }
+
+ @Override
+ public byte getByte(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("byte");
+ }
+
+ @Override
+ public byte[] getBytes(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("byte[]");
+ }
+
+ @Override
+ public Date getDate(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("Date");
+ }
+
+ @Override
+ public double getDouble(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("double");
+ }
+
+ @Override
+ public float getFloat(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("float");
+ }
+
+ @Override
+ public int getInt(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("int");
+ }
+
+ @Override
+ public long getLong(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("long");
+ }
+
+ @Override
+ public short getShort(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("short");
+ }
+
+ @Override
+ public InputStream getStream(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("InputStream");
+ }
+
+ @Override
+ public char getChar(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("Char");
+ }
+
+ @Override
+ public Reader getReader(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("Reader");
+ }
+
+ @Override
+ public String getString(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("String");
+ }
+
+ @Override
+ public Time getTime(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("Time");
+ }
+
+ @Override
+ public Timestamp getTimestamp(int index) throws InvalidAccessException{
+ throw new InvalidAccessException("Timestamp");
+ }
+
+ abstract MajorType getType();
+
+
+ public class InvalidAccessException extends SQLException{
+ public InvalidAccessException(String name){
+ super(String.format("Requesting class of type %s for an object of type %s:%s is not allowed.", name, getType().getMinorType().name(), getType().getMode().name()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java
new file mode 100644
index 0000000..5611354
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+
+import org.apache.drill.exec.vector.accessor.AbstractSqlAccessor.InvalidAccessException;
+
+public interface SqlAccessor {
+
+ public abstract boolean isNull(int index);
+
+ public abstract BigDecimal getBigDecimal(int index) throws InvalidAccessException;
+
+ public abstract boolean getBoolean(int index) throws InvalidAccessException;
+
+ public abstract byte getByte(int index) throws InvalidAccessException;
+
+ public abstract byte[] getBytes(int index) throws InvalidAccessException;
+
+ public abstract Date getDate(int index) throws InvalidAccessException;
+
+ public abstract double getDouble(int index) throws InvalidAccessException;
+
+ public abstract float getFloat(int index) throws InvalidAccessException;
+
+ public abstract char getChar(int index) throws InvalidAccessException;
+
+ public abstract int getInt(int index) throws InvalidAccessException;
+
+ public abstract long getLong(int index) throws InvalidAccessException;
+
+ public abstract short getShort(int index) throws InvalidAccessException;
+
+ public abstract InputStream getStream(int index) throws InvalidAccessException;
+
+ public abstract Reader getReader(int index) throws InvalidAccessException;
+
+ public abstract String getString(int index) throws InvalidAccessException;
+
+ public abstract Time getTime(int index) throws InvalidAccessException;
+
+ public abstract Timestamp getTimestamp(int index) throws InvalidAccessException;
+
+ public abstract Object getObject(int index) throws InvalidAccessException;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
index 29f011f..b6d441c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
@@ -119,6 +119,7 @@ public class WorkManager implements Closeable{
public class WorkerBee{
public void addFragmentRunner(FragmentExecutor runner){
+ logger.debug("Adding pending task {}", runner);
pendingTasks.add(runner);
}
@@ -167,7 +168,7 @@ public class WorkManager implements Closeable{
try {
while(true){
// logger.debug("Polling for pending work tasks.");
- Runnable r = pendingTasks.poll(10, TimeUnit.SECONDS);
+ Runnable r = pendingTasks.take();
if(r != null){
logger.debug("Starting pending task {}", r);
executor.execute(r);
[25/38] Integrate new SQL changes with Hive storage engine,
moving to automatic file detection. Rename storage engine to storage
plugin. Separate storage plugins from format plugins,
updating Parquet and JSON to format engines. Refactor distribution log
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
new file mode 100644
index 0000000..5b8ccff
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+
+import net.hydromatic.linq4j.expressions.DefaultExpression;
+import net.hydromatic.linq4j.expressions.Expression;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+import net.hydromatic.optiq.TableFunction;
+
+import org.apache.drill.exec.planner.logical.DrillTable;
+
+public abstract class AbstractSchema implements Schema{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSchema.class);
+
+ private final SchemaHolder parentSchema;
+
+ protected final String name;
+ private static final Expression EXPRESSION = new DefaultExpression(Object.class);
+
+ public AbstractSchema(SchemaHolder parentSchema, String name) {
+ super();
+ this.parentSchema = parentSchema;
+ this.name = name;
+ }
+
+
+ @Override
+ public SchemaPlus getParentSchema() {
+ return parentSchema.getSchema();
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Collection<TableFunction> getTableFunctions(String name) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Set<String> getTableFunctionNames() {
+ return Collections.emptySet();
+ }
+
+
+
+ @Override
+ public Schema getSubSchema(String name) {
+ return null;
+ }
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Expression getExpression() {
+ return EXPRESSION;
+ }
+
+ @Override
+ public boolean isMutable() {
+ return false;
+ }
+
+ @Override
+ public DrillTable getTable(String name){
+ return null;
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return Collections.emptySet();
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStorageEngine.java
deleted file mode 100644
index 77fad2a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStorageEngine.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.base.AbstractGroupScan;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Multimap;
-
-public abstract class AbstractStorageEngine implements StorageEngine{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractStorageEngine.class);
-
- protected AbstractStorageEngine(){
- }
-
-
- @Override
- public boolean supportsRead() {
- return false;
- }
-
- @Override
- public boolean supportsWrite() {
- return false;
- }
-
- @Override
- public List<QueryOptimizerRule> getOptimizerRules() {
- return Collections.emptyList();
- }
-
- @Override
- public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ListMultimap<ReadEntry, DrillbitEndpoint> getReadLocations(Collection<ReadEntry> entries) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RecordReader getReader(FragmentContext context, ReadEntry readEntry) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RecordRecorder getWriter(FragmentContext context, WriteEntry writeEntry) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Multimap<DrillbitEndpoint, ReadEntry> getEntryAssignments(List<DrillbitEndpoint> assignments,
- Collection<ReadEntry> entries) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Multimap<DrillbitEndpoint, WriteEntry> getWriteAssignments(List<DrillbitEndpoint> assignments,
- Collection<ReadEntry> entries) {
- throw new UnsupportedOperationException();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java
new file mode 100644
index 0000000..8baa72a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
+
+public abstract class AbstractStoragePlugin implements StoragePlugin{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractStoragePlugin.class);
+
+ protected AbstractStoragePlugin(){
+ }
+
+ @Override
+ public boolean supportsRead() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsWrite() {
+ return false;
+ }
+
+ @Override
+ public List<QueryOptimizerRule> getOptimizerRules() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
deleted file mode 100644
index 7061c58..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableRangeMap;
-import com.google.common.collect.Range;
-import com.codahale.metrics.*;
-import com.codahale.metrics.Timer;
-import org.apache.drill.exec.metrics.DrillMetrics;
-import org.apache.drill.exec.store.parquet.ParquetGroupScan;
-
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-public class AffinityCalculator {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AffinityCalculator.class);
- static final MetricRegistry metrics = DrillMetrics.getInstance();
- static final String BLOCK_MAP_BUILDER_TIMER = MetricRegistry.name(AffinityCalculator.class, "blockMapBuilderTimer");
-
-
- HashMap<String,ImmutableRangeMap<Long,BlockLocation>> blockMapMap = new HashMap<>();
- FileSystem fs;
- String fileName;
- Collection<DrillbitEndpoint> endpoints;
- HashMap<String,DrillbitEndpoint> endPointMap;
-
- public AffinityCalculator(FileSystem fs, Collection<DrillbitEndpoint> endpoints) {
- this.fs = fs;
- this.endpoints = endpoints;
- buildEndpointMap();
- }
-
- /**
- * Builds a mapping of block locations to file byte range
- */
- private void buildBlockMap(String fileName) {
- final Timer.Context context = metrics.timer(BLOCK_MAP_BUILDER_TIMER).time();
- BlockLocation[] blocks;
- ImmutableRangeMap<Long,BlockLocation> blockMap;
- try {
- FileStatus file = fs.getFileStatus(new Path(fileName));
- blocks = fs.getFileBlockLocations(file, 0 , file.getLen());
- } catch (IOException ioe) { throw new RuntimeException(ioe); }
- ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long,BlockLocation>();
- for (BlockLocation block : blocks) {
- long start = block.getOffset();
- long end = start + block.getLength();
- Range<Long> range = Range.closedOpen(start, end);
- blockMapBuilder = blockMapBuilder.put(range, block);
- }
- blockMap = blockMapBuilder.build();
- blockMapMap.put(fileName, blockMap);
- context.stop();
- }
- /**
- * For a given RowGroup, calculate how many bytes are available on each on drillbit endpoint
- *
- * @param rowGroup the RowGroup to calculate endpoint bytes for
- */
- public void setEndpointBytes(ParquetGroupScan.RowGroupInfo rowGroup) {
- Stopwatch watch = new Stopwatch();
- watch.start();
- String fileName = rowGroup.getPath();
- if (!blockMapMap.containsKey(fileName)) {
- buildBlockMap(fileName);
- }
-
- ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(fileName);
- HashMap<String,Long> hostMap = new HashMap<>();
- HashMap<DrillbitEndpoint,Long> endpointByteMap = new HashMap();
- long start = rowGroup.getStart();
- long end = start + rowGroup.getLength();
- Range<Long> rowGroupRange = Range.closedOpen(start, end);
-
- // Find submap of ranges that intersect with the rowGroup
- ImmutableRangeMap<Long,BlockLocation> subRangeMap = blockMap.subRangeMap(rowGroupRange);
-
- // Iterate through each block in this submap and get the host for the block location
- for (Map.Entry<Range<Long>,BlockLocation> block : subRangeMap.asMapOfRanges().entrySet()) {
- String[] hosts;
- Range<Long> blockRange = block.getKey();
- try {
- hosts = block.getValue().getHosts();
- } catch (IOException ioe) {
- throw new RuntimeException("Failed to get hosts for block location", ioe);
- }
- Range<Long> intersection = rowGroupRange.intersection(blockRange);
- long bytes = intersection.upperEndpoint() - intersection.lowerEndpoint();
-
- // For each host in the current block location, add the intersecting bytes to the corresponding endpoint
- for (String host : hosts) {
- DrillbitEndpoint endpoint = getDrillBitEndpoint(host);
- if (endpointByteMap.containsKey(endpoint)) {
- endpointByteMap.put(endpoint, endpointByteMap.get(endpoint) + bytes);
- } else {
- if (endpoint != null ) endpointByteMap.put(endpoint, bytes);
- }
- }
- }
-
- rowGroup.setEndpointBytes(endpointByteMap);
- rowGroup.setMaxBytes(endpointByteMap.size() > 0 ? Collections.max(endpointByteMap.values()) : 0);
- logger.debug("Row group ({},{}) max bytes {}", rowGroup.getPath(), rowGroup.getStart(), rowGroup.getMaxBytes());
- watch.stop();
- logger.debug("Took {} ms to set endpoint bytes", watch.elapsed(TimeUnit.MILLISECONDS));
- }
-
- private DrillbitEndpoint getDrillBitEndpoint(String hostName) {
- return endPointMap.get(hostName);
- }
-
- /**
- * Builds a mapping of drillbit endpoints to hostnames
- */
- private void buildEndpointMap() {
- Stopwatch watch = new Stopwatch();
- watch.start();
- endPointMap = new HashMap<String, DrillbitEndpoint>();
- for (DrillbitEndpoint d : endpoints) {
- String hostName = d.getAddress();
- endPointMap.put(hostName, d);
- }
- watch.stop();
- logger.debug("Took {} ms to build endpoint map", watch.elapsed(TimeUnit.MILLISECONDS));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
index 5ca575b..e39e1a3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
@@ -61,13 +61,27 @@ public class ClassPathFileSystem extends FileSystem{
throw new IOException(ERROR_MSG);
}
+ private String getFileName(Path path){
+ String file = path.toString();
+ if(file.charAt(0) == '/'){
+ file = file.substring(1);
+ }
+ return file;
+ }
+
@Override
public FileStatus getFileStatus(Path arg0) throws IOException {
- URL url = Resources.getResource(arg0.toString());
+ String file = getFileName(arg0);
+
+ try{
+ URL url = Resources.getResource(file);
if(url == null){
throw new IOException(String.format("Unable to find path %s.", arg0.toString()));
}
- return new FileStatus(-1, false, 1, 8096, System.currentTimeMillis(), arg0);
+ return new FileStatus(1, false, 1, 8096, System.currentTimeMillis(), arg0);
+ }catch(RuntimeException e){
+ throw new IOException(String.format("Failure trying to load file %s", arg0), e);
+ }
}
@Override
@@ -96,7 +110,8 @@ public class ClassPathFileSystem extends FileSystem{
@Override
public FSDataInputStream open(Path arg0, int arg1) throws IOException {
- URL url = Resources.getResource(arg0.toString());
+ String file = getFileName(arg0);
+ URL url = Resources.getResource(file);
if(url == null){
throw new IOException(String.format("Unable to find path %s.", arg0.getName()));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/NamedStoragePluginConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/NamedStoragePluginConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/NamedStoragePluginConfig.java
new file mode 100644
index 0000000..b67c019
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/NamedStoragePluginConfig.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import org.apache.drill.common.logical.StoragePluginConfig;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("named")
+public class NamedStoragePluginConfig implements StoragePluginConfig{
+ public String name;
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java
new file mode 100644
index 0000000..a8d053c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
+public interface SchemaFactory {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaFactory.class);
+
+ public Schema add(SchemaPlus parent);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaHolder.java
new file mode 100644
index 0000000..faaca2a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaHolder.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import net.hydromatic.optiq.SchemaPlus;
+
+/**
+ * Helper class to provide parent schema after initialization given Optiq's backwards schema build model.
+ */
+public class SchemaHolder {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaHolder.class);
+
+ private SchemaPlus schema;
+
+ public SchemaHolder(){}
+
+ public SchemaHolder(SchemaPlus schema){
+ this.schema = schema;
+ }
+
+ public SchemaPlus getSchema() {
+ return schema;
+ }
+
+ public void setSchema(SchemaPlus schema) {
+ this.schema = schema;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java
deleted file mode 100644
index 7611461..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-public interface SchemaProvider {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaProvider.class);
-
- public Object getSelectionBaseOnName(String tableName);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java
deleted file mode 100644
index 3ae6943..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.util.PathScanner;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.exception.SetupException;
-
-public class SchemaProviderRegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaProviderRegistry.class);
-
- private Map<Object, Constructor<? extends SchemaProvider>> allProviders = new HashMap<Object, Constructor<? extends SchemaProvider>>();
- private Map<StorageEngineConfig, SchemaProvider> activeEngines = new HashMap<StorageEngineConfig, SchemaProvider>();
-
- private final DrillConfig config;
-
- public SchemaProviderRegistry(DrillConfig config){
- init(config);
- this.config = config;
- }
-
- @SuppressWarnings("unchecked")
- public void init(DrillConfig config){
- Collection<Class<? extends SchemaProvider>> providers = PathScanner.scanForImplementations(SchemaProvider.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES));
- logger.debug("Loading schema providers {}", providers);
- for(Class<? extends SchemaProvider> schema: providers){
- int i =0;
- for(Constructor<?> c : schema.getConstructors()){
- Class<?>[] params = c.getParameterTypes();
- if(params.length != 2 || params[1] != DrillConfig.class || !StorageEngineConfig.class.isAssignableFrom(params[0])){
- logger.info("Skipping SchemaProvider constructor {} for provider class {} since it doesn't implement a [constructor(StorageEngineConfig, DrillConfig)]", c, schema);
- continue;
- }
- allProviders.put(params[0], (Constructor<? extends SchemaProvider>) c);
- i++;
- }
- if(i == 0){
- logger.debug("Skipping registration of StorageSchemaProvider {} as it doesn't have a constructor with the parameters of (StorangeEngineConfig, Config)", schema.getName());
- }
- }
- }
-
- public SchemaProvider getSchemaProvider(StorageEngineConfig engineConfig) throws SetupException{
- SchemaProvider engine = activeEngines.get(engineConfig);
- if(engine != null) return engine;
- Constructor<? extends SchemaProvider> c = allProviders.get(engineConfig.getClass());
- if(c == null) throw new SetupException(String.format("Failure finding StorageSchemaProvider constructor for config %s", engineConfig));
- try {
- return c.newInstance(engineConfig, config);
- } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
- Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException)e).getTargetException() : e;
- if(t instanceof SetupException) throw ((SetupException) t);
- throw new SetupException(String.format("Failure setting up new storage engine configuration for config %s", engineConfig), t);
- }
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java
deleted file mode 100644
index 00574e6..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.base.AbstractGroupScan;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Multimap;
-
-public interface StorageEngine {
- public boolean supportsRead();
-
- public boolean supportsWrite();
-
- public enum PartitionCapabilities {
- NONE, HASH, RANGE;
- }
-
- public List<QueryOptimizerRule> getOptimizerRules();
-
- /**
- * Get the physical scan operator populated with a set of read entries required for the particular GroupScan (read) node.
- * This is somewhat analogous to traditional MapReduce. The difference is, this is the most granular split paradigm.
- *
- * @param scan
- * The configured scan entries.
- * @return
- * @throws IOException
- */
- public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException;
-
- public SchemaProvider getSchemaProvider();
-
-
- /**
- * Get the set of Drillbit endpoints that are available for each read entry. Note that it is possible for a read entry
- * to have no Drillbit locations. In that case, the multimap will contain no values for that read entry.
- *
- * @return Multimap of ReadEntry > List<DrillbitEndpoint> for ReadEntries with available locations.
- */
- public ListMultimap<ReadEntry, DrillbitEndpoint> getReadLocations(Collection<ReadEntry> entries);
-
- /**
- * Apply read entry assignments based on the list of actually assigned Endpoints. A storage engine is allowed to
- * update or modify the read entries based on the nature of the assignments. For example, if two blocks are initially
- * considered separate read entries but then the storage engine realizes that the assignments for those two reads are
- * on the same system, the storage engine may decide to collapse those entries into a single read entry that covers
- * both original read entries.
- *
- * @param assignments
- * @param entries
- * @return
- */
- public Multimap<DrillbitEndpoint, ReadEntry> getEntryAssignments(List<DrillbitEndpoint> assignments,
- Collection<ReadEntry> entries);
-
- /**
- * Get a particular reader for a fragment context.
- *
- * @param context
- * @param readEntry
- * @return
- * @throws IOException
- */
- public RecordReader getReader(FragmentContext context, ReadEntry readEntry) throws IOException;
-
- /**
- * Apply write entry assignments based on the list of actually assigned endpoints. A storage engine is allowed to
- * rewrite the WriteEntries if desired based on the nature of the assignments. For example, a storage engine could
- * hold off actually determining the specific level of partitioning required until it finds out exactly the number of
- * nodes associated with the operation.
- *
- * @param assignments
- * @param entries
- * @return
- */
- public Multimap<DrillbitEndpoint, WriteEntry> getWriteAssignments(List<DrillbitEndpoint> assignments,
- Collection<ReadEntry> entries);
-
- /**
- *
- * @param context
- * @param writeEntry
- * @return
- * @throws IOException
- */
- public RecordRecorder getWriter(FragmentContext context, WriteEntry writeEntry) throws IOException;
-
- public interface WriteEntry {
- }
-
- public static class Cost {
- public long disk;
- public long network;
- public long memory;
- public long cpu;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java
deleted file mode 100644
index 4cc7346..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.util.PathScanner;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.server.DrillbitContext;
-
-public class StorageEngineRegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StorageEngineRegistry.class);
-
- private Map<Object, Constructor<? extends StorageEngine>> availableEngines = new HashMap<Object, Constructor<? extends StorageEngine>>();
- private Map<StorageEngineConfig, StorageEngine> activeEngines = new HashMap<StorageEngineConfig, StorageEngine>();
-
- private DrillbitContext context;
- public StorageEngineRegistry(DrillbitContext context){
- init(context.getConfig());
- this.context = context;
- }
-
- @SuppressWarnings("unchecked")
- public void init(DrillConfig config){
- Collection<Class<? extends StorageEngine>> engines = PathScanner.scanForImplementations(StorageEngine.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES));
- logger.debug("Loading storage engines {}", engines);
- for(Class<? extends StorageEngine> engine: engines){
- int i =0;
- for(Constructor<?> c : engine.getConstructors()){
- Class<?>[] params = c.getParameterTypes();
- if(params.length != 2 || params[1] != DrillbitContext.class || !StorageEngineConfig.class.isAssignableFrom(params[0])){
- logger.info("Skipping StorageEngine constructor {} for engine class {} since it doesn't implement a [constructor(StorageEngineConfig, DrillbitContext)]", c, engine);
- continue;
- }
- availableEngines.put(params[0], (Constructor<? extends StorageEngine>) c);
- i++;
- }
- if(i == 0){
- logger.debug("Skipping registration of StorageEngine {} as it doesn't have a constructor with the parameters of (StorangeEngineConfig, Config)", engine.getCanonicalName());
- }
- }
- }
-
- public synchronized StorageEngine getEngine(StorageEngineConfig engineConfig) throws ExecutionSetupException{
- StorageEngine engine = activeEngines.get(engineConfig);
- if(engine != null) return engine;
- Constructor<? extends StorageEngine> c = availableEngines.get(engineConfig.getClass());
- if(c == null) throw new ExecutionSetupException(String.format("Failure finding StorageEngine constructor for config %s", engineConfig));
- try {
- engine = c.newInstance(engineConfig, context);
- activeEngines.put(engineConfig, engine);
- return engine;
- } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
- Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException)e).getTargetException() : e;
- if(t instanceof ExecutionSetupException) throw ((ExecutionSetupException) t);
- throw new ExecutionSetupException(String.format("Failure setting up new storage engine configuration for config %s", engineConfig), t);
- }
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java
new file mode 100644
index 0000000..2e54b0d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import java.io.IOException;
+import java.util.List;
+
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
+
+public interface StoragePlugin {
+ public boolean supportsRead();
+
+ public boolean supportsWrite();
+
+ public List<QueryOptimizerRule> getOptimizerRules();
+
+ /**
+ * Get the physical scan operator for the particular GroupScan (read) node.
+ *
+ * @param scan
+ * The configured scan with a storage engine specific selection.
+ * @return
+ * @throws IOException
+ */
+ public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException;
+
+ public Schema createAndAddSchema(SchemaPlus parent);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
new file mode 100644
index 0000000..b182abe
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import net.hydromatic.linq4j.function.Function1;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.planner.logical.StorageEngines;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.dfs.FileSystemPlugin;
+import org.apache.drill.exec.store.dfs.FormatPlugin;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Resources;
+
+public class StoragePluginRegistry implements Iterable<Map.Entry<String, StoragePlugin>>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginRegistry.class);
+
+ private Map<Object, Constructor<? extends StoragePlugin>> availableEngines = new HashMap<Object, Constructor<? extends StoragePlugin>>();
+ private final ImmutableMap<String, StoragePlugin> engines;
+
+ private DrillbitContext context;
+ private final DrillSchemaFactory schemaFactory = new DrillSchemaFactory();
+
+ public StoragePluginRegistry(DrillbitContext context) {
+ try{
+ this.context = context;
+ init(context.getConfig());
+ this.engines = ImmutableMap.copyOf(createEngines());
+ }catch(RuntimeException e){
+ logger.error("Failure while loading storage engine registry.", e);
+ throw new RuntimeException("Faiure while reading and loading storage plugin configuration.", e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void init(DrillConfig config){
+ Collection<Class<? extends StoragePlugin>> engines = PathScanner.scanForImplementations(StoragePlugin.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES));
+ logger.debug("Loading storage engines {}", engines);
+ for(Class<? extends StoragePlugin> engine: engines){
+ int i =0;
+ for(Constructor<?> c : engine.getConstructors()){
+ Class<?>[] params = c.getParameterTypes();
+ if(params.length != 3 || params[1] != DrillbitContext.class || !StoragePluginConfig.class.isAssignableFrom(params[0]) || params[2] != String.class){
+ logger.info("Skipping StorageEngine constructor {} for engine class {} since it doesn't implement a [constructor(StorageEngineConfig, DrillbitContext, String)]", c, engine);
+ continue;
+ }
+ availableEngines.put(params[0], (Constructor<? extends StoragePlugin>) c);
+ i++;
+ }
+ if(i == 0){
+ logger.debug("Skipping registration of StorageEngine {} as it doesn't have a constructor with the parameters of (StorangeEngineConfig, Config)", engine.getCanonicalName());
+ }
+ }
+
+ }
+
+ private Map<String, StoragePlugin> createEngines(){
+ StorageEngines engines = null;
+ Map<String, StoragePlugin> activeEngines = new HashMap<String, StoragePlugin>();
+ try{
+ String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
+ engines = context.getConfig().getMapper().readValue(enginesData, StorageEngines.class);
+ }catch(IOException e){
+ throw new IllegalStateException("Failure while reading storage engines data.", e);
+ }
+
+ for(Map.Entry<String, StoragePluginConfig> config : engines){
+ try{
+ StoragePlugin plugin = create(config.getKey(), config.getValue());
+ activeEngines.put(config.getKey(), plugin);
+ }catch(ExecutionSetupException e){
+ logger.error("Failure while setting up StoragePlugin with name: '{}'.", config.getKey(), e);
+ }
+ }
+ return activeEngines;
+ }
+
+ public StoragePlugin getEngine(String registeredStorageEngineName) throws ExecutionSetupException {
+ return engines.get(registeredStorageEngineName);
+ }
+
+ public StoragePlugin getEngine(StoragePluginConfig config) throws ExecutionSetupException {
+ if(config instanceof NamedStoragePluginConfig){
+ return engines.get(((NamedStoragePluginConfig) config).name);
+ }else{
+ // TODO: for now, we'll throw away transient configs. we really ought to clean these up.
+ return create(null, config);
+ }
+ }
+
+ public FormatPlugin getFormatPlugin(StoragePluginConfig storageConfig, FormatPluginConfig formatConfig) throws ExecutionSetupException{
+ StoragePlugin p = getEngine(storageConfig);
+ if(!(p instanceof FileSystemPlugin)) throw new ExecutionSetupException(String.format("You tried to request a format plugin for a stroage engine that wasn't of type FileSystemPlugin. The actual type of plugin was %s.", p.getClass().getName()));
+ FileSystemPlugin storage = (FileSystemPlugin) p;
+ return storage.getFormatPlugin(formatConfig);
+ }
+
+ private StoragePlugin create(String name, StoragePluginConfig engineConfig) throws ExecutionSetupException {
+ StoragePlugin engine = null;
+ Constructor<? extends StoragePlugin> c = availableEngines.get(engineConfig.getClass());
+ if (c == null)
+ throw new ExecutionSetupException(String.format("Failure finding StorageEngine constructor for config %s",
+ engineConfig));
+ try {
+ engine = c.newInstance(engineConfig, context, name);
+ return engine;
+ } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+ Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException) e).getTargetException() : e;
+ if (t instanceof ExecutionSetupException)
+ throw ((ExecutionSetupException) t);
+ throw new ExecutionSetupException(String.format(
+ "Failure setting up new storage engine configuration for config %s", engineConfig), t);
+ }
+ }
+
+ @Override
+ public Iterator<Entry<String, StoragePlugin>> iterator() {
+ return engines.entrySet().iterator();
+ }
+
+ public DrillSchemaFactory getSchemaFactory(){
+ return schemaFactory;
+ }
+
+ public class DrillSchemaFactory implements Function1<SchemaPlus, Schema>{
+
+ @Override
+ public Schema apply(SchemaPlus parent) {
+ Schema defaultSchema = null;
+ for(Map.Entry<String, StoragePlugin> e : engines.entrySet()){
+ Schema s = e.getValue().createAndAddSchema(parent);
+ if(defaultSchema == null) defaultSchema = s;
+ }
+ return defaultSchema;
+ }
+
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
new file mode 100644
index 0000000..1c391de
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+
+import com.beust.jcommander.internal.Lists;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Range;
+
+public class BasicFormatMatcher extends FormatMatcher{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicFormatMatcher.class);
+
+ private final List<Pattern> patterns;
+ private final MagicStringMatcher matcher;
+ protected final DrillFileSystem fs;
+ protected final FormatPlugin plugin;
+
+ public BasicFormatMatcher(FormatPlugin plugin, DrillFileSystem fs, List<Pattern> patterns, List<MagicString> magicStrings) {
+ super();
+ this.patterns = ImmutableList.copyOf(patterns);
+ this.matcher = new MagicStringMatcher(magicStrings);
+ this.fs = fs;
+ this.plugin = plugin;
+ }
+
+ public BasicFormatMatcher(FormatPlugin plugin, DrillFileSystem fs, String extension){
+ this(plugin, fs, //
+ Lists.newArrayList(Pattern.compile(".*\\." + extension)), //
+ (List<MagicString>) Collections.EMPTY_LIST);
+ }
+
+ @Override
+ public boolean supportDirectoryReads() {
+ return false;
+ }
+
+ @Override
+ public FormatSelection isReadable(FileSelection file) throws IOException {
+ if(isReadable(file.getFirstPath(fs))){
+ return new FormatSelection(plugin.getConfig(), file);
+ }
+ return null;
+ }
+
+ protected final boolean isReadable(FileStatus status) throws IOException {
+ for(Pattern p : patterns){
+ if(p.matcher(status.getPath().toString()).matches()){
+ return true;
+ }
+ }
+
+ if(matcher.matches(status)) return true;
+ return false;
+ }
+
+
+ @Override
+ @JsonIgnore
+ public FormatPlugin getFormatPlugin() {
+ return plugin;
+ }
+
+
+ private class MagicStringMatcher{
+
+ private List<RangeMagics> ranges;
+
+ public MagicStringMatcher(List<MagicString> magicStrings){
+ ranges = Lists.newArrayList();
+ for(MagicString ms : magicStrings){
+ ranges.add(new RangeMagics(ms));
+ }
+ }
+
+ public boolean matches(FileStatus status) throws IOException{
+ if(ranges.isEmpty()) return false;
+ final Range<Long> fileRange = Range.closedOpen( 0L, status.getLen());
+
+ try(FSDataInputStream is = fs.open(status.getPath()).getInputStream()){
+ for(RangeMagics rMagic : ranges){
+ Range<Long> r = rMagic.range;
+ if(!fileRange.encloses(r)) continue;
+ int len = (int) (r.upperEndpoint() - r.lowerEndpoint());
+ byte[] bytes = new byte[len];
+ is.readFully(r.lowerEndpoint(), bytes);
+ for(byte[] magic : rMagic.magics){
+ if(Arrays.equals(magic, bytes)) return true;
+ }
+
+ }
+ }
+ return false;
+ }
+
+ private class RangeMagics{
+ Range<Long> range;
+ byte[][] magics;
+
+ public RangeMagics(MagicString ms){
+ this.range = Range.closedOpen( ms.getOffset(), (long) ms.getBytes().length);
+ this.magics = new byte[][]{ms.getBytes()};
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
new file mode 100644
index 0000000..5ab2c1a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.collect.Lists;
+
+/**
+ * Jackson serializable description of a file selection. Maintains an internal set of file statuses. However, also
+ * serializes out as a list of Strings. All accessing methods first regenerate the FileStatus objects if they are not
+ * available. This allows internal movement of FileStatus and the ability to serialize if need be.
+ */
+public class FileSelection {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSelection.class);
+
+ @JsonIgnore
+ private List<FileStatus> statuses;
+
+ public List<String> files;
+
+ public FileSelection() {
+ }
+
+
+ public FileSelection(List<String> files, boolean dummy){
+ this.files = files;
+ }
+
+ public FileSelection(List<FileStatus> statuses) {
+ this.statuses = statuses;
+ this.files = Lists.newArrayList();
+ for (FileStatus f : statuses) {
+ files.add(f.getPath().toString());
+ }
+ }
+
+ public boolean containsDirectories(DrillFileSystem fs) throws IOException {
+ init(fs);
+ for (FileStatus p : statuses) {
+ if (p.isDir()) return true;
+ }
+ return false;
+ }
+
+ public FileSelection minusDirectorries(DrillFileSystem fs) throws IOException {
+ init(fs);
+ List<FileStatus> newList = Lists.newArrayList();
+ for (FileStatus p : statuses) {
+ if (p.isDir()) {
+ List<FileStatus> statuses = fs.list(true, p.getPath());
+ for (FileStatus s : statuses) {
+ newList.add(s);
+ }
+
+ } else {
+ newList.add(p);
+ }
+ }
+ return new FileSelection(newList);
+ }
+
+ public FileStatus getFirstPath(DrillFileSystem fs) throws IOException {
+ init(fs);
+ return statuses.get(0);
+ }
+
+ public List<String> getAsFiles(){
+ if(!files.isEmpty()) return files;
+ if(statuses == null) return Collections.emptyList();
+ List<String> files = Lists.newArrayList();
+ for(FileStatus s : statuses){
+ files.add(s.getPath().toString());
+ }
+ return files;
+ }
+
+ private void init(DrillFileSystem fs) throws IOException {
+ if (files != null && statuses == null) {
+ statuses = Lists.newArrayList();
+ for (String p : files) {
+ statuses.add(fs.getFileStatus(new Path(p)));
+ }
+ }
+ }
+
+ public List<FileStatus> getFileStatusList(DrillFileSystem fs) throws IOException {
+ init(fs);
+ return statuses;
+ }
+
+ public static FileSelection create(DrillFileSystem fs, Path parent, String path) throws IOException {
+ if ( !(path.contains("*") || path.contains("?")) ) {
+ Path p = new Path(parent, path);
+ FileStatus status = fs.getFileStatus(p);
+ return new FileSelection(Collections.singletonList(status));
+ } else {
+ FileStatus[] status = fs.getUnderlying().globStatus(new Path(parent, path));
+ if(status == null || status.length == 0) return null;
+ return new FileSelection(Lists.newArrayList(status));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
new file mode 100644
index 0000000..455c4b2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.util.Map;
+
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("file")
+public class FileSystemConfig implements StoragePluginConfig{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemConfig.class);
+
+ public String connection;
+ public Map<String, String> workspaces;
+ public Map<String, FormatPluginConfig> formats;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemFormatConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemFormatConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemFormatConfig.java
new file mode 100644
index 0000000..be396a8
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemFormatConfig.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+
+public class FileSystemFormatConfig<T extends FormatPluginConfig> implements StoragePluginConfig{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemFormatConfig.class);
+
+ public T getFormatConfig(){
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
new file mode 100644
index 0000000..3762c1a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.AbstractStoragePlugin;
+import org.apache.drill.exec.store.ClassPathFileSystem;
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+import org.apache.drill.exec.store.dfs.shim.FileSystemCreator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.beust.jcommander.internal.Lists;
+import com.beust.jcommander.internal.Maps;
+
+/**
+ * A Storage engine associated with a Hadoop FileSystem Implementation. Examples include HDFS, MapRFS, QuantacastFileSystem,
+ * LocalFileSystem, as well Apache Drill specific CachedFileSystem, ClassPathFileSystem and LocalSyncableFileSystem.
+ * Tables are file names, directories and path patterns. This storage engine delegates to FSFormatEngines but shares
+ * references to the FileSystem configuration and path management.
+ */
+public class FileSystemPlugin extends AbstractStoragePlugin{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemPlugin.class);
+
+ private final FileSystemSchemaFactory schemaFactory;
+ private Map<String, FormatPlugin> formatsByName;
+ private Map<FormatPluginConfig, FormatPlugin> formatPluginsByConfig;
+ private FileSystemConfig config;
+ private DrillbitContext context;
+ private final DrillFileSystem fs;
+
+ public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String name) throws ExecutionSetupException{
+ try{
+ this.config = config;
+ this.context = context;
+
+ Configuration fsConf = new Configuration();
+ fsConf.set("fs.default.name", config.connection);
+ fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
+ this.fs = FileSystemCreator.getFileSystem(context.getConfig(), fsConf);
+ this.formatsByName = FormatCreator.getFormatPlugins(context, fs, config);
+ List<FormatMatcher> matchers = Lists.newArrayList();
+ formatPluginsByConfig = Maps.newHashMap();
+ for(FormatPlugin p : formatsByName.values()){
+ matchers.add(p.getMatcher());
+ formatPluginsByConfig.put(p.getConfig(), p);
+ }
+
+ List<WorkspaceSchemaFactory> factories = null;
+ if(config.workspaces == null || config.workspaces.isEmpty()){
+ factories = Collections.singletonList(new WorkspaceSchemaFactory("default", name, fs, "/", matchers));
+ }else{
+ factories = Lists.newArrayList();
+ for(Map.Entry<String, String> space : config.workspaces.entrySet()){
+ factories.add(new WorkspaceSchemaFactory(space.getKey(), name, fs, space.getValue(), matchers));
+ }
+ }
+ this.schemaFactory = new FileSystemSchemaFactory(name, factories);
+ }catch(IOException e){
+ throw new ExecutionSetupException("Failure setting up file system plugin.", e);
+ }
+ }
+
+ @Override
+ public boolean supportsRead() {
+ return true;
+ }
+
+ @Override
+ public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException {
+ FormatSelection formatSelection = scan.getSelection().getWith(context.getConfig(), FormatSelection.class);
+ FormatPlugin plugin;
+ if(formatSelection.getFormat() instanceof NamedFormatPluginConfig){
+ plugin = formatsByName.get( ((NamedFormatPluginConfig) formatSelection.getFormat()).name);
+ }else{
+ plugin = formatPluginsByConfig.get(formatSelection.getFormat());
+ }
+ if(plugin == null) throw new IOException(String.format("Failure getting requested format plugin named '%s'. It was not one of the format plugins registered.", formatSelection.getFormat()));
+ return plugin.getGroupScan(scan.getOutputReference(), formatSelection.getSelection());
+ }
+
+ @Override
+ public Schema createAndAddSchema(SchemaPlus parent) {
+ return schemaFactory.add(parent);
+ }
+
+ public FormatPlugin getFormatPlugin(String name){
+ return formatsByName.get(name);
+ }
+
+ public FormatPlugin getFormatPlugin(FormatPluginConfig config){
+ if(config instanceof NamedFormatPluginConfig){
+ return formatsByName.get(((NamedFormatPluginConfig) config).name);
+ }else{
+ return formatPluginsByConfig.get(config);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
new file mode 100644
index 0000000..fbea81c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+import net.hydromatic.optiq.TableFunction;
+
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.SchemaFactory;
+import org.apache.drill.exec.store.SchemaHolder;
+import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema;
+
+import com.google.common.collect.Maps;
+
+
+/**
+ * This is the top level schema that responds to root level path requests. Also supports
+ */
+public class FileSystemSchemaFactory implements SchemaFactory{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchemaFactory.class);
+
+ private List<WorkspaceSchemaFactory> factories;
+ private String schemaName;
+
+
+ public FileSystemSchemaFactory(String schemaName, List<WorkspaceSchemaFactory> factories) {
+ super();
+ this.schemaName = schemaName;
+ this.factories = factories;
+ }
+
+ @Override
+ public Schema add(SchemaPlus parent) {
+ FileSystemSchema schema = new FileSystemSchema(parent, schemaName);
+ schema.setHolder(parent.add(schema));
+ return schema;
+ }
+
+ public class FileSystemSchema extends AbstractSchema{
+
+ private final WorkspaceSchema defaultSchema;
+ private final Map<String, WorkspaceSchema> schemaMap = Maps.newHashMap();
+ final SchemaHolder selfHolder = new SchemaHolder();
+
+ public FileSystemSchema(SchemaPlus parentSchema, String name) {
+ super(new SchemaHolder(parentSchema), name);
+ for(WorkspaceSchemaFactory f : factories){
+ WorkspaceSchema s = f.create(selfHolder);
+ schemaMap.put(s.getName(), s);
+ }
+
+ defaultSchema = schemaMap.get("default");
+ }
+
+ void setHolder(SchemaPlus plusOfThis){
+ selfHolder.setSchema(plusOfThis);
+ for(WorkspaceSchema s : schemaMap.values()){
+ plusOfThis.add(s);
+ }
+ }
+
+ @Override
+ public DrillTable getTable(String name) {
+ return defaultSchema.getTable(name);
+ }
+
+ @Override
+ public Collection<TableFunction> getTableFunctions(String name) {
+ return defaultSchema.getTableFunctions(name);
+ }
+
+ @Override
+ public Set<String> getTableFunctionNames() {
+ return defaultSchema.getTableFunctionNames();
+ }
+
+ @Override
+ public Schema getSubSchema(String name) {
+ return defaultSchema.getSubSchema(name);
+ }
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ return defaultSchema.getSubSchemaNames();
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return defaultSchema.getTableNames();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java
new file mode 100644
index 0000000..7ce8c50
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.common.util.ConstructorChecker;
+import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+
+import com.google.common.collect.Maps;
+
+public class FormatCreator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatCreator.class);
+
+
+ static final ConstructorChecker FORMAT_BASED = new ConstructorChecker(String.class, DrillbitContext.class, DrillFileSystem.class, StoragePluginConfig.class, FormatPlugin.class);
+ static final ConstructorChecker DEFAULT_BASED = new ConstructorChecker(String.class, DrillbitContext.class, DrillFileSystem.class, StoragePluginConfig.class);
+
+ static Map<String, FormatPlugin> getFormatPlugins(DrillbitContext context, DrillFileSystem fileSystem, FileSystemConfig storageConfig){
+ final DrillConfig config = context.getConfig();
+ Map<String, FormatPlugin> plugins = Maps.newHashMap();
+
+ Collection<Class<? extends FormatPlugin>> pluginClasses = PathScanner.scanForImplementations(FormatPlugin.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES));
+
+
+ if(storageConfig.formats == null || storageConfig.formats.isEmpty()){
+
+ for(Class<? extends FormatPlugin> pluginClass: pluginClasses){
+ for(Constructor<?> c : pluginClass.getConstructors()){
+ try{
+
+ if(!DEFAULT_BASED.check(c)) continue;
+ FormatPlugin plugin = (FormatPlugin) c.newInstance(null, context, fileSystem, storageConfig);
+ plugins.put(plugin.getName(), plugin);
+ }catch(Exception e){
+ logger.warn(String.format("Failure while trying instantiate FormatPlugin %s.", pluginClass.getName()), e);
+ }
+ }
+ }
+
+ }else{
+
+ Map<Class<?>, Constructor<?>> constructors = Maps.newHashMap();
+ for(Class<? extends FormatPlugin> pluginClass: pluginClasses){
+ for(Constructor<?> c : pluginClass.getConstructors()){
+ try{
+ if(!FORMAT_BASED.check(c)) continue;
+ constructors.put(pluginClass, c);
+ }catch(Exception e){
+ logger.warn(String.format("Failure while trying instantiate FormatPlugin %s.", pluginClass.getName()), e);
+ }
+ }
+ }
+
+ for(Map.Entry<String, FormatPluginConfig> e : storageConfig.formats.entrySet()){
+ Constructor<?> c = constructors.get(e.getValue().getClass());
+ if(c == null){
+ logger.warn("Unable to find constructor for storage config named '{}' of type '{}'.", e.getKey(), e.getValue().getClass().getName());
+ continue;
+ }
+ try{
+ plugins.put(e.getKey(), (FormatPlugin) c.newInstance(e.getKey(), context, fileSystem, storageConfig, e.getValue()));
+ } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e1) {
+ logger.warn("Failure initializing storage config named '{}' of type '{}'.", e.getKey(), e.getValue().getClass().getName(), e1);
+ }
+ }
+
+
+ }
+
+ return plugins;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
new file mode 100644
index 0000000..e8521e4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.io.IOException;
+
+public abstract class FormatMatcher {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatMatcher.class);
+
+ public abstract boolean supportDirectoryReads();
+ public abstract FormatSelection isReadable(FileSelection file) throws IOException;
+ public abstract FormatPlugin getFormatPlugin();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java
new file mode 100644
index 0000000..a37142e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.QueryOptimizerRule;
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+
+/**
+ * Similar to a storage engine but built specifically to work within a FileSystem context.
+ */
+public interface FormatPlugin {
+
+ public boolean supportsRead();
+
+ public boolean supportsWrite();
+
+ public FormatMatcher getMatcher();
+
+ public AbstractGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException;
+
+ public List<QueryOptimizerRule> getOptimizerRules();
+
+ public FormatPluginConfig getConfig();
+ public StoragePluginConfig getStorageConfig();
+ public DrillFileSystem getFileSystem();
+ public DrillbitContext getContext();
+ public String getName();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java
new file mode 100644
index 0000000..5cf6ce3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.util.List;
+
+import org.apache.drill.common.logical.FormatPluginConfig;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+
+public class FormatSelection {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatSelection.class);
+
+ private FormatPluginConfig format;
+ private FileSelection selection;
+
+ public FormatSelection(){}
+
+ @JsonCreator
+ public FormatSelection(@JsonProperty("format") FormatPluginConfig format, @JsonProperty("files") List<String> files){
+ this.format = format;
+ this.selection = new FileSelection(files, true);
+ }
+
+ public FormatSelection(FormatPluginConfig format, FileSelection selection) {
+ super();
+ this.format = format;
+ this.selection = selection;
+ }
+
+ @JsonProperty("format")
+ public FormatPluginConfig getFormat(){
+ return format;
+ }
+
+ @JsonProperty("files")
+ public List<String> getAsFiles(){
+ return selection.getAsFiles();
+ }
+
+ @JsonIgnore
+ public FileSelection getSelection(){
+ return selection;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java
new file mode 100644
index 0000000..f514388
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+public class MagicString {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MagicString.class);
+
+ private long offset;
+ private byte[] bytes;
+
+ public MagicString(long offset, byte[] bytes) {
+ super();
+ this.offset = offset;
+ this.bytes = bytes;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public byte[] getBytes() {
+ return bytes;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java
new file mode 100644
index 0000000..6b98ea2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import org.apache.drill.common.logical.FormatPluginConfig;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+
+@JsonTypeName("named")
+public class NamedFormatPluginConfig implements FormatPluginConfig{
+ public String name;
+}
[17/38] git commit: DRILL-378: Initialize overLimit field in
UnlimitedRawBatchBuffer
Posted by ja...@apache.org.
DRILL-378: Initialize overLimit field in UnlimitedRawBatchBuffer
Signed-off-by: Jacques Nadeau <ja...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/7716c7e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/7716c7e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/7716c7e7
Branch: refs/heads/master
Commit: 7716c7e785116ebc6a77ef0342c1de9b4e73cffa
Parents: 7f7b6be
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Feb 13 00:37:38 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7716c7e7/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
index 0175bb6..97d8d34 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
@@ -34,7 +34,7 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
private volatile boolean finished = false;
private int softlimit;
private int startlimit;
- private AtomicBoolean overlimit;
+ private AtomicBoolean overlimit = new AtomicBoolean(false);
public UnlimitedRawBatchBuffer(FragmentContext context) {
softlimit = context.getConfig().getInt(ExecConstants.INCOMING_BUFFER_SIZE);
[20/38] Integrate new SQL changes with Hive storage engine,
moving to automatic file detection. Rename storage engine to storage
plugin. Separate storage plugins from format plugins,
updating Parquet and JSON to format engines. Refactor distribution log
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java
new file mode 100644
index 0000000..41960b6
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java
@@ -0,0 +1,390 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.json;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+
+import mockit.Expectations;
+import mockit.Injectable;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.store.easy.json.JSONRecordReader;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+
+public class JSONRecordReaderTest {
+ private static final Charset UTF_8 = Charset.forName("UTF-8");
+
+ private String getResource(String resourceName) {
+ return "resource:" + resourceName;
+ }
+
+ class MockOutputMutator implements OutputMutator {
+ List<MaterializedField> removedFields = Lists.newArrayList();
+ List<ValueVector> addFields = Lists.newArrayList();
+
+ @Override
+ public void removeField(MaterializedField field) throws SchemaChangeException {
+ removedFields.add(field);
+ }
+
+ @Override
+ public void addField(ValueVector vector) throws SchemaChangeException {
+ addFields.add(vector);
+ }
+
+ @Override
+ public void removeAllFields() {
+ addFields.clear();
+ }
+
+ @Override
+ public void setNewSchema() throws SchemaChangeException {
+ }
+
+ List<MaterializedField> getRemovedFields() {
+ return removedFields;
+ }
+
+ List<ValueVector> getAddFields() {
+ return addFields;
+ }
+ }
+
+ private <T> void assertField(ValueVector valueVector, int index, MinorType expectedMinorType, T value, String name) {
+ UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
+ SchemaDefProtos.FieldDef def = metadata.getDef();
+ assertEquals(expectedMinorType, def.getMajorType().getMinorType());
+ String[] parts = name.split("\\.");
+ int expected = parts.length;
+ assertEquals(expected, def.getNameList().size());
+ for(int i = 0; i < parts.length; ++i) {
+ assertEquals(parts[i], def.getName(i).getName());
+ }
+
+ if (expectedMinorType == MinorType.MAP) {
+ return;
+ }
+
+ T val = (T) valueVector.getAccessor().getObject(index);
+ assertValue(value, val);
+ }
+
+ private void assertValue(Object expected, Object found) {
+ if (found instanceof byte[]) {
+ assertTrue(Arrays.equals((byte[]) expected, (byte[]) found));
+ } else if(found instanceof ArrayList) {
+ List expectedArray = (List) expected;
+ List foundArray = (List) found;
+ assertEquals(expectedArray.size(), foundArray.size());
+ for(int i = 0; i < expectedArray.size(); ++i) {
+ assertValue(expectedArray.get(i), foundArray.get(i));
+ }
+ } else {
+ assertEquals(expected, found);
+ }
+ }
+
+ @Test
+ public void testSameSchemaInSameBatch(@Injectable final FragmentContext context) throws IOException,
+ ExecutionSetupException {
+ new Expectations() {
+ {
+ context.getAllocator();
+ returns(new TopLevelAllocator());
+ }
+ };
+ JSONRecordReader jr = new JSONRecordReader(context,
+ FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString(),
+ FileSystem.getLocal(new Configuration()), null, null);
+
+ MockOutputMutator mutator = new MockOutputMutator();
+ List<ValueVector> addFields = mutator.getAddFields();
+ jr.setup(mutator);
+ assertEquals(2, jr.next());
+ assertEquals(3, addFields.size());
+ assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
+ assertField(addFields.get(1), 0, MinorType.BIT, true, "b");
+ assertField(addFields.get(2), 0, MinorType.VARCHAR, "hi!", "c");
+ assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
+ assertField(addFields.get(1), 1, MinorType.BIT, false, "b");
+ assertField(addFields.get(2), 1, MinorType.VARCHAR, "drill!", "c");
+
+ assertEquals(0, jr.next());
+ assertTrue(mutator.getRemovedFields().isEmpty());
+ }
+
+ @Test
+ public void testChangedSchemaInSameBatch(@Injectable final FragmentContext context) throws IOException,
+ ExecutionSetupException {
+ new Expectations() {
+ {
+ context.getAllocator();
+ returns(new TopLevelAllocator());
+ }
+ };
+
+ JSONRecordReader jr = new JSONRecordReader(context,
+ FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
+ FileSystem.getLocal(new Configuration()), null, null);
+
+ MockOutputMutator mutator = new MockOutputMutator();
+ List<ValueVector> addFields = mutator.getAddFields();
+
+ jr.setup(mutator);
+ assertEquals(3, jr.next());
+ assertEquals(7, addFields.size());
+ assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
+ assertField(addFields.get(1), 0, MinorType.INT, 1, "b");
+ assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c");
+ assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
+ assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1", "str1");
+
+ assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
+ assertField(addFields.get(1), 1, MinorType.INT, 3, "b");
+ assertField(addFields.get(3), 1, MinorType.BIT, false, "bool");
+ assertField(addFields.get(4), 1, MinorType.VARCHAR, "test2", "str1");
+ assertField(addFields.get(5), 1, MinorType.INT, 4, "d");
+
+ assertField(addFields.get(0), 2, MinorType.INT, 12345, "test");
+ assertField(addFields.get(2), 2, MinorType.FLOAT4, (float) 5.16, "c");
+ assertField(addFields.get(3), 2, MinorType.BIT, true, "bool");
+ assertField(addFields.get(5), 2, MinorType.INT, 6, "d");
+ assertField(addFields.get(6), 2, MinorType.VARCHAR, "test3", "str2");
+ assertTrue(mutator.getRemovedFields().isEmpty());
+ assertEquals(0, jr.next());
+ }
+
+ @Test
+ public void testChangedSchemaInTwoBatchesColumnSelect(@Injectable final FragmentContext context) throws IOException,
+ ExecutionSetupException {
+ new Expectations() {
+ {
+ context.getAllocator();
+ returns(new TopLevelAllocator());
+ }
+ };
+
+ JSONRecordReader jr = new JSONRecordReader(context,
+ FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
+ FileSystem.getLocal(new Configuration()),
+ 64, null, Arrays.asList(new SchemaPath("test", ExpressionPosition.UNKNOWN))); // batch only fits 1 int
+ MockOutputMutator mutator = new MockOutputMutator();
+ List<ValueVector> addFields = mutator.getAddFields();
+ List<MaterializedField> removedFields = mutator.getRemovedFields();
+
+ jr.setup(mutator);
+ assertEquals(1, jr.next());
+ assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
+ assertTrue(removedFields.isEmpty());
+ assertEquals(addFields.size(), 1);
+ assertEquals(1, jr.next());
+ assertField(addFields.get(0), 0, MinorType.INT, 1234, "test");
+ assertEquals(addFields.size(), 1);
+ assertTrue(removedFields.isEmpty());
+ removedFields.clear();
+ assertEquals(1, jr.next());
+ assertField(addFields.get(0), 0, MinorType.INT, 12345, "test");
+ assertEquals(addFields.size(), 1);
+ assertTrue(removedFields.isEmpty());
+ assertEquals(0, jr.next());
+ }
+
+ @Test
+ public void testChangedSchemaInTwoBatches(@Injectable final FragmentContext context) throws IOException,
+ ExecutionSetupException {
+ new Expectations() {
+ {
+ context.getAllocator();
+ returns(new TopLevelAllocator());
+ }
+ };
+
+ JSONRecordReader jr = new JSONRecordReader(context,
+ FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
+ FileSystem.getLocal(new Configuration()),
+ 64, null, null); // batch only fits 1 int
+ MockOutputMutator mutator = new MockOutputMutator();
+ List<ValueVector> addFields = mutator.getAddFields();
+ List<MaterializedField> removedFields = mutator.getRemovedFields();
+
+ jr.setup(mutator);
+ assertEquals(1, jr.next());
+ assertEquals(5, addFields.size());
+ assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
+ assertField(addFields.get(1), 0, MinorType.INT, 1, "b");
+ assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c");
+ assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
+ assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1", "str1");
+ assertTrue(removedFields.isEmpty());
+ assertEquals(1, jr.next());
+ assertEquals(6, addFields.size());
+ assertField(addFields.get(0), 0, MinorType.INT, 1234, "test");
+ assertField(addFields.get(1), 0, MinorType.INT, 3, "b");
+ assertField(addFields.get(3), 0, MinorType.BIT, false, "bool");
+ assertField(addFields.get(4), 0, MinorType.VARCHAR, "test2", "str1");
+ assertField(addFields.get(5), 0, MinorType.INT, 4, "d");
+ assertEquals(1, removedFields.size());
+ assertEquals("c", removedFields.get(0).getName());
+ removedFields.clear();
+ assertEquals(1, jr.next());
+ assertEquals(7, addFields.size()); // The reappearing of field 'c' is also included
+ assertField(addFields.get(0), 0, MinorType.INT, 12345, "test");
+ assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
+ assertField(addFields.get(5), 0, MinorType.INT, 6, "d");
+ assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 5.16, "c");
+ assertField(addFields.get(6), 0, MinorType.VARCHAR, "test3", "str2");
+ assertEquals(2, removedFields.size());
+ Iterables.find(removedFields, new Predicate<MaterializedField>() {
+ @Override
+ public boolean apply(MaterializedField materializedField) {
+ return materializedField.getName().equals("str1");
+ }
+ });
+ Iterables.find(removedFields, new Predicate<MaterializedField>() {
+ @Override
+ public boolean apply(MaterializedField materializedField) {
+ return materializedField.getName().equals("b");
+ }
+ });
+ assertEquals(0, jr.next());
+ }
+
+ @Test
+ public void testNestedFieldInSameBatch(@Injectable final FragmentContext context) throws ExecutionSetupException, IOException {
+ new Expectations() {
+ {
+ context.getAllocator();
+ returns(new TopLevelAllocator());
+ }
+ };
+
+ JSONRecordReader jr = new JSONRecordReader(context,
+ FileUtils.getResourceAsFile("/scan_json_test_3.json").toURI().toString(),
+ FileSystem.getLocal(new Configuration()), null, null);
+
+ MockOutputMutator mutator = new MockOutputMutator();
+ List<ValueVector> addFields = mutator.getAddFields();
+ jr.setup(mutator);
+ assertEquals(2, jr.next());
+ assertEquals(3, addFields.size());
+ assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
+ assertField(addFields.get(1), 0, MinorType.VARCHAR, "test", "a.b");
+ assertField(addFields.get(2), 0, MinorType.BIT, true, "a.a.d");
+ assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
+ assertField(addFields.get(1), 1, MinorType.VARCHAR, "test2", "a.b");
+ assertField(addFields.get(2), 1, MinorType.BIT, false, "a.a.d");
+
+ assertEquals(0, jr.next());
+ assertTrue(mutator.getRemovedFields().isEmpty());
+ }
+
+ @Test
+ public void testRepeatedFields(@Injectable final FragmentContext context) throws ExecutionSetupException, IOException {
+ new Expectations() {
+ {
+ context.getAllocator();
+ returns(new TopLevelAllocator());
+ }
+ };
+
+ JSONRecordReader jr = new JSONRecordReader(context,
+ FileUtils.getResourceAsFile("/scan_json_test_4.json").toURI().toString(),
+ FileSystem.getLocal(new Configuration()), null, null);
+
+ MockOutputMutator mutator = new MockOutputMutator();
+ List<ValueVector> addFields = mutator.getAddFields();
+ jr.setup(mutator);
+ assertEquals(2, jr.next());
+ assertEquals(7, addFields.size());
+ assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
+ assertField(addFields.get(1), 0, MinorType.INT, Arrays.asList(1, 2, 3), "test2");
+ assertField(addFields.get(2), 0, MinorType.INT, Arrays.asList(4, 5, 6), "test3.a");
+ assertField(addFields.get(3), 0, MinorType.INT, Arrays.asList(7, 8, 9), "test3.b");
+ assertField(addFields.get(4), 0, MinorType.INT, Arrays.asList(10, 11, 12), "test3.c.d");
+ assertField(addFields.get(5), 0, MinorType.FLOAT4, Arrays.<Float>asList((float) 1.1, (float) 1.2, (float) 1.3), "testFloat");
+ assertField(addFields.get(6), 0, MinorType.VARCHAR, Arrays.asList("hello", "drill"), "testStr");
+ assertField(addFields.get(1), 1, MinorType.INT, Arrays.asList(1, 2), "test2");
+ assertField(addFields.get(2), 1, MinorType.INT, Arrays.asList(7, 7, 7, 8), "test3.a");
+ assertField(addFields.get(5), 1, MinorType.FLOAT4, Arrays.<Float>asList((float) 2.2, (float) 2.3,(float) 2.4), "testFloat");
+
+ assertEquals(0, jr.next());
+ assertTrue(mutator.getRemovedFields().isEmpty());
+ }
+
+ @Test
+ public void testRepeatedMissingFields(@Injectable final FragmentContext context) throws ExecutionSetupException, IOException {
+ new Expectations() {
+ {
+ context.getAllocator();
+ returns(new TopLevelAllocator());
+ }
+ };
+
+ JSONRecordReader jr = new JSONRecordReader(context,
+ FileUtils.getResourceAsFile("/scan_json_test_5.json").toURI().toString(),
+ FileSystem.getLocal(new Configuration()), null, null);
+
+ MockOutputMutator mutator = new MockOutputMutator();
+ List<ValueVector> addFields = mutator.getAddFields();
+ jr.setup(mutator);
+ assertEquals(9, jr.next());
+ assertEquals(1, addFields.size());
+ assertField(addFields.get(0), 0, MinorType.INT, Arrays.<Integer>asList(), "test");
+ assertField(addFields.get(0), 1, MinorType.INT, Arrays.asList(1, 2, 3), "test");
+ assertField(addFields.get(0), 2, MinorType.INT, Arrays.<Integer>asList(), "test");
+ assertField(addFields.get(0), 3, MinorType.INT, Arrays.<Integer>asList(), "test");
+ assertField(addFields.get(0), 4, MinorType.INT, Arrays.asList(4, 5, 6), "test");
+ assertField(addFields.get(0), 5, MinorType.INT, Arrays.<Integer>asList(), "test");
+ assertField(addFields.get(0), 6, MinorType.INT, Arrays.<Integer>asList(), "test");
+ assertField(addFields.get(0), 7, MinorType.INT, Arrays.asList(7, 8, 9), "test");
+ assertField(addFields.get(0), 8, MinorType.INT, Arrays.<Integer>asList(), "test");
+
+
+ assertEquals(0, jr.next());
+ assertTrue(mutator.getRemovedFields().isEmpty());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/FieldInfo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/FieldInfo.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/FieldInfo.java
new file mode 100644
index 0000000..34f60ba
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/FieldInfo.java
@@ -0,0 +1,40 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.common.types.TypeProtos;
+
+public class FieldInfo {
+ String parquetType;
+ String name;
+ int bitLength;
+ int numberOfPages;
+ Object[] values;
+ TypeProtos.MinorType type;
+
+ FieldInfo(String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type, ParquetTestProperties props){
+ this.parquetType = parquetType;
+ this.name = name;
+ this.bitLength = bitLength;
+ this.numberOfPages = Math.max(1, (int) Math.ceil( ((long) props.recordsPerRowGroup) * bitLength / 8.0 / props.bytesPerPage));
+ this.values = values;
+ // generator is designed to use 3 values
+ assert values.length == 3;
+ this.type = type;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index 52430e1..9e999e3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -17,38 +17,45 @@
*/
package org.apache.drill.exec.store.parquet;
+import static org.apache.drill.exec.store.parquet.TestFileGenerator.populateFieldInfoMap;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import mockit.Injectable;
+
import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.rpc.user.UserServer;
import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.store.parquet.TestFileGenerator.FieldInfo;
-import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.store.CachedSingleFileSystem;
+import org.apache.drill.exec.store.TestOutputMutator;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -57,7 +64,9 @@ import parquet.bytes.BytesInput;
import parquet.column.page.Page;
import parquet.column.page.PageReadStore;
import parquet.column.page.PageReader;
+import parquet.hadoop.CodecFactoryExposer;
import parquet.hadoop.Footer;
+import parquet.hadoop.ParquetFileReader;
import parquet.hadoop.metadata.ParquetMetadata;
import parquet.schema.MessageType;
@@ -65,30 +74,33 @@ import com.google.common.base.Charsets;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
-import com.google.common.util.concurrent.SettableFuture;
public class ParquetRecordReaderTest {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest.class);
- private boolean VERBOSE_DEBUG = false;
+ static boolean VERBOSE_DEBUG = false;
private boolean checkValues = true;
static final int numberRowGroups = 1;
static final int recordsPerRowGroup = 300;
+ static int DEFAULT_BYTES_PER_PAGE = 1024 * 1024 * 1;
static final String fileName = "/tmp/parquet_test_file_many_types";
@BeforeClass
public static void generateFile() throws Exception{
File f = new File(fileName);
- if(!f.exists()) TestFileGenerator.generateParquetFile(fileName, numberRowGroups, recordsPerRowGroup);
+ ParquetTestProperties props = new ParquetTestProperties(numberRowGroups, recordsPerRowGroup, DEFAULT_BYTES_PER_PAGE, new HashMap<String, FieldInfo>());
+ populateFieldInfoMap(props);
+ if(!f.exists()) TestFileGenerator.generateParquetFile(fileName, props);
}
-
+
+
@Test
- public void testMultipleRowGroupsAndReads() throws Exception {
+ public void testMultipleRowGroupsAndReads3() throws Exception {
String planName = "/parquet/parquet_scan_screen.json";
testParquetFullEngineLocalPath(planName, fileName, 2, numberRowGroups, recordsPerRowGroup);
}
-
+
@Test
public void testMultipleRowGroupsAndReads2() throws Exception {
String readEntries;
@@ -96,7 +108,7 @@ public class ParquetRecordReaderTest {
// number of times to read the file
int i = 3;
for (int j = 0; j < i; j++){
- readEntries += "{path: \""+fileName+"\"}";
+ readEntries += "\""+fileName+"\"";
if (j < i - 1)
readEntries += ",";
}
@@ -117,150 +129,6 @@ public class ParquetRecordReaderTest {
testParquetFullEngineRemote(planName, fileName, 1, numberRowGroups, recordsPerRowGroup);
}
-
- private class ParquetResultListener implements UserResultsListener {
- private SettableFuture<Void> future = SettableFuture.create();
- RecordBatchLoader batchLoader;
-
- int batchCounter = 1;
- private final HashMap<String, Long> valuesChecked = new HashMap<>();
- private final Map<String, FieldInfo> fields;
- private final long totalRecords;
-
- ParquetResultListener(int recordsPerRowGroup, RecordBatchLoader batchLoader, int numberRowGroups, int numberOfTimesRead){
- this.batchLoader = batchLoader;
- this.fields = TestFileGenerator.getFieldMap(recordsPerRowGroup);
- this.totalRecords = recordsPerRowGroup * numberRowGroups * numberOfTimesRead;
- }
-
- @Override
- public void submissionFailed(RpcException ex) {
- logger.debug("Submission failed.", ex);
- future.setException(ex);
- }
-
- @Override
- public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
- long columnValCounter = 0;
- int i = 0;
- FieldInfo currentField;
-
- boolean schemaChanged = false;
- try {
- schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData());
- } catch (SchemaChangeException e) {
- logger.error("Failure while loading batch", e);
- }
-
- // print headers.
- if (schemaChanged) {
- } // do not believe any change is needed for when the schema changes, with the current mock scan use case
-
- for (VectorWrapper<?> vw : batchLoader) {
- ValueVector vv = vw.getValueVector();
- currentField = fields.get(vv.getField().getName());
- if (VERBOSE_DEBUG){
- System.out.println("\n" + (String) currentField.name);
- }
- if ( ! valuesChecked.containsKey(vv.getField().getName())){
- valuesChecked.put(vv.getField().getName(), (long) 0);
- columnValCounter = 0;
- } else {
- columnValCounter = valuesChecked.get(vv.getField().getName());
- }
- for (int j = 0; j < ((BaseDataValueVector)vv).getAccessor().getValueCount(); j++) {
- if (VERBOSE_DEBUG){
- System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
- }
- if (checkValues) {
- try {
- assertField(vv, j, (TypeProtos.MinorType) currentField.type,
- currentField.values[(int) (columnValCounter % 3)], (String) currentField.name + "/");
- } catch (AssertionError e) { submissionFailed(new RpcException(e)); }
- }
- columnValCounter++;
- }
- if (VERBOSE_DEBUG){
- System.out.println("\n" + ((BaseDataValueVector)vv).getAccessor().getValueCount());
- }
- valuesChecked.remove(vv.getField().getName());
- valuesChecked.put(vv.getField().getName(), columnValCounter);
- }
-
-
- if (VERBOSE_DEBUG){
- for (i = 0; i < batchLoader.getRecordCount(); i++) {
- if (i % 50 == 0){
- System.out.println();
- for (VectorWrapper<?> vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getField().getName(), 20) + " ");
- }
- System.out.println();
- System.out.println();
- }
-
- for (VectorWrapper<?> vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getAccessor().getObject(i).toString(), 20) + " ");
- }
- System.out.println(
-
- );
- }
- }
-
- for(VectorWrapper<?> vw : batchLoader){
- vw.clear();
- }
- result.release();
-
- batchCounter++;
- if(result.getHeader().getIsLastChunk()){
- for (String s : valuesChecked.keySet()) {
- try {
- assertEquals("Record count incorrect for column: " + s, totalRecords, (long) valuesChecked.get(s));
- } catch (AssertionError e) { submissionFailed(new RpcException(e)); }
- }
-
- assert valuesChecked.keySet().size() > 0;
- future.set(null);
- }
- }
-
- public void get() throws RpcException{
- try{
- future.get();
- return;
- }catch(Throwable t){
- throw RpcException.mapException(t);
- }
- }
-
- @Override
- public void queryIdArrived(QueryId queryId) {
- }
- }
-
-
-
-
- public void testParquetFullEngineRemote(String plan, String filename, int numberOfTimesRead /* specified in json plan */, int numberOfRowGroups, int recordsPerRowGroup) throws Exception{
-
- DrillConfig config = DrillConfig.create();
-
- checkValues = false;
-
- try(DrillClient client = new DrillClient(config);){
- client.connect();
- RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
- ParquetResultListener resultListener = new ParquetResultListener(recordsPerRowGroup, batchLoader, numberOfRowGroups, numberOfTimesRead);
- client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8), resultListener);
- resultListener.get();
- }
-
- }
-
public void testParquetFullEngineLocalPath(String planFileName, String filename, int numberOfTimesRead /* specified in json plan */, int numberOfRowGroups, int recordsPerRowGroup) throws Exception{
testParquetFullEngineLocalText(Files.toString(FileUtils.getResourceAsFile(planFileName), Charsets.UTF_8), filename, numberOfTimesRead, numberOfRowGroups, recordsPerRowGroup);
@@ -277,14 +145,17 @@ public class ParquetRecordReaderTest {
bit1.run();
client.connect();
RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
- ParquetResultListener resultListener = new ParquetResultListener(recordsPerRowGroup, batchLoader, numberOfRowGroups, numberOfTimesRead);
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(numberRowGroups, recordsPerRowGroup, DEFAULT_BYTES_PER_PAGE, fields);
+ TestFileGenerator.populateFieldInfoMap(props);
+ ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props, numberOfTimesRead, true);
Stopwatch watch = new Stopwatch().start();
client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener);
- resultListener.get();
+ resultListener.getResults();
System.out.println(String.format("Took %d ms to run query", watch.elapsed(TimeUnit.MILLISECONDS)));
}
-
+
}
@@ -301,10 +172,13 @@ public class ParquetRecordReaderTest {
bit1.run();
client.connect();
RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
- ParquetResultListener resultListener = new ParquetResultListener(recordsPerRowGroup, batchLoader, numberOfRowGroups, numberOfTimesRead);
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(numberRowGroups, recordsPerRowGroup, DEFAULT_BYTES_PER_PAGE, fields);
+ TestFileGenerator.populateFieldInfoMap(props);
+ ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props, numberOfTimesRead, true);
Stopwatch watch = new Stopwatch().start();
client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(planName), Charsets.UTF_8), resultListener);
- resultListener.get();
+ resultListener.getResults();
System.out.println(String.format("Took %d ms to run query", watch.elapsed(TimeUnit.MILLISECONDS)));
}
@@ -326,6 +200,25 @@ public class ParquetRecordReaderTest {
return result.toString();
}
+ public void testParquetFullEngineRemote(String plan, String filename, int numberOfTimesRead /* specified in json plan */, int numberOfRowGroups, int recordsPerRowGroup) throws Exception{
+
+ DrillConfig config = DrillConfig.create();
+
+ checkValues = false;
+
+ try(DrillClient client = new DrillClient(config);){
+ client.connect();
+ RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(numberRowGroups, recordsPerRowGroup, DEFAULT_BYTES_PER_PAGE, fields);
+ TestFileGenerator.populateFieldInfoMap(props);
+ ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props, numberOfTimesRead, true);
+ client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8), resultListener);
+ resultListener.getResults();
+ }
+
+ }
+
class MockOutputMutator implements OutputMutator {
List<MaterializedField> removedFields = Lists.newArrayList();
List<ValueVector> addFields = Lists.newArrayList();
@@ -358,24 +251,6 @@ public class ParquetRecordReaderTest {
}
}
- private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) {
- assertField(valueVector, index, expectedMinorType, value, name, 0);
- }
-
- @SuppressWarnings("unchecked")
- private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
-
- if (expectedMinorType == TypeProtos.MinorType.MAP) {
- return;
- }
-
- T val = (T) valueVector.getAccessor().getObject(index);
- if (val instanceof byte[]) {
- assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
- } else {
- assertEquals(value, val);
- }
- }
private void validateFooters(final List<Footer> metadata) {
logger.debug(metadata.toString());
@@ -391,8 +266,8 @@ public class ParquetRecordReaderTest {
assertEquals(footer.getFile().getName(), keyValueMetaData.get(footer.getFile().getName()));
}
}
-
-
+
+
private void validateContains(MessageType schema, PageReadStore pages, String[] path, int values, BytesInput bytes)
throws IOException {
PageReader pageReader = pages.getPageReader(schema.getColumnDescription(path));
@@ -402,5 +277,208 @@ public class ParquetRecordReaderTest {
}
-
+ @Test
+ public void testMultipleRowGroups() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(3, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ populateFieldInfoMap(props);
+ testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+ }
+
+ // TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was
+ // generated using pig. Will need to find a good place to keep files like this.
+ // For now I will upload it to the JIRA as an attachment.
+ @Ignore
+ @Test
+ public void testNullableColumns() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
+ Object[] boolVals = {true, null, null};
+ props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ testParquetFullEngineEventBased(false, "/parquet_nullable.json", "/tmp/nullable_test.parquet", 1, props);
+ }
+
+ @Ignore
+ @Test
+ public void testNullableColumnsVarLen() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(1, 300000, DEFAULT_BYTES_PER_PAGE, fields);
+ byte[] val = {'b'};
+ byte[] val2 = {'b', '2'};
+ byte[] val3 = { 'l','o','n','g','e','r',' ','s','t','r','i','n','g'};
+ Object[] boolVals = { val, val2, val3};
+ props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ testParquetFullEngineEventBased(false, "/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props);
+ }
+
+ @Test
+ public void testMultipleRowGroupsAndReads() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ populateFieldInfoMap(props);
+ String readEntries = "";
+ // number of times to read the file
+ int i = 3;
+ for (int j = 0; j < i; j++){
+ readEntries += "\"/tmp/test.parquet\"";
+ if (j < i - 1)
+ readEntries += ",";
+ }
+ testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
+ "/tmp/test.parquet", i, props);
+ }
+
+ // requires binary file generated by pig from TPCH data, also have to disable assert where data is coming in
+ @Ignore
+ @Test
+ public void testMultipleRowGroupsAndReadsPigError() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields);
+ TestFileGenerator.populatePigTPCHCustomerFields(props);
+ String readEntries = "{path: \"/tmp/tpc-h/customer\"}";
+ testParquetFullEngineEventBased(false, false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
+ "unused, no file is generated", 1, props, true);
+
+ fields = new HashMap();
+ props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields);
+ TestFileGenerator.populatePigTPCHSupplierFields(props);
+ readEntries = "{path: \"/tmp/tpc-h/supplier\"}";
+ testParquetFullEngineEventBased(false, false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
+ "unused, no file is generated", 1, props, true);
+ }
+
+ @Test
+ public void testMultipleRowGroupsEvent() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ populateFieldInfoMap(props);
+ testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+ }
+
+
+ /**
+ * Tests the attribute in a scan node to limit the columns read by a scan.
+ *
+ * The functionality of selecting all columns is tested in all of the other tests that leave out the attribute.
+ * @throws Exception
+ */
+ @Test
+ public void testSelectColumnRead() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ // generate metatdata for a series of test columns, these columns are all generated in the test file
+ populateFieldInfoMap(props);
+ TestFileGenerator.generateParquetFile("/tmp/test.parquet", props);
+ fields.clear();
+ // create a new object to describe the dataset expected out of the scan operation
+ // the fields added below match those requested in the plan specified in parquet_selective_column_read.json
+ // that is used below in the test query
+ props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ props.fields.put("integer", new FieldInfo("int32", "integer", 32, TestFileGenerator.intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, TestFileGenerator.longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("bin", new FieldInfo("binary", "bin", -1, TestFileGenerator.binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, TestFileGenerator.bin2Vals, TypeProtos.MinorType.VARBINARY, props));
+ testParquetFullEngineEventBased(true, false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
+ }
+
+ public static void main(String[] args) throws Exception{
+ // TODO - not sure why this has a main method, test below can be run directly
+ //new ParquetRecordReaderTest().testPerformance();
+ }
+
+ @Test
+ @Ignore
+ public void testPerformance(@Injectable final DrillbitContext bitContext,
+ @Injectable UserServer.UserClientConnection connection) throws Exception {
+ DrillConfig c = DrillConfig.create();
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
+
+// new NonStrictExpectations() {
+// {
+// context.getAllocator(); result = BufferAllocator.getAllocator(DrillConfig.create());
+// }
+// };
+
+ final String fileName = "/tmp/parquet_test_performance.parquet";
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(1, 20 * 1000 * 1000, DEFAULT_BYTES_PER_PAGE, fields);
+ populateFieldInfoMap(props);
+ //generateParquetFile(fileName, props);
+
+ Configuration dfsConfig = new Configuration();
+ List<Footer> footers = ParquetFileReader.readFooters(dfsConfig, new Path(fileName));
+ Footer f = footers.iterator().next();
+
+ List<SchemaPath> columns = Lists.newArrayList();
+ columns.add(new SchemaPath("_MAP.integer", ExpressionPosition.UNKNOWN));
+ columns.add(new SchemaPath("_MAP.bigInt", ExpressionPosition.UNKNOWN));
+ columns.add(new SchemaPath("_MAP.f", ExpressionPosition.UNKNOWN));
+ columns.add(new SchemaPath("_MAP.d", ExpressionPosition.UNKNOWN));
+ columns.add(new SchemaPath("_MAP.b", ExpressionPosition.UNKNOWN));
+ columns.add(new SchemaPath("_MAP.bin", ExpressionPosition.UNKNOWN));
+ columns.add(new SchemaPath("_MAP.bin2", ExpressionPosition.UNKNOWN));
+ int totalRowCount = 0;
+
+ FileSystem fs = new CachedSingleFileSystem(fileName);
+ for(int i = 0; i < 25; i++){
+ ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
+ new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), new FieldReference("_MAP",
+ ExpressionPosition.UNKNOWN), columns);
+ TestOutputMutator mutator = new TestOutputMutator();
+ rr.setup(mutator);
+ Stopwatch watch = new Stopwatch();
+ watch.start();
+
+ int rowCount = 0;
+ while ((rowCount = rr.next()) > 0) {
+ totalRowCount += rowCount;
+ }
+ System.out.println(String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS)));
+ rr.cleanup();
+ }
+ System.out.println(String.format("Total row count %s", totalRowCount));
+ }
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngineEventBased(boolean generateNew, String plan, String readEntries, String filename,
+ int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
+ testParquetFullEngineEventBased(true, generateNew, plan, readEntries,filename,
+ numberOfTimesRead /* specified in json plan */, props, true);
+ }
+
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngineEventBased(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
+ testParquetFullEngineEventBased(true, generateNew, plan, null, filename, numberOfTimesRead, props, true);
+ }
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngineEventBased(boolean testValues, boolean generateNew, String plan, String readEntries, String filename,
+ int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props,
+ boolean runAsLogicalPlan) throws Exception{
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ if (generateNew) TestFileGenerator.generateParquetFile(filename, props);
+ DrillConfig config = DrillConfig.create();
+ try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+ bit1.run();
+ client.connect();
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
+ ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props, numberOfTimesRead, testValues);
+ long C = System.nanoTime();
+ String planText = Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8);
+ // substitute in the string for the read entries, allows reuse of the plan file for several tests
+ if (readEntries != null) {
+ planText = planText.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries);
+ }
+ if (runAsLogicalPlan)
+ client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener);
+ else
+ client.runQuery(UserProtos.QueryType.PHYSICAL, planText, resultListener);
+ resultListener.getResults();
+ long D = System.nanoTime();
+ System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
new file mode 100644
index 0000000..2ab12cb
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
@@ -0,0 +1,742 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static parquet.column.Encoding.PLAIN;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.store.ByteArrayUtil;
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import parquet.bytes.BytesInput;
+import parquet.column.ColumnDescriptor;
+import parquet.hadoop.ParquetFileWriter;
+import parquet.hadoop.metadata.CompressionCodecName;
+import parquet.schema.MessageType;
+import parquet.schema.MessageTypeParser;
+
+import com.beust.jcommander.internal.Lists;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.google.common.util.concurrent.SettableFuture;
+
+public class ParquetRecordReaderTest2 {
+ org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest2.class);
+
+ private static final boolean VERBOSE_DEBUG = false;
+
+ // { 00000001, 00000010, 00000100, 00001000, 00010000, ... }
+ byte[] bitFields = {1, 2, 4, 8, 16, 32, 64, -128};
+ byte allBitsTrue = -1;
+ byte allBitsFalse = 0;
+ int DEFAULT_BYTES_PER_PAGE = 1024 * 1024 * 1;
+ static Object[] intVals = {-200, 100, Integer.MAX_VALUE };
+ static Object[] longVals = { -5000l, 5000l, Long.MAX_VALUE};
+ static Object[] floatVals = { 1.74f, Float.MAX_VALUE, Float.MIN_VALUE};
+ static Object[] doubleVals = {100.45d, Double.MAX_VALUE, Double.MIN_VALUE,};
+ static Object[] boolVals = {false, false, true};
+ static byte[] varLen1 = {50, 51, 52, 53, 54, 55, 56, 57, 58, 59};
+ static byte[] varLen2 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
+ static byte[] varLen3 = {100, 99, 98};
+ static Object[] binVals = { varLen3, varLen2, varLen3};
+ static Object[] bin2Vals = { varLen3, varLen2, varLen1};
+
+ private void populateFieldInfoMap(ParquetTestProperties props){
+ props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("f", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
+ props.fields.put("d", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
+ props.fields.put("b", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
+ }
+
+ private void populatePigTPCHCustomerFields(ParquetTestProperties props){
+ // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
+ props.fields.put("C_CUSTKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("C_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("C_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
+ props.fields.put("C_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
+ props.fields.put("C_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ props.fields.put("C_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("C_MKTSEGMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("C_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
+ }
+
+ private void populatePigTPCHSupplierFields(ParquetTestProperties props){
+ // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
+ props.fields.put("S_SUPPKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("S_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("S_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
+ props.fields.put("S_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
+ props.fields.put("S_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ props.fields.put("S_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("S_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
+ }
+
+ @Test
+ public void testMultipleRowGroups() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(3, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ populateFieldInfoMap(props);
+ testParquetFullEngine(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+ }
+
+ // TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was
+ // generated using pig. Will need to find a good place to keep files like this.
+ // For now I will upload it to the JIRA as an attachment.
+ @Ignore
+ @Test
+ public void testNullableColumns() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
+ Object[] boolVals = {true, null, null};
+ props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ testParquetFullEngine(false, "/parquet_nullable.json", "/tmp/nullable.parquet", 1, props);
+ }
+
+ @Ignore
+ @Test
+ public void testNullableColumnsVarLen() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
+ byte[] val = {'b'};
+// Object[] boolVals = { val, null, null};
+// Object[] boolVals = { null, null, null};
+ Object[] boolVals = { val, val, val};
+ props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
+ testParquetFullEngine(false, "/parquet_nullable_varlen.json", "/tmp/nullable.parquet", 1, props);
+ }
+
+ /**
+ * Tests the attribute in a scan node to limit the columns read by a scan.
+ *
+ * The functionality of selecting all columns is tested in all of the other tests that leave out the attribute.
+ * @throws Exception
+ */
+ @Test
+ public void testSelectColumnRead() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ // generate metatdata for a series of test columns, these columns are all generated in the test file
+ populateFieldInfoMap(props);
+ generateParquetFile("/tmp/test.parquet", props);
+ fields.clear();
+ // create a new object to describe the dataset expected out of the scan operation
+ // the fields added below match those requested in the plan specified in parquet_selective_column_read.json
+ // that is used below in the test query
+ props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
+ testParquetFullEngineEventBased(false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
+ }
+
+
+ @Test
+ public void testMultipleRowGroupsAndReads() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ populateFieldInfoMap(props);
+ String readEntries = "";
+ // number of times to read the file
+ int i = 3;
+ for (int j = 0; j < i; j++){
+ readEntries += "{path: \"/tmp/test.parquet\"}";
+ if (j < i - 1)
+ readEntries += ",";
+ }
+ testParquetFullEngineEventBased(true, "/parquet_scan_screen_read_entry_replace.json", readEntries,
+ "/tmp/test.parquet", i, props, true);
+ }
+
+ // requires binary file generated by pig from TPCH data, also have to disable assert where data is coming in
+ @Ignore
+ @Test
+ public void testMultipleRowGroupsAndReadsPigError() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ populatePigTPCHCustomerFields(props);
+// populatePigTPCHSupplierFields(props);
+ String readEntries = "";
+ // number of times to read the file
+ int i = 1;
+ for (int j = 0; j < i; j++){
+ readEntries += "{path: \"/tmp/tpc-h/customer\"}";
+ if (j < i - 1)
+ readEntries += ",";
+ }
+ testParquetFullEngineEventBased(false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
+ "/tmp/test.parquet", i, props, true);
+ }
+/*
+ @Test
+ public void testMultipleRowGroupsEvent() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(10, 30000, DEFAULT_BYTES_PER_PAGE, fields);
+ props.fields.put("a", new FieldInfo("a", "asdf", 1, new Object[3], TypeProtos.MinorType.BIGINT, props));
+ testParquetFullEngineEventBased(false, "/parquet_scan_screen.json", "/tmp/out", 1, props);
+ }
+
+*/
+ private class ParquetTestProperties{
+ int numberRowGroups;
+ int recordsPerRowGroup;
+ int bytesPerPage = 1024 * 1024 * 1;
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+
+ public ParquetTestProperties(int numberRowGroups, int recordsPerRowGroup, int bytesPerPage,
+ HashMap<String, FieldInfo> fields){
+ this.numberRowGroups = numberRowGroups;
+ this.recordsPerRowGroup = recordsPerRowGroup;
+ this.bytesPerPage = bytesPerPage;
+ this.fields = fields;
+ }
+
+ }
+
+ private static class FieldInfo {
+
+ String parquetType;
+ String name;
+ int bitLength;
+ int numberOfPages;
+ Object[] values;
+ TypeProtos.MinorType type;
+
+ FieldInfo(String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type, ParquetTestProperties props){
+ this.parquetType = parquetType;
+ this.name = name;
+ this.bitLength = bitLength;
+ this.numberOfPages = Math.max(1, (int) Math.ceil( ((long) props.recordsPerRowGroup) * bitLength / 8.0 / props.bytesPerPage));
+ this.values = values;
+ // generator is designed to use 3 values
+ assert values.length == 3;
+ this.type = type;
+ }
+ }
+
+ private String getResource(String resourceName) {
+ return "resource:" + resourceName;
+ }
+
+ public void generateParquetFile(String filename, ParquetTestProperties props) throws Exception {
+
+ int currentBooleanByte = 0;
+ WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7);
+
+ Configuration configuration = new Configuration();
+ configuration.set("fs.default.name", "file:///");
+ //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
+
+ FileSystem fs = FileSystem.get(configuration);
+ Path path = new Path(filename);
+ if (fs.exists(path)) fs.delete(path, false);
+
+
+ String messageSchema = "message m {";
+ for (FieldInfo fieldInfo : props.fields.values()) {
+ messageSchema += " required " + fieldInfo.parquetType + " " + fieldInfo.name + ";";
+ }
+ // remove the last semicolon, java really needs a join method for strings...
+ // TODO - nvm apparently it requires a semicolon after every field decl, might want to file a bug
+ //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
+ messageSchema += "}";
+
+ MessageType schema = MessageTypeParser.parseMessageType(messageSchema);
+
+ CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
+ ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
+ w.start();
+ HashMap<String, Integer> columnValuesWritten = new HashMap();
+ int valsWritten;
+ for (int k = 0; k < props.numberRowGroups; k++){
+ w.startBlock(1);
+ currentBooleanByte = 0;
+ booleanBitCounter.reset();
+
+ for (FieldInfo fieldInfo : props.fields.values()) {
+
+ if ( ! columnValuesWritten.containsKey(fieldInfo.name)){
+ columnValuesWritten.put((String) fieldInfo.name, 0);
+ valsWritten = 0;
+ } else {
+ valsWritten = columnValuesWritten.get(fieldInfo.name);
+ }
+
+ String[] path1 = {(String) fieldInfo.name};
+ ColumnDescriptor c1 = schema.getColumnDescription(path1);
+
+ w.startColumn(c1, props.recordsPerRowGroup, codec);
+ int valsPerPage = (int) Math.ceil(props.recordsPerRowGroup / (float) fieldInfo.numberOfPages);
+ byte[] bytes;
+ // for variable length binary fields
+ int bytesNeededToEncodeLength = 4;
+ if ((int) fieldInfo.bitLength > 0) {
+ bytes = new byte[(int) Math.ceil(valsPerPage * (int) fieldInfo.bitLength / 8.0)];
+ } else {
+ // the twelve at the end is to account for storing a 4 byte length with each value
+ int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength;
+ // used for the case where there is a number of values in this row group that is not divisible by 3
+ int leftOverBytes = 0;
+ if ( valsPerPage % 3 > 0 ) leftOverBytes += ((byte[])fieldInfo.values[1]).length + bytesNeededToEncodeLength;
+ if ( valsPerPage % 3 > 1 ) leftOverBytes += ((byte[])fieldInfo.values[2]).length + bytesNeededToEncodeLength;
+ bytes = new byte[valsPerPage / 3 * totalValLength + leftOverBytes];
+ }
+ int bytesPerPage = (int) (valsPerPage * ((int) fieldInfo.bitLength / 8.0));
+ int bytesWritten = 0;
+ for (int z = 0; z < (int) fieldInfo.numberOfPages; z++, bytesWritten = 0) {
+ for (int i = 0; i < valsPerPage; i++) {
+ //System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : ""));
+ if (fieldInfo.values[0] instanceof Boolean) {
+
+ bytes[currentBooleanByte] |= bitFields[booleanBitCounter.val] & ((boolean) fieldInfo.values[valsWritten % 3]
+ ? allBitsTrue : allBitsFalse);
+ booleanBitCounter.increment();
+ if (booleanBitCounter.val == 0) {
+ currentBooleanByte++;
+ }
+ valsWritten++;
+ if (currentBooleanByte > bytesPerPage) break;
+ } else {
+ if (fieldInfo.values[valsWritten % 3] instanceof byte[]){
+ System.arraycopy(ByteArrayUtil.toByta(((byte[])fieldInfo.values[valsWritten % 3]).length),
+ 0, bytes, bytesWritten, bytesNeededToEncodeLength);
+ try{
+ System.arraycopy(fieldInfo.values[valsWritten % 3],
+ 0, bytes, bytesWritten + bytesNeededToEncodeLength, ((byte[])fieldInfo.values[valsWritten % 3]).length);
+ }
+ catch (Exception ex){
+ Math.min(4, 5);
+ }
+ bytesWritten += ((byte[])fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength;
+ }
+ else{
+ System.arraycopy( ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]),
+ 0, bytes, i * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8);
+ }
+ valsWritten++;
+ }
+
+ }
+ w.writeDataPage((int) (props.recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length, BytesInput.from(bytes), PLAIN, PLAIN, PLAIN);
+ currentBooleanByte = 0;
+ }
+ w.endColumn();
+ columnValuesWritten.remove((String) fieldInfo.name);
+ columnValuesWritten.put((String) fieldInfo.name, valsWritten);
+ }
+
+ w.endBlock();
+ }
+ w.end(new HashMap<String, String>());
+ logger.debug("Finished generating parquet file.");
+ }
+
+ private class ParquetResultListener implements UserResultsListener {
+ private SettableFuture<Void> future = SettableFuture.create();
+ int count = 0;
+ RecordBatchLoader batchLoader;
+
+ int batchCounter = 1;
+ HashMap<String, Integer> valuesChecked = new HashMap();
+ ParquetTestProperties props;
+
+ ParquetResultListener(RecordBatchLoader batchLoader, ParquetTestProperties props){
+ this.batchLoader = batchLoader;
+ this.props = props;
+ }
+
+ @Override
+ public void submissionFailed(RpcException ex) {
+ logger.debug("Submission failed.", ex);
+ future.setException(ex);
+ }
+
+ @Override
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
+ logger.debug("result arrived in test batch listener.");
+ if(result.getHeader().getIsLastChunk()){
+ future.set(null);
+ }
+ int columnValCounter = 0;
+ int i = 0;
+ FieldInfo currentField;
+ count += result.getHeader().getRowCount();
+ boolean schemaChanged = false;
+ try {
+ schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData());
+ } catch (SchemaChangeException e) {
+ throw new RuntimeException(e);
+ }
+
+ int recordCount = 0;
+ // print headers.
+ if (schemaChanged) {
+ } // do not believe any change is needed for when the schema changes, with the current mock scan use case
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector vv = vw.getValueVector();
+ currentField = props.fields.get(vv.getField().getName());
+ if (VERBOSE_DEBUG){
+ System.out.println("\n" + (String) currentField.name);
+ }
+ if ( ! valuesChecked.containsKey(vv.getField().getName())){
+ valuesChecked.put(vv.getField().getName(), 0);
+ columnValCounter = 0;
+ } else {
+ columnValCounter = valuesChecked.get(vv.getField().getName());
+ }
+ for (int j = 0; j < ((BaseDataValueVector)vv).getAccessor().getValueCount(); j++) {
+ if (VERBOSE_DEBUG){
+ System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
+ }
+ assertField(vv, j, (TypeProtos.MinorType) currentField.type,
+ currentField.values[columnValCounter % 3], (String) currentField.name + "/");
+ columnValCounter++;
+ }
+ if (VERBOSE_DEBUG){
+ System.out.println("\n" + ((BaseDataValueVector)vv).getAccessor().getValueCount());
+ }
+ valuesChecked.remove(vv.getField().getName());
+ valuesChecked.put(vv.getField().getName(), columnValCounter);
+ }
+
+ if (VERBOSE_DEBUG){
+ for (i = 0; i < batchLoader.getRecordCount(); i++) {
+ recordCount++;
+ if (i % 50 == 0){
+ System.out.println();
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(pad(v.getField().getName(), 20) + " ");
+
+ }
+ System.out.println();
+ System.out.println();
+ }
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(pad(v.getAccessor().getObject(i).toString(), 20) + " ");
+ }
+ System.out.println(
+
+ );
+ }
+ }
+ batchCounter++;
+ if(result.getHeader().getIsLastChunk()){
+ future.set(null);
+ }
+ }
+
+ public void getResults() throws RpcException{
+ try{
+ future.get();
+ }catch(Throwable t){
+ throw RpcException.mapException(t);
+ }
+ }
+
+ @Override
+ public void queryIdArrived(QueryId queryId) {
+ }
+ }
+
+
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngineEventBased(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
+ testParquetFullEngineEventBased(generateNew, plan, null, filename, numberOfTimesRead, props, true);
+ }
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngineEventBased(boolean generateNew, String plan, String readEntries, String filename,
+ int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props,
+ boolean runAsLogicalPlan) throws Exception{
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ if (generateNew) generateParquetFile(filename, props);
+
+ DrillConfig config = DrillConfig.create();
+
+ try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+ bit1.run();
+ client.connect();
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
+ ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props);
+ long C = System.nanoTime();
+ String planText = Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8);
+ // substitute in the string for the read entries, allows reuse of the plan file for several tests
+ if (readEntries != null) {
+ planText = planText.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries);
+ }
+ if (runAsLogicalPlan)
+ client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener);
+ else
+ client.runQuery(UserProtos.QueryType.PHYSICAL, planText, resultListener);
+ resultListener.getResults();
+ for (String s : resultListener.valuesChecked.keySet()) {
+ assertEquals("Record count incorrect for column: " + s,
+ props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead, (long) resultListener.valuesChecked.get(s));
+ logger.debug("Column {}, Values read:{}", s, resultListener.valuesChecked.get(s));
+ }
+ long D = System.nanoTime();
+ System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
+ }
+ }
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngine(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
+ testParquetFullEngine(generateNew, plan, null, filename, numberOfTimesRead, props);
+ }
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngine(boolean generateNew, String plan, String readEntries, String filename,
+ int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ if (generateNew) generateParquetFile(filename, props);
+
+ DrillConfig config = DrillConfig.create();
+
+ try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
+ long A = System.nanoTime();
+ bit1.run();
+ long B = System.nanoTime();
+ client.connect();
+ long C = System.nanoTime();
+ List<QueryResultBatch> results;
+ // insert a variable number of reads
+ if (readEntries != null){
+ results = client.runQuery(UserProtos.QueryType.LOGICAL, (Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8).replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries)));
+ }
+ else{
+ results = client.runQuery(UserProtos.QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8));
+ }
+// List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
+ long D = System.nanoTime();
+ System.out.println(String.format("Took %f s to start drillbit", (float)(B-A) / 1E9));
+ System.out.println(String.format("Took %f s to connect", (float)(C-B) / 1E9));
+ System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
+ //List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
+ int count = 0;
+// RecordBatchLoader batchLoader = new RecordBatchLoader(new BootStrapContext(config).getAllocator());
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
+ byte[] bytes;
+
+ int batchCounter = 1;
+ int columnValCounter = 0;
+ int i = 0;
+ FieldInfo currentField;
+ HashMap<String, Integer> valuesChecked = new HashMap();
+ for(QueryResultBatch b : results){
+
+ count += b.getHeader().getRowCount();
+ boolean schemaChanged = batchLoader.load(b.getHeader().getDef(), b.getData());
+
+ int recordCount = 0;
+ // print headers.
+ if (schemaChanged) {
+ } // do not believe any change is needed for when the schema changes, with the current mock scan use case
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector vv = vw.getValueVector();
+ currentField = props.fields.get(vv.getField().getName());
+ if (VERBOSE_DEBUG){
+ System.out.println("\n" + (String) currentField.name);
+ }
+ if ( ! valuesChecked.containsKey(vv.getField().getName())){
+ valuesChecked.put(vv.getField().getName(), 0);
+ columnValCounter = 0;
+ } else {
+ columnValCounter = valuesChecked.get(vv.getField().getName());
+ }
+ for (int j = 0; j < vv.getAccessor().getValueCount(); j++) {
+ if (VERBOSE_DEBUG){
+ System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
+ }
+ assertField(vv, j, currentField.type,
+ currentField.values[columnValCounter % 3], currentField.name + "/");
+ columnValCounter++;
+ }
+ if (VERBOSE_DEBUG){
+ System.out.println("\n" + vv.getAccessor().getValueCount());
+ }
+ valuesChecked.remove(vv.getField().getName());
+ valuesChecked.put(vv.getField().getName(), columnValCounter);
+ }
+
+ if (VERBOSE_DEBUG){
+ for (i = 0; i < batchLoader.getRecordCount(); i++) {
+ recordCount++;
+ if (i % 50 == 0){
+ System.out.println();
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(pad(v.getField().getName(), 20) + " ");
+
+ }
+ System.out.println();
+ System.out.println();
+ }
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(pad(v.getAccessor().getObject(i) + "", 20) + " ");
+ }
+ System.out.println(
+
+ );
+ }
+ }
+ batchCounter++;
+ }
+ for (String s : valuesChecked.keySet()) {
+ assertEquals("Record count incorrect for column: " + s, props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead, (long) valuesChecked.get(s));
+ }
+ assert valuesChecked.keySet().size() > 0;
+ }
+ }
+
+ public String pad(String value, int length) {
+ return pad(value, length, " ");
+ }
+
+ public String pad(String value, int length, String with) {
+ StringBuilder result = new StringBuilder(length);
+ result.append(value);
+
+ while (result.length() < length) {
+ result.insert(0, with);
+ }
+
+ return result.toString();
+ }
+
+ class MockOutputMutator implements OutputMutator {
+ List<MaterializedField> removedFields = Lists.newArrayList();
+ List<ValueVector> addFields = Lists.newArrayList();
+
+ @Override
+ public void removeField(MaterializedField field) throws SchemaChangeException {
+ removedFields.add(field);
+ }
+
+ @Override
+ public void addField(ValueVector vector) throws SchemaChangeException {
+ addFields.add(vector);
+ }
+
+ @Override
+ public void removeAllFields() {
+ addFields.clear();
+ }
+
+ @Override
+ public void setNewSchema() throws SchemaChangeException {
+ }
+
+ List<MaterializedField> getRemovedFields() {
+ return removedFields;
+ }
+
+ List<ValueVector> getAddFields() {
+ return addFields;
+ }
+ }
+
+ private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) {
+ assertField(valueVector, index, expectedMinorType, value, name, 0);
+ }
+
+ private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
+// UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
+// SchemaDefProtos.FieldDef def = metadata.getDef();
+// assertEquals(expectedMinorType, def.getMajorType().getMinorType());
+// assertEquals(name, def.getNameList().get(0).getName());
+// assertEquals(parentFieldId, def.getParentId());
+
+ if (expectedMinorType == TypeProtos.MinorType.MAP) {
+ return;
+ }
+
+ T val = (T) valueVector.getAccessor().getObject(index);
+ if (val instanceof String){
+ assertEquals(value, val);
+ }
+ else if (val instanceof byte[]) {
+ assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
+ } else {
+ assertEquals(value, val);
+ }
+ }
+
+ private class WrapAroundCounter {
+
+ int maxVal;
+ int val;
+
+ public WrapAroundCounter(int maxVal) {
+ this.maxVal = maxVal;
+ }
+
+ public int increment() {
+ val++;
+ if (val > maxVal) {
+ val = 0;
+ }
+ return val;
+ }
+
+ public void reset() {
+ val = 0;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
new file mode 100644
index 0000000..f99721a
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java
@@ -0,0 +1,203 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.HashMap;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.vector.ValueVector;
+
+import com.google.common.base.Strings;
+import com.google.common.util.concurrent.SettableFuture;
+
+public class ParquetResultListener implements UserResultsListener {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetResultListener.class);
+
+ private SettableFuture<Void> future = SettableFuture.create();
+ int count = 0;
+ int totalRecords;
+ RecordBatchLoader batchLoader;
+ boolean testValues;
+
+ int batchCounter = 1;
+ HashMap<String, Integer> valuesChecked = new HashMap();
+ ParquetTestProperties props;
+
+ ParquetResultListener(RecordBatchLoader batchLoader, ParquetTestProperties props, int numberOfTimesRead, boolean testValues){
+ this.batchLoader = batchLoader;
+ this.props = props;
+ this.totalRecords = props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead;
+ this.testValues = testValues;
+ }
+
+ @Override
+ public void submissionFailed(RpcException ex) {
+ logger.debug("Submission failed.", ex);
+ future.setException(ex);
+ }
+
+
+ private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) {
+ assertField(valueVector, index, expectedMinorType, value, name, 0);
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
+
+ if (expectedMinorType == TypeProtos.MinorType.MAP) {
+ return;
+ }
+
+ T val = (T) valueVector.getAccessor().getObject(index);
+ if (val instanceof byte[]) {
+ assertEquals(true, Arrays.equals((byte[]) value, (byte[]) val));
+ } else {
+ assertEquals(value, val);
+ }
+ }
+
+ @Override
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
+ logger.debug("result arrived in test batch listener.");
+ if(result.getHeader().getIsLastChunk()){
+ future.set(null);
+ }
+ int columnValCounter = 0;
+ FieldInfo currentField;
+ count += result.getHeader().getRowCount();
+ boolean schemaChanged = false;
+ try {
+ schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData());
+ } catch (SchemaChangeException e) {
+ throw new RuntimeException(e);
+ }
+
+ int recordCount = 0;
+ // print headers.
+ if (schemaChanged) {
+ } // do not believe any change is needed for when the schema changes, with the current mock scan use case
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector vv = vw.getValueVector();
+ currentField = props.fields.get(vv.getField().getName());
+ if (ParquetRecordReaderTest.VERBOSE_DEBUG){
+ System.out.println("\n" + (String) currentField.name);
+ }
+ if ( ! valuesChecked.containsKey(vv.getField().getName())){
+ valuesChecked.put(vv.getField().getName(), 0);
+ columnValCounter = 0;
+ } else {
+ columnValCounter = valuesChecked.get(vv.getField().getName());
+ }
+ for (int j = 0; j < vv.getAccessor().getValueCount(); j++) {
+ if (ParquetRecordReaderTest.VERBOSE_DEBUG){
+ if (vv.getAccessor().getObject(j) instanceof byte[]){
+ System.out.print("[len:" + ((byte[]) vv.getAccessor().getObject(j)).length + " - (");
+ for (int k = 0; k < ((byte[]) vv.getAccessor().getObject(j)).length; k++){
+ System.out.print((char)((byte[])vv.getAccessor().getObject(j))[k] + ",");
+ }
+ System.out.print(") ]");
+ }
+ else{
+ System.out.print(Strings.padStart(vv.getAccessor().getObject(j) + "", 20, ' ') + " ");
+ }
+ System.out.print(", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
+ }
+ if (testValues){
+ assertField(vv, j, currentField.type,
+ currentField.values[columnValCounter % 3], currentField.name + "/");
+ }
+ columnValCounter++;
+ }
+ if (ParquetRecordReaderTest.VERBOSE_DEBUG){
+ System.out.println("\n" + vv.getAccessor().getValueCount());
+ }
+ valuesChecked.remove(vv.getField().getName());
+ valuesChecked.put(vv.getField().getName(), columnValCounter);
+ }
+
+ if (ParquetRecordReaderTest.VERBOSE_DEBUG){
+ for (int i = 0; i < batchLoader.getRecordCount(); i++) {
+ recordCount++;
+ if (i % 50 == 0){
+ System.out.println();
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(Strings.padStart(v.getField().getName(), 20, ' ') + " ");
+
+ }
+ System.out.println();
+ System.out.println();
+ }
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ if (v.getAccessor().getObject(i) instanceof byte[]){
+ System.out.print("[len:" + ((byte[]) v.getAccessor().getObject(i)).length + " - (");
+ for (int j = 0; j < ((byte[]) v.getAccessor().getObject(i)).length; j++){
+ System.out.print(((byte[])v.getAccessor().getObject(i))[j] + ",");
+ }
+ System.out.print(") ]");
+ }
+ else{
+ System.out.print(Strings.padStart(v.getAccessor().getObject(i) + "", 20, ' ') + " ");
+ }
+ }
+ System.out.println(
+
+ );
+ }
+ }
+ batchCounter++;
+ if(result.getHeader().getIsLastChunk()){
+ // ensure the right number of columns was returned, especially important to ensure selective column read is working
+ assert valuesChecked.keySet().size() == props.fields.keySet().size() : "Unexpected number of output columns from parquet scan,";
+ for (String s : valuesChecked.keySet()) {
+ try {
+ assertEquals("Record count incorrect for column: " + s, totalRecords, (long) valuesChecked.get(s));
+ } catch (AssertionError e) { submissionFailed(new RpcException(e)); }
+ }
+
+ assert valuesChecked.keySet().size() > 0;
+ future.set(null);
+ }
+ }
+
+ public void getResults() throws RpcException{
+ try{
+ future.get();
+ }catch(Throwable t){
+ throw RpcException.mapException(t);
+ }
+ }
+
+ @Override
+ public void queryIdArrived(UserBitShared.QueryId queryId) {
+ }
+}
[02/38] git commit: DRILL-357: Hive Storage Engine phase 2 - hive
record reader
Posted by ja...@apache.org.
DRILL-357: Hive Storage Engine phase 2 - hive record reader
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/cdf46fd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/cdf46fd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/cdf46fd3
Branch: refs/heads/master
Commit: cdf46fd36fdfc2e3029a6b2e077330c665e43c2e
Parents: a9a7ea8
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Jan 23 18:32:03 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:21:50 2014 -0800
----------------------------------------------------------------------
.../apache/drill/common/config/DrillConfig.java | 2 +-
distribution/src/assemble/bin.xml | 1 +
exec/java-exec/pom.xml | 11 +
.../templates/VariableLengthVectors.java | 38 +-
.../exec/compile/TemplateClassDefinition.java | 2 +-
.../drill/exec/planner/fragment/Wrapper.java | 3 +
.../drill/exec/store/hive/HiveInputReader.java | 173 +++++++
.../drill/exec/store/hive/HiveReadEntry.java | 47 +-
.../drill/exec/store/hive/HiveReadEntryOld.java | 50 ++
.../drill/exec/store/hive/HiveRecordReader.java | 473 +++++++++++++++++++
.../apache/drill/exec/store/hive/HiveScan.java | 247 ++++++++++
.../exec/store/hive/HiveScanBatchCreator.java | 71 +++
.../exec/store/hive/HiveSchemaProvider.java | 39 --
.../exec/store/hive/HiveStorageEngine.java | 194 ++++++++
.../store/hive/HiveStorageEngineConfig.java | 6 +-
.../drill/exec/store/hive/HiveSubScan.java | 124 +++++
.../apache/drill/exec/store/hive/HiveTable.java | 325 +++++++++++++
.../exec/store/hive/HiveTextRecordReader.java | 172 +++++++
.../store/parquet/ParquetSchemaProvider.java | 1 -
.../apache/drill/exec/util/BatchPrinter.java | 93 ++++
.../src/main/resources/drill-module.conf | 3 +-
.../java/org/apache/drill/exec/TestPlan.java | 48 ++
.../drill/exec/store/hive/TestHiveScan.java | 67 +++
.../apache/drill/exec/util/BatchPrinter.java | 93 ----
.../java-exec/src/test/resources/hive/test.json | 75 +++
pom.xml | 51 ++
.../org/apache/drill/jdbc/DrillHandler.java | 2 +-
.../drill/sql/client/full/BatchLoaderMap.java | 2 +-
.../sql/client/full/HiveDatabaseSchema.java | 115 +++--
.../drill/sql/client/full/HiveSchema.java | 5 +-
.../apache/drill/jdbc/test/FullEngineTest.java | 1 +
31 files changed, 2349 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index d81ef79..a0e261f 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -64,7 +64,7 @@ public final class DrillConfig extends NestedConfig{
mapper.registerModule(deserModule);
mapper.enable(SerializationFeature.INDENT_OUTPUT);
mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
- mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, false);
+ mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
mapper.configure(Feature.ALLOW_COMMENTS, true);
mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this));
mapper.registerSubtypes(StorageEngineConfigBase.getSubTypes(this));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index 5538d40..c50df09 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -22,6 +22,7 @@
<id>binary-release</id>
<formats>
<format>tar.gz</format>
+ <format>dir</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<moduleSets>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 2cc3fb4..9ea8304 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -237,6 +237,17 @@
<artifactId>maprfs</artifactId>
<version>1.0.3-mapr-3.0.0</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.13-mapr-1401-m7-3.0.2</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</profile>
<profile>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index 57927a7..b059d89 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -260,7 +260,43 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
assert index >= 0;
int currentOffset = offsetVector.getAccessor().get(index);
offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
- data.setBytes(currentOffset, bytes);
+ data.setBytes(currentOffset, bytes, 0, bytes.length);
+ }
+
+ public boolean setSafe(int index, byte[] bytes) {
+ assert index >= 0;
+ int currentOffset = offsetVector.getAccessor().get(index);
+ if (data.capacity() < currentOffset + bytes.length) return false;
+ offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
+ data.setBytes(currentOffset, bytes, 0, bytes.length);
+ return true;
+ }
+
+ /**
+ * Set the variable length element at the specified index to the supplied byte array.
+ *
+ * @param index position of the bit to set
+ * @param bytes array of bytes to write
+ * @param start start index of bytes to write
+ * @param length length of bytes to write
+ */
+ public void set(int index, byte[] bytes, int start, int length) {
+ assert index >= 0;
+ int currentOffset = offsetVector.getAccessor().get(index);
+ offsetVector.getMutator().set(index + 1, currentOffset + length);
+ data.setBytes(currentOffset, bytes, start, length);
+ }
+
+ public boolean setSafe(int index, byte[] bytes, int start, int length) {
+ assert index >= 0;
+
+ int currentOffset = offsetVector.getAccessor().get(index);
+
+ if (data.capacity() < currentOffset + length) return false;
+
+ offsetVector.getMutator().set(index + 1, currentOffset + length);
+ data.setBytes(currentOffset, bytes, start, length);
+ return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
index e9946df..688e680 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java
@@ -28,7 +28,7 @@ public class TemplateClassDefinition<T>{
private final Class<T> iface;
private final Class<?> template;
private final SignatureHolder signature;
- private final AtomicLong classNumber = new AtomicLong(0);
+ private static final AtomicLong classNumber = new AtomicLong(0);
public <X extends T> TemplateClassDefinition(Class<T> iface, Class<X> template) {
super();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
index 8c1487c..94fcac5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
@@ -72,6 +72,7 @@ public class Wrapper {
public void addEndpointAffinity(DrillbitEndpoint endpoint, float affinity) {
Preconditions.checkState(!endpointsAssigned);
+ Preconditions.checkNotNull(endpoint);
EndpointAffinity ea = endpointAffinity.get(endpoint);
if (ea == null) {
ea = new EndpointAffinity(endpoint);
@@ -149,6 +150,7 @@ public class Wrapper {
int start = ThreadLocalRandom.current().nextInt(div);
// round robin with random start.
for (int i = start; i < start + width; i++) {
+ Preconditions.checkNotNull(all.get(i % div));
endpoints.add(all.get(i % div));
}
} else {
@@ -156,6 +158,7 @@ public class Wrapper {
Collections.sort(values);
values = Lists.reverse(values);
for (int i = 0; i < width; i++) {
+ Preconditions.checkNotNull(values.get(i%values.size()).getEndpoint());
endpoints.add(values.get(i%values.size()).getEndpoint());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
new file mode 100644
index 0000000..8809706
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveInputReader.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.google.common.collect.Lists;
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.drill.common.util.DataOutputOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.hbase.HBaseSerDe;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.serializer.WritableSerialization;
+import org.apache.hadoop.mapred.*;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.Constructor;
+import java.util.*;
+
+public class HiveInputReader {
+ public static void main(String args[]) throws Exception {
+/*
+ String[] columnNames = {"n_nationkey", "n_name", "n_regionkey", "n_comment"};
+ String[] columnTypes = {"bigint", "string", "bigint", "string"};
+
+ List<FieldSchema> cols = Lists.newArrayList();
+
+ for (int i = 0; i < columnNames.length; i++) {
+ cols.add(new FieldSchema(columnNames[i], columnTypes[i], null));
+ }
+ String location = "file:///tmp/nation_s";
+ String inputFormat = TextInputFormat.class.getCanonicalName();
+ String serdeLib = LazySimpleSerDe.class.getCanonicalName();
+// String inputFormat = HiveHBaseTableInputFormat.class.getCanonicalName();
+// String serdeLib = HBaseSerDe.class.getCanonicalName();
+ Map<String, String> serdeParams = new HashMap();
+// serdeParams.put("serialization.format", "1");
+// serdeParams.put("hbase.columns.mapping", ":key,f:name,f:regionkey,f:comment");
+ serdeParams.put("serialization.format", "|");
+ serdeParams.put("field.delim", "|");
+
+
+ Map<String, String> tableParams = new HashMap();
+ tableParams.put("hbase.table.name", "nation");
+ SerDeInfo serDeInfo = new SerDeInfo(null, serdeLib, serdeParams);
+ StorageDescriptor storageDescriptor = new StorageDescriptor(cols, location, inputFormat, null, false, -1, serDeInfo, null, null, null);
+ Table table = new Table("table", "default", "sphillips", 0, 0, 0, storageDescriptor, new ArrayList<FieldSchema>(), tableParams, null, null, "MANAGED_TABLE");
+ Properties properties = MetaStoreUtils.getTableMetadata(table);
+ */
+
+ HiveConf conf = new HiveConf();
+ conf.set("hive.metastore.uris", "thrift://10.10.31.51:9083");
+ HiveMetaStoreClient client = new HiveMetaStoreClient(conf);
+ Table table = client.getTable("default", "nation");
+ Properties properties = MetaStoreUtils.getTableMetadata(table);
+
+ Path path = new Path(table.getSd().getLocation());
+ JobConf job = new JobConf();
+ for (Object obj : properties.keySet()) {
+ job.set((String) obj, (String) properties.get(obj));
+ }
+// job.set("hbase.zookeeper.quorum", "10.10.31.51");
+// job.set("hbase.zookeeper.property.clientPort", "5181");
+ InputFormat f = (InputFormat) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance();
+ job.setInputFormat(f.getClass());
+ FileInputFormat.addInputPath(job, path);
+ InputFormat format = job.getInputFormat();
+ SerDe serde = (SerDe) Class.forName(table.getSd().getSerdeInfo().getSerializationLib()).getConstructor().newInstance();
+ serde.initialize(job, properties);
+ ObjectInspector inspector = serde.getObjectInspector();
+ ObjectInspector.Category cat = inspector.getCategory();
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(inspector);
+ List<String> columns = null;
+ List<TypeInfo> colTypes = null;
+ List<ObjectInspector> fieldObjectInspectors = Lists.newArrayList();
+
+ switch(typeInfo.getCategory()) {
+ case STRUCT:
+ columns = ((StructTypeInfo) typeInfo).getAllStructFieldNames();
+ colTypes = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ for (int i = 0; i < columns.size(); i++) {
+ System.out.print(columns.get(i));
+ System.out.print(" ");
+ System.out.print(colTypes.get(i));
+ }
+ System.out.println("");
+ for (StructField field : ((StructObjectInspector)inspector).getAllStructFieldRefs()) {
+ fieldObjectInspectors.add(field.getFieldObjectInspector());
+ }
+ }
+
+ for (InputSplit split : format.getSplits(job, 1)) {
+ String encoded = serializeInputSplit(split);
+ System.out.println(encoded);
+ InputSplit newSplit = deserializeInputSplit(encoded, split.getClass().getCanonicalName());
+ System.out.print("Length: " + newSplit.getLength() + " ");
+ System.out.print("Locations: ");
+ for (String loc : newSplit.getLocations()) System.out.print(loc + " " );
+ System.out.println();
+ }
+
+ for (InputSplit split : format.getSplits(job, 1)) {
+ RecordReader reader = format.getRecordReader(split, job, Reporter.NULL);
+ Object key = reader.createKey();
+ Object value = reader.createValue();
+ int count = 0;
+ while (reader.next(key, value)) {
+ List<Object> values = ((StructObjectInspector) inspector).getStructFieldsDataAsList(serde.deserialize((Writable) value));
+ StructObjectInspector sInsp = (StructObjectInspector) inspector;
+ Object obj = sInsp.getStructFieldData(serde.deserialize((Writable) value) , sInsp.getStructFieldRef("n_name"));
+ System.out.println(obj);
+ /*
+ for (Object obj : values) {
+ PrimitiveObjectInspector.PrimitiveCategory pCat = ((PrimitiveObjectInspector)fieldObjectInspectors.get(count)).getPrimitiveCategory();
+ Object pObj = ((PrimitiveObjectInspector)fieldObjectInspectors.get(count)).getPrimitiveJavaObject(obj);
+ System.out.print(pObj + " ");
+ }
+ */
+ System.out.println("");
+ }
+ }
+ }
+
+ public static String serializeInputSplit(InputSplit split) throws IOException {
+ ByteArrayDataOutput byteArrayOutputStream = ByteStreams.newDataOutput();
+ split.write(byteArrayOutputStream);
+ return Base64.encodeBase64String(byteArrayOutputStream.toByteArray());
+ }
+
+ public static InputSplit deserializeInputSplit(String base64, String className) throws Exception {
+ InputSplit split;
+ if (Class.forName(className) == FileSplit.class) {
+ split = new FileSplit((Path) null, 0, 0, (String[])null);
+ } else {
+ split = (InputSplit) Class.forName(className).getConstructor().newInstance();
+ }
+ ByteArrayDataInput byteArrayDataInput = ByteStreams.newDataInput(Base64.decodeBase64(base64));
+ split.readFields(byteArrayDataInput);
+ return split;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
index 41a4d3d..6211e21 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
@@ -17,19 +17,47 @@
*/
package org.apache.drill.exec.store.hive;
+import com.beust.jcommander.internal.Lists;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.ReadEntry;
import org.apache.drill.exec.physical.base.Size;
-import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+import java.util.List;
public class HiveReadEntry implements ReadEntry {
- private final HiveConf conf;
- private final String table;
- private Size size;
- public HiveReadEntry(HiveConf conf, String table) {
- this.conf = conf;
+ @JsonProperty("table")
+ public HiveTable table;
+ @JsonProperty("partitions")
+ public List<HiveTable.HivePartition> partitions;
+
+ @JsonIgnore
+ private List<Partition> partitionsUnwrapped = Lists.newArrayList();
+
+ @JsonCreator
+ public HiveReadEntry(@JsonProperty("table") HiveTable table, @JsonProperty("partitions") List<HiveTable.HivePartition> partitions) {
this.table = table;
+ this.partitions = partitions;
+ if (partitions != null) {
+ for(HiveTable.HivePartition part : partitions) {
+ partitionsUnwrapped.add(part.getPartition());
+ }
+ }
+ }
+
+ @JsonIgnore
+ public Table getTable() {
+ return table.getTable();
+ }
+
+ @JsonIgnore
+ public List<Partition> getPartitions() {
+ return partitionsUnwrapped;
}
@Override
@@ -40,11 +68,10 @@ public class HiveReadEntry implements ReadEntry {
@Override
public Size getSize() {
- if (size != null) {
- // TODO: contact the metastore and find the size of the data in table
- size = new Size(1, 1);
- }
+ // TODO: contact the metastore and find the size of the data in table
+ Size size = new Size(1, 1);
return size;
}
}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java
new file mode 100644
index 0000000..ef7266c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.ReadEntry;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+public class HiveReadEntryOld implements ReadEntry {
+ private final HiveConf conf;
+ private final String table;
+ private Size size;
+
+ public HiveReadEntryOld(HiveConf conf, String table) {
+ this.conf = conf;
+ this.table = table;
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ // TODO: need to come up with way to calculate the cost for Hive tables
+ return new OperatorCost(1, 1, 2, 2);
+ }
+
+ @Override
+ public Size getSize() {
+ if (size != null) {
+ // TODO: contact the metastore and find the size of the data in table
+ size = new Size(1, 1);
+ }
+
+ return size;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
new file mode 100644
index 0000000..0a31a12
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
@@ -0,0 +1,473 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.google.common.collect.Lists;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reporter;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class HiveRecordReader implements RecordReader {
+
+ protected Table table;
+ protected Partition partition;
+ protected InputSplit inputSplit;
+ protected FragmentContext context;
+ protected List<FieldReference> columns;
+ protected List<String> columnNames;
+ protected List<String> partitionNames = Lists.newArrayList();
+ protected List<String> selectedPartitionNames = Lists.newArrayList();
+ protected List<String> selectedPartitionTypes = Lists.newArrayList();
+ protected List<String> tableColumns;
+ protected SerDe serde;
+ protected StructObjectInspector sInspector;
+ protected List<PrimitiveObjectInspector> fieldInspectors = Lists.newArrayList();
+ protected List<PrimitiveCategory> primitiveCategories = Lists.newArrayList();
+ protected Object key, value;
+ protected org.apache.hadoop.mapred.RecordReader reader;
+ protected List<ValueVector> vectors = Lists.newArrayList();
+ protected List<ValueVector> pVectors = Lists.newArrayList();
+ protected Object redoRecord;
+ List<Object> partitionValues = Lists.newArrayList();
+
+ protected static final int TARGET_RECORD_COUNT = 4000;
+
+ public HiveRecordReader(Table table, Partition partition, InputSplit inputSplit, List<FieldReference> columns, FragmentContext context) throws ExecutionSetupException {
+ this.table = table;
+ this.partition = partition;
+ this.inputSplit = inputSplit;
+ this.context = context;
+ this.columns = columns;
+ init();
+ }
+
+ private void init() throws ExecutionSetupException {
+ Properties properties;
+ JobConf job = new JobConf();
+ if (partition != null) {
+ properties = MetaStoreUtils.getPartitionMetadata(partition, table);
+ } else {
+ properties = MetaStoreUtils.getTableMetadata(table);
+ }
+ for (Object obj : properties.keySet()) {
+ job.set((String) obj, (String) properties.get(obj));
+ }
+ InputFormat format;
+ String sLib = (partition == null) ? table.getSd().getSerdeInfo().getSerializationLib() : partition.getSd().getSerdeInfo().getSerializationLib();
+ String inputFormatName = (partition == null) ? table.getSd().getInputFormat() : partition.getSd().getInputFormat();
+ try {
+ format = (InputFormat) Class.forName(inputFormatName).getConstructor().newInstance();
+ Class c = Class.forName(sLib);
+ serde = (SerDe) c.getConstructor().newInstance();
+ serde.initialize(job, properties);
+ } catch (ReflectiveOperationException | SerDeException e) {
+ throw new ExecutionSetupException("Unable to instantiate InputFormat", e);
+ }
+ job.setInputFormat(format.getClass());
+
+ if (partition != null) {
+ List<FieldSchema> partitionKeys = table.getPartitionKeys();
+ for (FieldSchema field : partitionKeys) {
+ partitionNames.add(field.getName());
+ }
+ }
+
+ try {
+ ObjectInspector oi = serde.getObjectInspector();
+ if (oi.getCategory() != ObjectInspector.Category.STRUCT) {
+ throw new UnsupportedOperationException(String.format("%s category not supported", oi.getCategory()));
+ }
+ sInspector = (StructObjectInspector) oi;
+ StructTypeInfo sTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(sInspector);
+ if (columns == null) {
+ columnNames = sTypeInfo.getAllStructFieldNames();
+ tableColumns = columnNames;
+ } else {
+ tableColumns = sTypeInfo.getAllStructFieldNames();
+ List<Integer> columnIds = Lists.newArrayList();
+ columnNames = Lists.newArrayList();
+ for (FieldReference field : columns) {
+ String columnName = field.getPath().toString();
+ if (!tableColumns.contains(columnName)) {
+ if (partition != null && partitionNames.contains(columnName)) {
+ selectedPartitionNames.add(columnName);
+ } else {
+ throw new ExecutionSetupException(String.format("Column %s does not exist", columnName));
+ }
+ } else {
+ columnIds.add(tableColumns.indexOf(columnName));
+ columnNames.add(columnName);
+ }
+ }
+ ColumnProjectionUtils.appendReadColumnIDs(job, columnIds);
+ ColumnProjectionUtils.appendReadColumnNames(job, columnNames);
+ }
+ for (String columnName : columnNames) {
+ ObjectInspector poi = sInspector.getStructFieldRef(columnName).getFieldObjectInspector();
+ if(poi.getCategory() != ObjectInspector.Category.PRIMITIVE) {
+ throw new UnsupportedOperationException(String.format("%s type not supported", poi.getCategory()));
+ }
+ PrimitiveObjectInspector pInspector = (PrimitiveObjectInspector) poi;
+ fieldInspectors.add(pInspector);
+ primitiveCategories.add(pInspector.getPrimitiveCategory());
+ }
+
+ if (columns == null) {
+ selectedPartitionNames = partitionNames;
+ }
+
+ if (partition != null) {
+ for (int i = 0; i < table.getPartitionKeys().size(); i++) {
+ FieldSchema field = table.getPartitionKeys().get(i);
+ if (selectedPartitionNames.contains(field.getName())) {
+ selectedPartitionTypes.add(field.getType());
+ partitionValues.add(convertPartitionType(field.getType(), partition.getValues().get(i)));
+ }
+ }
+ }
+ } catch (SerDeException e) {
+ throw new ExecutionSetupException(e);
+ }
+ try {
+ reader = format.getRecordReader(inputSplit, job, Reporter.NULL);
+ } catch (IOException e) {
+ throw new ExecutionSetupException("Failed to get Recordreader", e);
+ }
+ key = reader.createKey();
+ value = reader.createValue();
+ }
+
+ @Override
+ public void setup(OutputMutator output) throws ExecutionSetupException {
+ output.removeAllFields();
+ try {
+ for (int i = 0; i < columnNames.size(); i++) {
+ PrimitiveCategory pCat = primitiveCategories.get(i);
+ MaterializedField field = MaterializedField.create(new SchemaPath(columnNames.get(i), ExpressionPosition.UNKNOWN), getMajorType(pCat));
+ ValueVector vv = TypeHelper.getNewVector(field, context.getAllocator());
+ vectors.add(vv);
+ output.addField(vv);
+ }
+ for (int i = 0; i < selectedPartitionNames.size(); i++) {
+ String type = selectedPartitionTypes.get(i);
+ MaterializedField field = MaterializedField.create(new SchemaPath(selectedPartitionNames.get(i), ExpressionPosition.UNKNOWN), getMajorType(type));
+ ValueVector vv = TypeHelper.getNewVector(field, context.getAllocator());
+ pVectors.add(vv);
+ output.addField(vv);
+ }
+ output.setNewSchema();
+ } catch(SchemaChangeException e) {
+ throw new ExecutionSetupException(e);
+ }
+ }
+
+ protected void populatePartitionVectors(int recordCount) {
+ for (int i = 0; i < pVectors.size(); i++) {
+ int size = 50;
+ ValueVector vector = pVectors.get(i);
+ Object val = partitionValues.get(i);
+ if (selectedPartitionTypes.get(i).equals("string") || selectedPartitionTypes.get(i).equals("binary")) {
+ size = ((byte[]) partitionValues.get(i)).length;
+ }
+ VectorAllocator.getAllocator(vector, size).alloc(recordCount);
+ switch(selectedPartitionTypes.get(i)) {
+ case "boolean": {
+ BitVector v = (BitVector) vector;
+ Boolean value = (Boolean) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().set(j, value ? 1 : 0);
+ }
+ break;
+ }
+ case "tinyint": {
+ TinyIntVector v = (TinyIntVector) vector;
+ byte value = (byte) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().set(j, value);
+ }
+ break;
+ }
+ case "double": {
+ Float8Vector v = (Float8Vector) vector;
+ double value = (double) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().set(j, value);
+ }
+ break;
+ }
+ case "float": {
+ Float4Vector v = (Float4Vector) vector;
+ float value = (float) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().set(j, value);
+ }
+ break;
+ }
+ case "int": {
+ IntVector v = (IntVector) vector;
+ int value = (int) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().set(j, value);
+ }
+ break;
+ }
+ case "bigint": {
+ BigIntVector v = (BigIntVector) vector;
+ long value = (long) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().set(j, value);
+ }
+ break;
+ }
+ case "smallint": {
+ SmallIntVector v = (SmallIntVector) vector;
+ short value = (short) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().set(j, value);
+ }
+ break;
+ }
+ case "string": {
+ VarCharVector v = (VarCharVector) vector;
+ byte[] value = (byte[]) val;
+ for (int j = 0; j < recordCount; j++) {
+ v.getMutator().set(j, value);
+ }
+ break;
+ }
+ default:
+ throw new UnsupportedOperationException("Could not determine type: " + selectedPartitionTypes.get(i));
+ }
+ vector.getMutator().setValueCount(recordCount);
+ }
+ }
+
+ private Object convertPartitionType(String type, String value) {
+ switch (type) {
+ case "boolean":
+ return Boolean.parseBoolean(value);
+ case "tinyint":
+ return Byte.parseByte(value);
+ case "double":
+ return Double.parseDouble(value);
+ case "float":
+ return Float.parseFloat(value);
+ case "int":
+ return Integer.parseInt(value);
+ case "bigint":
+ return Long.parseLong(value);
+ case "smallint":
+ return Short.parseShort(value);
+ case "string":
+ return value.getBytes();
+ default:
+ throw new UnsupportedOperationException("Could not determine type: " + type);
+ }
+ }
+
+ public static TypeProtos.MajorType getMajorType(String type) {
+ switch(type) {
+ case "binary":
+ return Types.required(TypeProtos.MinorType.VARBINARY);
+ case "boolean":
+ return Types.required(TypeProtos.MinorType.BIT);
+ case "tinyint":
+ return Types.required(TypeProtos.MinorType.TINYINT);
+ case "decimal":
+ return Types.required(TypeProtos.MinorType.DECIMAL16);
+ case "double":
+ return Types.required(TypeProtos.MinorType.FLOAT8);
+ case "float":
+ return Types.required(TypeProtos.MinorType.FLOAT4);
+ case "int":
+ return Types.required(TypeProtos.MinorType.INT);
+ case "bigint":
+ return Types.required(TypeProtos.MinorType.BIGINT);
+ case "smallint":
+ return Types.required(TypeProtos.MinorType.SMALLINT);
+ case "string":
+ return Types.required(TypeProtos.MinorType.VARCHAR);
+ case "varchar":
+
+ default:
+ throw new UnsupportedOperationException("Could not determine type: " + type);
+ }
+ }
+
+ public static TypeProtos.MajorType getMajorType(PrimitiveCategory pCat) {
+ switch(pCat) {
+ case BINARY:
+ return Types.required(TypeProtos.MinorType.VARBINARY);
+ case BOOLEAN:
+ return Types.required(TypeProtos.MinorType.BIT);
+ case BYTE:
+ return Types.required(TypeProtos.MinorType.TINYINT);
+ case DECIMAL:
+ return Types.required(TypeProtos.MinorType.DECIMAL16);
+ case DOUBLE:
+ return Types.required(TypeProtos.MinorType.FLOAT8);
+ case FLOAT:
+ return Types.required(TypeProtos.MinorType.FLOAT4);
+ case INT:
+ return Types.required(TypeProtos.MinorType.INT);
+ case LONG:
+ return Types.required(TypeProtos.MinorType.BIGINT);
+ case SHORT:
+ return Types.required(TypeProtos.MinorType.SMALLINT);
+ case STRING:
+ return Types.required(TypeProtos.MinorType.VARCHAR);
+ case TIMESTAMP:
+
+ default:
+ throw new UnsupportedOperationException("Could not determine type");
+ }
+ }
+
+ public boolean setValue(PrimitiveCategory pCat, ValueVector vv, int index, Object fieldValue) {
+ switch(pCat) {
+ case BINARY:
+ ((VarBinaryVector) vv).getMutator().setSafe(index, (byte[]) fieldValue);
+ case BOOLEAN:
+ boolean isSet = (boolean) fieldValue;
+ return ((BitVector) vv).getMutator().setSafe(index, isSet ? 1 : 0 );
+ case BYTE:
+ return ((TinyIntVector) vv).getMutator().setSafe(index, (byte) fieldValue);
+ case DECIMAL:
+ throw new UnsupportedOperationException();
+ case DOUBLE:
+ return ((Float8Vector) vv).getMutator().setSafe(index, (double) fieldValue);
+ case FLOAT:
+ return ((Float4Vector) vv).getMutator().setSafe(index, (float) fieldValue);
+ case INT:
+ return ((IntVector) vv).getMutator().setSafe(index, (int) fieldValue);
+ case LONG:
+ return ((BigIntVector) vv).getMutator().setSafe(index, (long) fieldValue);
+ case SHORT:
+ return ((SmallIntVector) vv).getMutator().setSafe(index, (short) fieldValue);
+ case STRING:
+ int len = ((Text) fieldValue).getLength();
+ byte[] bytes = ((Text) fieldValue).getBytes();
+ return ((VarCharVector) vv).getMutator().setSafe(index, bytes, 0, len);
+ case TIMESTAMP:
+ throw new UnsupportedOperationException();
+
+ default:
+ throw new UnsupportedOperationException("Could not determine type");
+ }
+ }
+
+ @Override
+ public int next() {
+ for (ValueVector vv : vectors) {
+ VectorAllocator.getAllocator(vv, 50).alloc(TARGET_RECORD_COUNT);
+ }
+ try {
+ int recordCount = 0;
+ if (redoRecord != null) {
+ Object deSerializedValue = serde.deserialize((Writable) redoRecord);
+ for (int i = 0; i < columnNames.size(); i++) {
+ Object obj;
+ String columnName = columnNames.get(i);
+ if (primitiveCategories.get(i) == PrimitiveCategory.STRING) {
+ obj = fieldInspectors.get(i).getPrimitiveWritableObject(sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName)));
+ } else {
+ obj = fieldInspectors.get(i).getPrimitiveJavaObject(sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName)));
+ }
+ boolean success = setValue(primitiveCategories.get(i), vectors.get(i), recordCount, obj);
+ if (!success) {
+ throw new DrillRuntimeException(String.format("Failed to write value for column %s", columnName));
+ }
+ }
+ redoRecord = null;
+ recordCount++;
+ }
+ while (recordCount < TARGET_RECORD_COUNT && reader.next(key, value)) {
+ Object deSerializedValue = serde.deserialize((Writable) value);
+ for (int i = 0; i < columnNames.size(); i++) {
+ Object obj;
+ String columnName = columnNames.get(i);
+ if (primitiveCategories.get(i) == PrimitiveCategory.STRING) {
+ obj = fieldInspectors.get(i).getPrimitiveWritableObject(sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName)));
+ } else {
+ obj = fieldInspectors.get(i).getPrimitiveJavaObject(sInspector.getStructFieldData(deSerializedValue, sInspector.getStructFieldRef(columnName)));
+ }
+ boolean success = setValue(primitiveCategories.get(i), vectors.get(i), recordCount, obj);
+ if (!success) {
+ redoRecord = value;
+ if (partition != null) populatePartitionVectors(recordCount);
+ for (ValueVector v : vectors) {
+ v.getMutator().setValueCount(recordCount);
+ }
+ if (partition != null) populatePartitionVectors(recordCount);
+ return recordCount;
+ }
+ }
+ recordCount++;
+ }
+ for (ValueVector v : vectors) {
+ v.getMutator().setValueCount(recordCount);
+ }
+ if (partition != null) populatePartitionVectors(recordCount);
+ return recordCount;
+ } catch (IOException | SerDeException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
new file mode 100644
index 0000000..bc2a16b
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.fasterxml.jackson.annotation.*;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.physical.EndpointAffinity;
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.SubScan;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+
+import java.io.IOException;
+import java.util.*;
+
+@JsonTypeName("hive-scan")
+public class HiveScan extends AbstractGroupScan {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveScan.class);
+
+ @JsonProperty("hive-table")
+ public HiveReadEntry hiveReadEntry;
+ @JsonIgnore
+ private Table table;
+ @JsonIgnore
+ private List<InputSplit> inputSplits = Lists.newArrayList();
+ @JsonIgnore
+ public HiveStorageEngine storageEngine;
+ @JsonProperty("storageengine")
+ public HiveStorageEngineConfig engineConfig;
+
+ @JsonIgnore
+ public List<Partition> partitions;
+ @JsonIgnore
+ private Collection<DrillbitEndpoint> endpoints;
+
+ @JsonProperty("columns")
+ public List<FieldReference> columns;
+
+ @JsonIgnore
+ List<List<InputSplit>> mappings;
+
+ @JsonIgnore
+ Map<InputSplit, Partition> partitionMap = new HashMap();
+
+ @JsonCreator
+ public HiveScan(@JsonProperty("hive-table") HiveReadEntry hiveReadEntry, @JsonProperty("storageengine") HiveStorageEngineConfig config,
+ @JsonProperty("columns") List<FieldReference> columns,
+ @JacksonInject StorageEngineRegistry engineRegistry) throws ExecutionSetupException {
+ this.hiveReadEntry = hiveReadEntry;
+ this.table = hiveReadEntry.getTable();
+ this.engineConfig = config;
+ this.storageEngine = (HiveStorageEngine) engineRegistry.getEngine(config);
+ this.columns = columns;
+ this.partitions = hiveReadEntry.getPartitions();
+ getSplits();
+ endpoints = storageEngine.getContext().getBits();
+ }
+
+ public HiveScan(HiveReadEntry hiveReadEntry, HiveStorageEngine storageEngine, List<FieldReference> columns) throws ExecutionSetupException {
+ this.table = hiveReadEntry.getTable();
+ this.hiveReadEntry = hiveReadEntry;
+ this.columns = columns;
+ this.partitions = hiveReadEntry.getPartitions();
+ getSplits();
+ endpoints = storageEngine.getContext().getBits();
+ this.engineConfig = storageEngine.getConfig();
+ }
+
+ public List<FieldReference> getColumns() {
+ return columns;
+ }
+
+ private void getSplits() throws ExecutionSetupException {
+ try {
+ if (partitions == null || partitions.size() == 0) {
+ Properties properties = MetaStoreUtils.getTableMetadata(table);
+ JobConf job = new JobConf();
+ for (Object obj : properties.keySet()) {
+ job.set((String) obj, (String) properties.get(obj));
+ }
+ InputFormat format = (InputFormat) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance();
+ job.setInputFormat(format.getClass());
+ Path path = new Path(table.getSd().getLocation());
+ FileInputFormat.addInputPath(job, path);
+ format = job.getInputFormat();
+ for (InputSplit split : format.getSplits(job, 1)) {
+ inputSplits.add(split);
+ }
+ for (InputSplit split : inputSplits) {
+ partitionMap.put(split, null);
+ }
+ } else {
+ for (Partition partition : partitions) {
+ Properties properties = MetaStoreUtils.getPartitionMetadata(partition, table);
+ JobConf job = new JobConf();
+ for (Object obj : properties.keySet()) {
+ job.set((String) obj, (String) properties.get(obj));
+ }
+ InputFormat format = (InputFormat) Class.forName(partition.getSd().getInputFormat()).getConstructor().newInstance();
+ job.setInputFormat(format.getClass());
+ FileInputFormat.addInputPath(job, new Path(partition.getSd().getLocation()));
+ format = job.getInputFormat();
+ InputSplit[] splits = format.getSplits(job,1);
+ for (InputSplit split : splits) {
+ inputSplits.add(split);
+ partitionMap.put(split, partition);
+ }
+ }
+ }
+ } catch (ReflectiveOperationException | IOException e) {
+ throw new ExecutionSetupException(e);
+ }
+ }
+
+ @Override
+ public void applyAssignments(List<CoordinationProtos.DrillbitEndpoint> endpoints) {
+ mappings = Lists.newArrayList();
+ for (int i = 0; i < endpoints.size(); i++) {
+ mappings.add(new ArrayList<InputSplit>());
+ }
+ int count = endpoints.size();
+ for (int i = 0; i < inputSplits.size(); i++) {
+ mappings.get(i % count).add(inputSplits.get(i));
+ }
+ }
+
+ public static String serializeInputSplit(InputSplit split) throws IOException {
+ ByteArrayDataOutput byteArrayOutputStream = ByteStreams.newDataOutput();
+ split.write(byteArrayOutputStream);
+ String encoded = Base64.encodeBase64String(byteArrayOutputStream.toByteArray());
+ logger.debug("Encoded split string for split {} : {}", split, encoded);
+ return encoded;
+ }
+
+ @Override
+ public SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException {
+ try {
+ List<InputSplit> splits = mappings.get(minorFragmentId);
+ List<Partition> parts = Lists.newArrayList();
+ List<String> encodedInputSplits = Lists.newArrayList();
+ List<String> splitTypes = Lists.newArrayList();
+ for (InputSplit split : splits) {
+ parts.add(partitionMap.get(split));
+ encodedInputSplits.add(serializeInputSplit(split));
+ splitTypes.add(split.getClass().getCanonicalName());
+ }
+ if (parts.contains(null)) parts = null;
+ return new HiveSubScan(table, parts, encodedInputSplits, splitTypes, columns);
+ } catch (IOException | ReflectiveOperationException e) {
+ throw new ExecutionSetupException(e);
+ }
+ }
+
+ @Override
+ public int getMaxParallelizationWidth() {
+ return inputSplits.size();
+ }
+
+ @Override
+ public List<EndpointAffinity> getOperatorAffinity() {
+ Map<String, DrillbitEndpoint> endpointMap = new HashMap();
+ for (DrillbitEndpoint endpoint : endpoints) {
+ endpointMap.put(endpoint.getAddress(), endpoint);
+ logger.debug("endpoing address: {}", endpoint.getAddress());
+ }
+ Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap();
+ try {
+ long totalSize = 0;
+ for (InputSplit split : inputSplits) {
+ totalSize += Math.max(1, split.getLength());
+ }
+ for (InputSplit split : inputSplits) {
+ float affinity = ((float) Math.max(1, split.getLength())) / totalSize;
+ for (String loc : split.getLocations()) {
+ logger.debug("split location: {}", loc);
+ DrillbitEndpoint endpoint = endpointMap.get(loc);
+ if (endpoint != null) {
+ if (affinityMap.containsKey(endpoint)) {
+ affinityMap.get(endpoint).addAffinity(affinity);
+ } else {
+ affinityMap.put(endpoint, new EndpointAffinity(endpoint, affinity));
+ }
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new DrillRuntimeException(e);
+ }
+ for (DrillbitEndpoint ep : affinityMap.keySet()) {
+ Preconditions.checkNotNull(ep);
+ }
+ for (EndpointAffinity a : affinityMap.values()) {
+ Preconditions.checkNotNull(a.getEndpoint());
+ }
+ return Lists.newArrayList(affinityMap.values());
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ return new OperatorCost(1, 2, 1, 1);
+ }
+
+ @Override
+ public Size getSize() {
+ // TODO - this is wrong, need to populate correctly
+ return new Size(10,10);
+ }
+
+ @Override
+ public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
+ return new HiveScan(hiveReadEntry, storageEngine, columns);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
new file mode 100644
index 0000000..b155661
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.beust.jcommander.internal.Lists;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.physical.impl.ScanBatch;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.store.RecordReader;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+import java.util.List;
+
+public class HiveScanBatchCreator implements BatchCreator<HiveSubScan> {
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, HiveSubScan config, List<RecordBatch> children) throws ExecutionSetupException {
+ List<RecordReader> readers = Lists.newArrayList();
+ Table table = config.table;
+ List<InputSplit> splits = config.getInputSplits();
+ List<Partition> partitions = config.partitions;
+ if (partitions == null || partitions.size() == 0) {
+ if (table.getSd().getInputFormat().equals(TextInputFormat.class.getCanonicalName()) &&
+ table.getSd().getSerdeInfo().getSerializationLib().equals(LazySimpleSerDe.class.getCanonicalName()) &&
+ config.getColumns() != null) {
+ for (InputSplit split : splits) {
+ readers.add(new HiveTextRecordReader(table, null, split, config.getColumns(), context));
+ }
+ } else {
+ for (InputSplit split : splits) {
+ readers.add(new HiveRecordReader(table, null, split, config.getColumns(), context));
+ }
+ }
+ } else {
+ int i = 0;
+ if (table.getSd().getInputFormat().equals(TextInputFormat.class.getCanonicalName()) &&
+ table.getSd().getSerdeInfo().getSerializationLib().equals(LazySimpleSerDe.class.getCanonicalName()) &&
+ config.getColumns() != null) {
+ for (InputSplit split : splits) {
+ readers.add(new HiveTextRecordReader(table, partitions.get(i++), split, config.getColumns(), context));
+ }
+ } else {
+ for (InputSplit split : splits) {
+ readers.add(new HiveRecordReader(config.table, partitions.get(i++), split, config.getColumns(), context));
+ }
+ }
+ }
+ return new ScanBatch(context, readers.iterator());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSchemaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSchemaProvider.java
deleted file mode 100644
index 38ec007..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSchemaProvider.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.hive;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.store.SchemaProvider;
-
-import com.beust.jcommander.internal.Lists;
-
-public class HiveSchemaProvider implements SchemaProvider{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveSchemaProvider.class);
-
- final HiveStorageEngineConfig configuration;
-
- public HiveSchemaProvider(HiveStorageEngineConfig configuration, DrillConfig config){
- this.configuration = configuration;
- }
-
- @Override
- public Object getSelectionBaseOnName(String tableName) {
- HiveReadEntry re = new HiveReadEntry(configuration.getHiveConf(), tableName);
- return Lists.newArrayList(re);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java
new file mode 100644
index 0000000..0f6f3bc
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.ReadEntry;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.AbstractStorageEngine;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.thrift.TException;
+
+import java.io.IOException;
+import java.util.List;
+
+public class HiveStorageEngine extends AbstractStorageEngine {
+
+ private HiveStorageEngineConfig config;
+ private HiveConf hiveConf;
+ private HiveSchemaProvider schemaProvider;
+ static private DrillbitContext context;
+
+ public HiveStorageEngine(HiveStorageEngineConfig config, DrillbitContext context) throws ExecutionSetupException {
+ this.config = config;
+ this.context = context;
+ this.hiveConf = config.getHiveConf();
+ }
+
+ public HiveStorageEngineConfig getConfig() {
+ return config;
+ }
+
+ public DrillbitContext getContext() {
+ return context;
+ }
+
+ @Override
+ public HiveScan getPhysicalScan(Scan scan) throws IOException {
+ HiveReadEntry hiveReadEntry = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){});
+ try {
+ List<Partition> partitions = getSchemaProvider().getPartitions(hiveReadEntry.getTable().getDbName(), hiveReadEntry.getTable().getTableName());
+ return new HiveScan(hiveReadEntry, this, null);
+ } catch (ExecutionSetupException | TException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+
+ @Override
+ public HiveSchemaProvider getSchemaProvider() {
+ try {
+ if (schemaProvider == null) {
+ schemaProvider = new HiveSchemaProvider(config, context.getConfig());
+ }
+ return schemaProvider;
+ } catch (ExecutionSetupException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+
+ List<String> getPartitions(String dbName, String tableName) throws TException {
+ List<Partition> partitions = getSchemaProvider().getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE);
+ List<String> partitionLocations = Lists.newArrayList();
+ if (partitions == null) return null;
+ for (Partition part : partitions) {
+ partitionLocations.add(part.getSd().getLocation());
+ }
+ return partitionLocations;
+ }
+
+ public static class HiveEntry implements ReadEntry {
+
+ private Table table;
+
+ public HiveEntry(Table table) {
+ this.table = table;
+ }
+
+ public Table getTable() {
+ return table;
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " +
+ "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost.");
+ }
+
+ @Override
+ public Size getSize() {
+ throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " +
+ "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost.");
+ }
+ }
+
+ public static class HiveSchemaProvider implements SchemaProvider {
+
+ private HiveConf hiveConf;
+ private HiveMetaStoreClient metaClient;
+
+ public HiveSchemaProvider(HiveStorageEngineConfig config, DrillConfig dConfig) throws ExecutionSetupException {
+ hiveConf = config.getHiveConf();
+ }
+
+ public HiveMetaStoreClient getMetaClient() throws MetaException {
+ if (metaClient == null) {
+ metaClient = new HiveMetaStoreClient(hiveConf);
+ }
+ return metaClient;
+ }
+
+ public Table getTable(String dbName, String tableName) throws TException {
+ HiveMetaStoreClient mClient = getMetaClient();
+ try {
+ return mClient.getTable(dbName, tableName);
+ }catch (NoSuchObjectException e) {
+ logger.error("Database: {} table: {} not found", dbName, tableName);
+ throw new RuntimeException(e);
+ } catch (TException e) {
+ mClient.reconnect();
+ return mClient.getTable(dbName, tableName);
+ }
+ }
+
+ List<Partition> getPartitions(String dbName, String tableName) throws TException {
+ HiveMetaStoreClient mClient = getMetaClient();
+ List<Partition> partitions;
+ try {
+ partitions = getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE);
+ } catch (TException e) {
+ mClient.reconnect();
+ partitions = getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE);
+ }
+ return partitions;
+ }
+
+ @Override
+ public HiveReadEntry getSelectionBaseOnName(String name) {
+ String[] dbNameTableName = name.split("\\.");
+ String dbName;
+ String t;
+ if (dbNameTableName.length > 1) {
+ dbName = dbNameTableName[0];
+ t = dbNameTableName[1];
+ } else {
+ dbName = "default";
+ t = name;
+ }
+
+ try {
+ Table table = getTable(dbName, t);
+ List<Partition> partitions = getPartitions(dbName, t);
+ List<HiveTable.HivePartition> hivePartitions = Lists.newArrayList();
+ for(Partition part : partitions) {
+ hivePartitions.add(new HiveTable.HivePartition(part));
+ }
+ if (hivePartitions.size() == 0) hivePartitions = null;
+ return new HiveReadEntry(new HiveTable(table), hivePartitions);
+ } catch (NoSuchObjectException e) {
+ throw new DrillRuntimeException(e);
+ } catch (TException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
index 0a2c5de..91fec3b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.store.hive;
import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import org.apache.drill.common.logical.StorageEngineConfigBase;
@@ -27,9 +28,12 @@ import java.util.Map;
@JsonTypeName("hive")
public class HiveStorageEngineConfig extends StorageEngineConfigBase {
- private Map<String, String> configProps;
+ @JsonProperty
+ public Map<String, String> configProps;
+ @JsonIgnore
private HiveConf hiveConf;
+ @JsonIgnore
public HiveConf getHiveConf() {
if (hiveConf == null) {
hiveConf = new HiveConf();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
new file mode 100644
index 0000000..8ff7c82
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSubScan.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.beust.jcommander.internal.Lists;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Iterators;
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteStreams;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.base.*;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputSplit;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+public class HiveSubScan extends AbstractBase implements SubScan {
+
+ @JsonProperty("splits")
+ public List<String> encodedSplits;
+ @JsonProperty("hive-table")
+ public Table table;
+ @JsonProperty("partitions")
+ public List<Partition> partitions;
+ @JsonIgnore
+ private List<InputSplit> inputSplits = Lists.newArrayList();
+ @JsonProperty("splitClass")
+ public List<String> splitClasses;
+
+ @JsonProperty("columns")
+ public List<FieldReference> columns;
+
+ @JsonCreator
+ public HiveSubScan(@JsonProperty("hive-table") Table table,
+ @JsonProperty("partition") List<Partition> partitions,
+ @JsonProperty("splits") List<String> encodedSplits,
+ @JsonProperty("splitClasses") List<String> splitClasses,
+ @JsonProperty("columns") List<FieldReference> columns) throws IOException, ReflectiveOperationException {
+ this.table = table;
+ this.partitions = partitions;
+ this.encodedSplits = encodedSplits;
+ this.splitClasses = splitClasses;
+ this.columns = columns;
+
+ for (int i = 0; i < encodedSplits.size(); i++) {
+ inputSplits.add(deserializeInputSplit(encodedSplits.get(i), splitClasses.get(i)));
+ }
+ }
+
+ public static InputSplit deserializeInputSplit(String base64, String className) throws IOException, ReflectiveOperationException{
+ InputSplit split;
+ if (Class.forName(className) == FileSplit.class) {
+ split = new FileSplit((Path) null, 0, 0, (String[])null);
+ } else {
+ split = (InputSplit) Class.forName(className).getConstructor().newInstance();
+ }
+ ByteArrayDataInput byteArrayDataInput = ByteStreams.newDataInput(Base64.decodeBase64(base64));
+ split.readFields(byteArrayDataInput);
+ return split;
+ }
+
+ public List<FieldReference> getColumns() {
+ return columns;
+ }
+
+ public List<InputSplit> getInputSplits() {
+ return inputSplits;
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ return new OperatorCost(1, 2, 1, 1);
+ }
+
+ @Override
+ public Size getSize() {
+ // TODO - this is wrong, need to populate correctly
+ return new Size(10,10);
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
+ return physicalVisitor.visitSubScan(this, value);
+ }
+
+ @Override
+ public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
+ try {
+ return new HiveSubScan(table, partitions, encodedSplits, splitClasses, columns);
+ } catch (IOException | ReflectiveOperationException e) {
+ throw new ExecutionSetupException(e);
+ }
+ }
+
+ @Override
+ public Iterator<PhysicalOperator> iterator() {
+ return Iterators.emptyIterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java
new file mode 100644
index 0000000..3858804
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTable.java
@@ -0,0 +1,325 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.beust.jcommander.internal.Lists;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.hadoop.hive.metastore.api.*;
+
+import java.util.List;
+import java.util.Map;
+
+@JsonTypeName("table")
+public class HiveTable {
+
+ @JsonIgnore
+ private Table table;
+
+ @JsonProperty
+ public String tableName;
+ @JsonProperty
+ public String dbName;
+ @JsonProperty
+ public String owner;
+ @JsonProperty
+ public int createTime;
+ @JsonProperty
+ public int lastAccessTime;
+ @JsonProperty
+ public int retention;
+ @JsonProperty
+ public StorageDescriptorWrapper sd;
+ @JsonProperty
+ public List<FieldSchemaWrapper> partitionKeys;
+ @JsonProperty
+ public Map<String,String> parameters;
+ @JsonProperty
+ public String viewOriginalText;
+ @JsonProperty
+ public String viewExpandedText;
+ @JsonProperty
+ public String tableType;
+
+ @JsonCreator
+ public HiveTable(@JsonProperty("tableName") String tableName, @JsonProperty("dbName") String dbName, @JsonProperty("owner") String owner, @JsonProperty("createTime") int createTime,
+ @JsonProperty("lastAccessTime") int lastAccessTime, @JsonProperty("retention") int retention, @JsonProperty("sd") StorageDescriptorWrapper sd,
+ @JsonProperty("partitionKeys") List<FieldSchemaWrapper> partitionKeys, @JsonProperty("parameters") Map<String, String> parameters,
+ @JsonProperty("viewOriginalText") String viewOriginalText, @JsonProperty("viewExpandedText") String viewExpandedText, @JsonProperty("tableType") String tableType
+ ) {
+ this.tableName = tableName;
+ this.dbName = dbName;
+ this.owner = owner;
+ this.createTime = createTime;
+ this.lastAccessTime = lastAccessTime;
+ this.retention = retention;
+ this.sd = sd;
+ this.partitionKeys = partitionKeys;
+ this.parameters = parameters;
+ this.viewOriginalText = viewOriginalText;
+ this.viewExpandedText = viewExpandedText;
+ this.tableType = tableType;
+
+ List<FieldSchema> partitionKeysUnwrapped = Lists.newArrayList();
+ for (FieldSchemaWrapper w : partitionKeys) partitionKeysUnwrapped.add(w.getFieldSchema());
+ StorageDescriptor sdUnwrapped = sd.getSd();
+ this.table = new Table(tableName, dbName, owner, createTime, lastAccessTime, retention, sdUnwrapped, partitionKeysUnwrapped,
+ parameters, viewOriginalText, viewExpandedText, tableType);
+ }
+
+ public HiveTable(Table table) {
+ if (table == null) return;
+ this.table = table;
+ this.tableName = table.getTableName();
+ this.dbName = table.getDbName();
+ this.owner = table.getOwner();
+ this.createTime = table.getCreateTime();
+ this.lastAccessTime = table.getLastAccessTime();
+ this.retention = table.getRetention();
+ this.sd = new StorageDescriptorWrapper(table.getSd());
+ this.partitionKeys = Lists.newArrayList();
+ for (FieldSchema f : table.getPartitionKeys()) this.partitionKeys.add(new FieldSchemaWrapper(f));
+ this.parameters = table.getParameters();
+ this.viewOriginalText = table.getViewOriginalText();
+ this.viewExpandedText = table.getViewExpandedText();
+ this.tableType = table.getTableType();
+ }
+
+ @JsonIgnore
+ public Table getTable() {
+ return table;
+ }
+
+ public static class HivePartition {
+
+ @JsonIgnore
+ private Partition partition;
+
+ @JsonProperty
+ public List<String> values;
+ @JsonProperty
+ public String tableName;
+ @JsonProperty
+ public String dbName;
+ @JsonProperty
+ public int createTime;
+ @JsonProperty
+ public int lastAccessTime;
+ @JsonProperty
+ public StorageDescriptorWrapper sd;
+ @JsonProperty
+ public Map<String,String> parameters;
+
+ @JsonCreator
+ public HivePartition(@JsonProperty("values") List<String> values, @JsonProperty("tableName") String tableName, @JsonProperty("dbName") String dbName, @JsonProperty("createTime") int createTime,
+ @JsonProperty("lastAccessTime") int lastAccessTime, @JsonProperty("sd") StorageDescriptorWrapper sd,
+ @JsonProperty("parameters") Map<String, String> parameters
+ ) {
+ this.values = values;
+ this.tableName = tableName;
+ this.dbName = dbName;
+ this.createTime = createTime;
+ this.lastAccessTime = lastAccessTime;
+ this.sd = sd;
+ this.parameters = parameters;
+
+ StorageDescriptor sdUnwrapped = sd.getSd();
+ this.partition = new org.apache.hadoop.hive.metastore.api.Partition(values, tableName, dbName, createTime, lastAccessTime, sdUnwrapped, parameters);
+ }
+
+ public HivePartition(Partition partition) {
+ if (partition == null) return;
+ this.partition = partition;
+ this.values = partition.getValues();
+ this.tableName = partition.getTableName();
+ this.dbName = partition.getDbName();
+ this.createTime = partition.getCreateTime();
+ this.lastAccessTime = partition.getLastAccessTime();
+ this.sd = new StorageDescriptorWrapper(partition.getSd());
+ this.parameters = partition.getParameters();
+ }
+
+ @JsonIgnore
+ public Partition getPartition() {
+ return partition;
+ }
+ }
+
+ public static class StorageDescriptorWrapper {
+ @JsonIgnore
+ private StorageDescriptor sd;
+ @JsonProperty
+ public List<FieldSchemaWrapper> cols;
+ @JsonProperty
+ public String location;
+ @JsonProperty
+ public String inputFormat;
+ @JsonProperty
+ public String outputFormat;
+ @JsonProperty
+ public boolean compressed;
+ @JsonProperty
+ public int numBuckets;
+ @JsonProperty
+ public SerDeInfoWrapper serDeInfo;
+// @JsonProperty
+// public List<String> bucketCols;
+ @JsonProperty
+ public List<OrderWrapper> sortCols;
+ @JsonProperty
+ public Map<String,String> parameters;
+
+ @JsonCreator
+ public StorageDescriptorWrapper(@JsonProperty("cols") List<FieldSchemaWrapper> cols, @JsonProperty("location") String location, @JsonProperty("inputFormat") String inputFormat,
+ @JsonProperty("outputFormat") String outputFormat, @JsonProperty("compressed") boolean compressed, @JsonProperty("numBuckets") int numBuckets,
+ @JsonProperty("serDeInfo") SerDeInfoWrapper serDeInfo, @JsonProperty("sortCols") List<OrderWrapper> sortCols,
+ @JsonProperty("parameters") Map<String,String> parameters) {
+ this.cols = cols;
+ this.location = location;
+ this.inputFormat = inputFormat;
+ this.outputFormat = outputFormat;
+ this.compressed = compressed;
+ this.numBuckets = numBuckets;
+ this.serDeInfo = serDeInfo;
+// this.bucketCols = bucketCols;
+ this.sortCols = sortCols;
+ this.parameters = parameters;
+ List<FieldSchema> colsUnwrapped = Lists.newArrayList();
+ for (FieldSchemaWrapper w: cols) colsUnwrapped.add(w.getFieldSchema());
+ SerDeInfo serDeInfoUnwrapped = serDeInfo.getSerDeInfo();
+ List<Order> sortColsUnwrapped = Lists.newArrayList();
+ for (OrderWrapper w : sortCols) sortColsUnwrapped.add(w.getOrder());
+// this.sd = new StorageDescriptor(colsUnwrapped, location, inputFormat, outputFormat, compressed, numBuckets, serDeInfoUnwrapped,
+// bucketCols, sortColsUnwrapped, parameters);
+ this.sd = new StorageDescriptor(colsUnwrapped, location, inputFormat, outputFormat, compressed, numBuckets, serDeInfoUnwrapped,
+ null, sortColsUnwrapped, parameters);
+ }
+
+ public StorageDescriptorWrapper(StorageDescriptor sd) {
+ this.sd = sd;
+ this.cols = Lists.newArrayList();
+ for (FieldSchema f : sd.getCols()) this.cols.add(new FieldSchemaWrapper(f));
+ this.location = sd.getLocation();
+ this.inputFormat = sd.getInputFormat();
+ this.outputFormat = sd.getOutputFormat();
+ this.compressed = sd.isCompressed();
+ this.numBuckets = sd.getNumBuckets();
+ this.serDeInfo = new SerDeInfoWrapper(sd.getSerdeInfo());
+// this.bucketCols = sd.getBucketCols();
+ this.sortCols = Lists.newArrayList();
+ for (Order o : sd.getSortCols()) this.sortCols.add(new OrderWrapper(o));
+ this.parameters = sd.getParameters();
+ }
+
+ @JsonIgnore
+ public StorageDescriptor getSd() {
+ return sd;
+ }
+
+ }
+
+ public static class SerDeInfoWrapper {
+ @JsonIgnore
+ private SerDeInfo serDeInfo;
+ @JsonProperty
+ public String name;
+ @JsonProperty
+ public String serializationLib;
+ @JsonProperty
+ public Map<String,String> parameters;
+
+ @JsonCreator
+ public SerDeInfoWrapper(@JsonProperty("name") String name, @JsonProperty("serializationLib") String serializationLib, @JsonProperty("parameters") Map<String, String> parameters) {
+ this.name = name;
+ this.serializationLib = serializationLib;
+ this.parameters = parameters;
+ this.serDeInfo = new SerDeInfo(name, serializationLib, parameters);
+ }
+
+ public SerDeInfoWrapper(SerDeInfo serDeInfo) {
+ this.serDeInfo = serDeInfo;
+ this.name = serDeInfo.getName();
+ this.serializationLib = serDeInfo.getSerializationLib();
+ this.parameters = serDeInfo.getParameters();
+ }
+
+ @JsonIgnore
+ public SerDeInfo getSerDeInfo() {
+ return serDeInfo;
+ }
+ }
+
+ public static class FieldSchemaWrapper {
+ @JsonIgnore
+ private FieldSchema fieldSchema;
+ @JsonProperty
+ public String name;
+ @JsonProperty
+ public String type;
+ @JsonProperty
+ public String comment;
+
+ @JsonCreator
+ public FieldSchemaWrapper(@JsonProperty("name") String name, @JsonProperty("type") String type, @JsonProperty("comment") String comment) {
+ this.name = name;
+ this.type = type;
+ this.comment = comment;
+ this.fieldSchema = new FieldSchema(name, type, comment);
+ }
+
+ public FieldSchemaWrapper(FieldSchema fieldSchema) {
+ this.fieldSchema = fieldSchema;
+ this.name = fieldSchema.getName();
+ this.type = fieldSchema.getType();
+ this.comment = fieldSchema.getComment();
+ }
+
+ @JsonIgnore
+ public FieldSchema getFieldSchema() {
+ return fieldSchema;
+ }
+ }
+
+ public static class OrderWrapper {
+ @JsonIgnore
+ private Order ord;
+ @JsonProperty
+ public String col;
+ @JsonProperty
+ public int order;
+
+ @JsonCreator
+ public OrderWrapper(@JsonProperty("col") String col, @JsonProperty("order") int order) {
+ this.col = col;
+ this.order = order;
+ }
+
+ public OrderWrapper(Order ord) {
+ this.ord = ord;
+ this.col = ord.getCol();
+ this.order = ord.getOrder();
+ }
+
+ @JsonIgnore
+ public Order getOrder() {
+ return ord;
+ }
+ }
+}
[36/38] git commit: Add support for generating optiq planner
independent schema plus root for information schema purposes.
Posted by ja...@apache.org.
Add support for generating optiq planner independent schema plus root for information schema purposes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ae041d07
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ae041d07
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ae041d07
Branch: refs/heads/master
Commit: ae041d07b351c6e8f744b28df083fa8556838ee6
Parents: d90d5b1
Author: Jacques Nadeau <ja...@apache.org>
Authored: Thu Feb 27 06:56:19 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:18 2014 -0800
----------------------------------------------------------------------
.../drill/exec/store/StoragePluginRegistry.java | 155 +++++++++++++++++++
.../drill/exec/store/TestOrphanSchema.java | 58 +++++++
2 files changed, 213 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae041d07/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
index b182abe..0e9d1e6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
@@ -21,14 +21,20 @@ import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
+import net.hydromatic.linq4j.expressions.DefaultExpression;
+import net.hydromatic.linq4j.expressions.Expression;
import net.hydromatic.linq4j.function.Function1;
import net.hydromatic.optiq.Schema;
import net.hydromatic.optiq.SchemaPlus;
+import net.hydromatic.optiq.Table;
+import net.hydromatic.optiq.TableFunction;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
@@ -44,6 +50,7 @@ import org.apache.drill.exec.store.dfs.FormatPlugin;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
+import com.google.hive12.common.collect.Maps;
public class StoragePluginRegistry implements Iterable<Map.Entry<String, StoragePlugin>>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginRegistry.class);
@@ -167,9 +174,157 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage
return defaultSchema;
}
+ /**
+ * Used in situations where we want to get a schema without having to use in the context of an Optiq planner.
+ * @return Root schema of the storage engine hiearchy.
+ */
+ public SchemaPlus getOrphanedRootSchema(){
+ SchemaPlus p = new OrphanPlus();
+ apply(p);
+ return p;
+ }
+
}
+ private class OrphanPlusWrap extends OrphanPlus{
+ private Schema inner;
+
+ public OrphanPlusWrap(Schema inner) {
+ super();
+ this.inner = inner;
+ }
+
+ @Override
+ public SchemaPlus getParentSchema() {
+ return inner.getParentSchema();
+ }
+
+ @Override
+ public String getName() {
+ return inner.getName();
+ }
+
+ @Override
+ public Table getTable(String name) {
+ return inner.getTable(name);
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return inner.getTableNames();
+ }
+
+ @Override
+ public Collection<TableFunction> getTableFunctions(String name) {
+ return inner.getTableFunctions(name);
+ }
+
+ @Override
+ public Set<String> getTableFunctionNames() {
+ return inner.getTableFunctionNames();
+ }
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ return inner.getSubSchemaNames();
+ }
+ @Override
+ public Expression getExpression() {
+ return inner.getExpression();
+ }
+
+ @Override
+ public SchemaPlus addRecursive(Schema schema) {
+ return schema.getParentSchema().add(schema);
+ }
+
+
+
+ }
+
+ private class OrphanPlus implements SchemaPlus{
+
+ private HashMap<String, SchemaPlus> schemas = Maps.newHashMap();
+
+ @Override
+ public SchemaPlus getParentSchema() {
+ return null;
+ }
+
+ @Override
+ public String getName() {
+ return "";
+ }
+
+ @Override
+ public Table getTable(String name) {
+ return null;
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Collection<TableFunction> getTableFunctions(String name) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Set<String> getTableFunctionNames() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ return schemas.keySet();
+ }
+
+ @Override
+ public Expression getExpression() {
+ return new DefaultExpression(Object.class);
+ }
+
+ @Override
+ public SchemaPlus getSubSchema(String name) {
+ return schemas.get(name);
+ }
+
+ @Override
+ public SchemaPlus add(Schema schema) {
+ OrphanPlusWrap plus = new OrphanPlusWrap(schema);
+ schemas.put(schema.getName(), plus);
+ return plus;
+ }
+
+ @Override
+ public void add(String name, Table table) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void add(String name, TableFunction table) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isMutable() {
+ return false;
+ }
+
+ @Override
+ public <T> T unwrap(Class<T> clazz) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SchemaPlus addRecursive(Schema schema) {
+ return schema.getParentSchema().add(schema);
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae041d07/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
new file mode 100644
index 0000000..2087dc2
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOrphanSchema.java
@@ -0,0 +1,58 @@
+package org.apache.drill.exec.store;
+
+import mockit.NonStrictExpectations;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.junit.Test;
+
+import com.codahale.metrics.MetricRegistry;
+
+public class TestOrphanSchema {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestOrphanSchema.class);
+
+
+ @Test
+ public void test(final DrillbitContext bitContext){
+ final DrillConfig c = DrillConfig.create();
+
+ new NonStrictExpectations() {
+ {
+ bitContext.getMetrics();
+ result = new MetricRegistry();
+ bitContext.getAllocator();
+ result = new TopLevelAllocator();
+ bitContext.getConfig();
+ result = c;
+ }
+ };
+
+ StoragePluginRegistry r = new StoragePluginRegistry(bitContext);
+ SchemaPlus plus = r.getSchemaFactory().getOrphanedRootSchema();
+
+ printSchema(plus, 0);
+
+ }
+
+ private static void t(final int t){
+ for(int i =0; i < t; i++) System.out.print('\t');
+ }
+ private static void printSchema(SchemaPlus s, int indent){
+ t(indent);
+ System.out.print("Schema: ");
+ System.out.println(s.getName().equals("") ? "root" : s.getName());
+ for(String table : s.getTableNames()){
+ t(indent + 1);
+ System.out.print("Table: ");
+ System.out.println(table);
+ }
+
+ for(String schema : s.getSubSchemaNames()){
+ SchemaPlus p = s.getSubSchema(schema);
+ printSchema(p, indent + 1);
+ }
+
+ }
+}
[09/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RSEBase.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RSEBase.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RSEBase.java
deleted file mode 100644
index bbebc12..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RSEBase.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.common.util.PathScanner;
-import org.apache.drill.exec.ref.ExecRefConstants;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.exceptions.MajorException;
-import org.apache.drill.exec.ref.rops.ROP;
-
-import com.typesafe.config.Config;
-
-public abstract class RSEBase implements ReferenceStorageEngine{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RSEBase.class);
-
- @Override
- public boolean supportsRead() {
- return false;
- }
-
- @Override
- public boolean supportsWrite() {
- return false;
- }
-
- @Override
- public Collection<ReadEntry> getReadEntries(Scan scan) throws IOException {
- throw new UnsupportedOperationException(String.format("%s does not support reads.", this.getClass().getCanonicalName()));
- }
-
- @Override
- public RecordReader getReader(ReadEntry readEntry, ROP parentROP) throws IOException {
- throw new UnsupportedOperationException(String.format("%s does not support reads.", this.getClass().getCanonicalName()));
- }
-
- @Override
- public RecordRecorder getWriter(Store store) throws IOException {
- throw new UnsupportedOperationException(String.format("%s does not support writes.", this.getClass().getCanonicalName()));
- }
-
- public static Class<?>[] getSubTypes(Config config){
- Collection<Class<? extends ReferenceStorageEngine>> engines = PathScanner.scanForImplementations(ReferenceStorageEngine.class, config.getStringList(ExecRefConstants.STORAGE_ENGINE_SCAN_PACKAGES));
- return engines.toArray(new Class<?>[engines.size()]);
- }
-
- @SuppressWarnings("unchecked")
- protected <T extends ReadEntry> T getReadEntry(Class<T> c, ReadEntry entry){
- if(!c.isAssignableFrom(entry.getClass())) throw new MajorException(String.format("Expected entry type was invalid. Expected entry of type %s but received type of %s.", c.getCanonicalName(), entry.getClass().getCanonicalName()));
- return (T) entry;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RSERegistry.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RSERegistry.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RSERegistry.java
deleted file mode 100644
index 6eb48e1..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RSERegistry.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.util.PathScanner;
-import org.apache.drill.exec.ref.ExecRefConstants;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-
-import com.typesafe.config.Config;
-
-public class RSERegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RSERegistry.class);
-
- private Map<Object, Constructor<? extends ReferenceStorageEngine>> availableEngines = new HashMap<Object, Constructor<? extends ReferenceStorageEngine>>();
- private Map<StorageEngineConfig, ReferenceStorageEngine> activeEngines = new HashMap<StorageEngineConfig, ReferenceStorageEngine>();
- private DrillConfig config;
-
- public RSERegistry(DrillConfig config){
- this.config = config;
- setup(config);
- }
-
- @SuppressWarnings("unchecked")
- public void setup(DrillConfig config){
- Collection<Class<? extends ReferenceStorageEngine>> engines = PathScanner.scanForImplementations(ReferenceStorageEngine.class, config.getStringList(ExecRefConstants.STORAGE_ENGINE_SCAN_PACKAGES));
- logger.debug("Loading storage engines {}", engines);
- for(Class<? extends ReferenceStorageEngine> engine: engines){
- int i =0;
- for(Constructor<?> c : engine.getConstructors()){
- Class<?>[] params = c.getParameterTypes();
- if(params.length != 2 || params[1] == Config.class || !StorageEngineConfig.class.isAssignableFrom(params[0])){
- logger.debug("Skipping ReferenceStorageEngine constructor {} for engine class {} since it doesn't implement a [constructor(StorageEngineConfig, Config)]", c, engine);
- continue;
- }
- availableEngines.put(params[0], (Constructor<? extends ReferenceStorageEngine>) c);
- i++;
- }
- if(i == 0){
- logger.debug("Skipping registration of ReferenceStorageEngine {} as it doesn't have a constructor with the parameters of (StorangeEngineConfig, Config)", engine.getCanonicalName());
- }
- }
- }
-
- public ReferenceStorageEngine getEngine(StorageEngineConfig engineConfig) throws SetupException{
- ReferenceStorageEngine engine = activeEngines.get(engineConfig);
- if(engine != null) return engine;
- Constructor<? extends ReferenceStorageEngine> c = availableEngines.get(engineConfig.getClass());
- if(c == null) throw new SetupException(String.format("Failure finding StorageEngine constructor for config %s", engineConfig));
- try {
- engine = c.newInstance(engineConfig, config);
- activeEngines.put(engineConfig,engine);
- return engine;
- } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
- Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException)e).getTargetException() : e;
- if(t instanceof SetupException) throw ((SetupException) t);
- throw new SetupException(String.format("Failure setting up new storage engine configuration for config %s", engineConfig), t);
- }
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RecordReader.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RecordReader.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RecordReader.java
deleted file mode 100644
index a441fc1..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RecordReader.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import org.apache.drill.exec.ref.RecordIterator;
-
-public interface RecordReader {
-
- public abstract RecordIterator getIterator();
- public abstract void setup();
- public abstract void cleanup();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RecordRecorder.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RecordRecorder.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RecordRecorder.java
deleted file mode 100644
index 848797e..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/RecordRecorder.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.IOException;
-
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.RunOutcome;
-
-public interface RecordRecorder {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RecordRecorder.class);
-
- public void setup() throws IOException;
- public long recordRecord(RecordPointer pointer) throws IOException;
- public void finish(RunOutcome.OutcomeType outcome) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ReferenceStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ReferenceStorageEngine.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ReferenceStorageEngine.java
deleted file mode 100644
index 997d7d1..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ReferenceStorageEngine.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rse;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.exec.ref.rops.ROP;
-
-
-public interface ReferenceStorageEngine {
- public boolean supportsRead();
- public boolean supportsWrite();
-
- public enum PartitionCapabilities {
- NONE, HASH, SORTED;
- }
-
- public enum MemoryFormat {
- RECORD, FIELD;
- }
-
- public Collection<ReadEntry> getReadEntries(Scan scan) throws IOException;
- public RecordReader getReader(ReadEntry readEntry, ROP parentROP) throws IOException;
- public RecordRecorder getWriter(Store store) throws IOException;
-
- public interface ReadEntry{}
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/util/BatchBreaker.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/util/BatchBreaker.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/util/BatchBreaker.java
deleted file mode 100644
index 4806327..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/util/BatchBreaker.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.util;
-
-import org.apache.drill.exec.ref.RecordPointer;
-
-public interface BatchBreaker {
- public boolean shouldBreakAfter(RecordPointer record);
-
- public static class CountBreaker implements BatchBreaker{
- private final int max;
- private int count = 0;
-
- public CountBreaker(int max) {
- super();
- this.max = max;
- }
-
- @Override
- public boolean shouldBreakAfter(RecordPointer record) {
- count++;
- if(count > max){
- count = 0;
- return true;
- }{
- return false;
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/util/ByteRange.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/util/ByteRange.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/util/ByteRange.java
deleted file mode 100644
index 6f39759..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/util/ByteRange.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-
-
-/**
- * Facade interface for ByteBuffer and byte[]
- */
-public interface ByteRange {
- public byte getByte(int index);
- public int getLength();
- public void copyTo(byte[] buffer, int offset);
- public void copyTo(InputStream is) throws IOException;
- public void copyTo(ByteBuffer bb);
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseArrayValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseArrayValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseArrayValue.java
deleted file mode 100644
index 7c42151..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseArrayValue.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-
-public abstract class BaseArrayValue extends BaseDataValue implements ContainerValue{
-
- private MajorType initiatingType;
- private MajorType runningType = MajorType.newBuilder().setMode(DataMode.REPEATED).setMinorType(MinorType.LATE).build();
-
- @Override
- public void addValue(PathSegment segment, DataValue v) {
- if(initiatingType == null){
- initiatingType = v.getDataType();
- runningType = initiatingType.toBuilder().setMode(DataMode.REPEATED).build();
- }else{
- if(!v.getDataType().equals(initiatingType)){
- throw new RuntimeException("The reference interpreter doesn't support polymorphic types.");
- }
- }
-
- DataValue fullPathValue = ValueUtils.getIntermediateValues(segment.getChild(), v);
- if(segment.isArray()){ // we need to place this object in the given position.
- int index = segment.getArraySegment().getIndex();
- DataValue mergedValue = ValueUtils.getMergedDataValue(segment.getCollisionBehavior(), getByArrayIndex(index), fullPathValue);
- addToArray(index, mergedValue);
- }else{ // add to end of array.
- addToArray(getNextIndex(), fullPathValue);
- }
- }
-
- protected abstract void addToArray(int index, DataValue v);
- public abstract DataValue getByArrayIndex(int index);
- protected abstract int getNextIndex();
- public abstract void append(BaseArrayValue container);
- public abstract int size();
-
- @Override
- public DataValue getValue(PathSegment segment) {
- if(segment == null){ // return entire array
- return this;
- }else if(!segment.isArray()){ // requesting a named value from within an array. No value should be returned.
- return DataValue.NULL_VALUE;
- }else{
- DataValue v = getByArrayIndex(segment.getArraySegment().getIndex());
- if(v == null) return DataValue.NULL_VALUE;
- return v.getValue(segment.getChild());
- }
- }
-
- @Override
- public ContainerValue getAsContainer() {
- return this;
- }
-
- @Override
- public MajorType getDataType() {
- return runningType;
- }
-
-
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseDataValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseDataValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseDataValue.java
deleted file mode 100644
index d979ed2..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseDataValue.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.expression.PathSegment;
-
-public abstract class BaseDataValue implements DataValue{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseDataValue.class);
-
- @Override
- public DataValue getValue(PathSegment segment) {
- if(segment == null){
- return this;
- }else{ // looking for a lower level value when there is none.
- return DataValue.NULL_VALUE;
- }
- }
-
- public void addValue(PathSegment segment, DataValue v) {
- throw new IllegalArgumentException("You can't add a value to a non-container type.");
- }
-
- public void removeValue(PathSegment segment) {
- throw new IllegalArgumentException("You can't remove a value from a non-container type.");
- }
-
- @Override
- public NumericValue getAsNumeric() {
- throw new DrillRuntimeException(String.format("A %s value is not a NumericValue.", this.getClass().getCanonicalName()));
- }
-
- @Override
- public ContainerValue getAsContainer() {
- throw new DrillRuntimeException(String.format("A %s value is not a ContainerValue.", this.getClass().getCanonicalName()));
- }
-
- @Override
- public StringValue getAsStringValue() {
- throw new DrillRuntimeException(String.format("A %s value is not a StringValue.", this.getClass().getCanonicalName()));
- }
-
- public BytesValue getAsBytesValue(){
- throw new DrillRuntimeException(String.format("A %s value is not a BytesValue.", this.getClass().getCanonicalName()));
- }
-
- public BooleanValue getAsBooleanValue(){
- throw new DrillRuntimeException(String.format("A %s value is not a BooleanValue.", this.getClass().getCanonicalName()));
- }
-
- @Override
- public abstract int hashCode();
-
- @Override
- public boolean equals(Object obj) {
- if(obj instanceof DataValue){
- return this.equals((DataValue) obj);
- }else{
- return false;
- }
- }
-
-
- @Override
- public abstract DataValue copy();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseMapValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseMapValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseMapValue.java
deleted file mode 100644
index b42390f..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BaseMapValue.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import java.util.Map;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.ValueExpressions.CollisionBehavior;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.ref.exceptions.RecordException;
-
-public abstract class BaseMapValue extends BaseDataValue implements ContainerValue,
- Iterable<Map.Entry<CharSequence, DataValue>> {
-
- @Override
- public void addValue(PathSegment segment, DataValue v) {
- if(v == null) throw new RecordException("You attempted to add a null value to a map.", null);
- if (segment.isArray())
- throw new RecordException(
- "You're attempted to save something at a particular array location while the location of that setting was a Map.", null);
-
- CharSequence name = segment.getNameSegment().getPath();
- DataValue current = getByNameNoNulls(name);
- if (!segment.isLastPath() && current != DataValue.NULL_VALUE) {
- current.addValue(segment.getChild(), v);
- return;
- } else {
- DataValue fullPathValue = ValueUtils.getIntermediateValues(segment.getChild(), v);
- DataValue mergedValue = ValueUtils.getMergedDataValue(segment.getCollisionBehavior(), getByNameNoNulls(name),
- fullPathValue);
- setByName(name, mergedValue);
-
- }
-
- }
-
- @Override
- public void removeValue(PathSegment segment) {
- if (segment.isArray())
- throw new RecordException(
- "You're attempted to remove something at a particular array location while the location of that setting was a Map.", null);
-
- CharSequence name = segment.getNameSegment().getPath();
- DataValue current = getByNameNoNulls(name);
- if (!segment.isLastPath() && current != DataValue.NULL_VALUE) {
- current.removeValue(segment.getChild());
- return;
- } else {
- removeByName(name);
- }
-
- }
-
- protected abstract void setByName(CharSequence name, DataValue v);
-
- protected abstract DataValue getByName(CharSequence name);
-
- protected abstract void removeByName(CharSequence name);
-
- private DataValue getByNameNoNulls(CharSequence name) {
- DataValue v = getByName(name);
- if (v == null) return NULL_VALUE;
- return v;
- }
-
- @Override
- public DataValue getValue(PathSegment segment) {
- if (segment == null) return this;
- if (segment.isArray()) return NULL_VALUE;
- return getByNameNoNulls(segment.getNameSegment().getPath()).getValue(segment.getChild());
- }
-
- @Override
- public ContainerValue getAsContainer() {
- return this;
- }
-
- @Override
- public MajorType getDataType() {
- return MajorType.newBuilder().setMinorType(MinorType.REPEATMAP).setMode(DataMode.REPEATED).build();
- }
-
- public void merge(BaseMapValue otherMap) {
- for (Map.Entry<CharSequence, DataValue> e : otherMap) {
- final DataValue oldValue = getByName(e.getKey());
- if (oldValue == DataValue.NULL_VALUE || oldValue == null) {
- setByName(e.getKey(), e.getValue());
- } else {
- setByName(e.getKey(), ValueUtils.getMergedDataValue(CollisionBehavior.MERGE_OVERRIDE, oldValue, e.getValue()));
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BooleanValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BooleanValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BooleanValue.java
deleted file mode 100644
index 81288cb..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BooleanValue.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-public interface BooleanValue extends DataValue{
- public boolean getBoolean();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BytesValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BytesValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BytesValue.java
deleted file mode 100644
index 5635051..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/BytesValue.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-public interface BytesValue extends DataValue, ComparableValue{
-
- public byte[] getAsArray();
-
- // byteBuffer like
- public int getLength();
- public byte get(int pos);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ComparableValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ComparableValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ComparableValue.java
deleted file mode 100644
index a87c929..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ComparableValue.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-public interface ComparableValue extends Comparable<DataValue>, DataValue {
-
- /**
- * Tells whether or not comparisons between the current ComparableValue and the provided DataValue are possible.
- * @param dv2 The other DataValue
- * @return True if comaprable. False if not comparable.
- */
- public boolean supportsCompare(DataValue dv2);
-
- /**
- * Similar to standard comparable. However, the expectation is that supportsCompare should be called first. Likely
- * will have unexpected outcome if you don't call supportsCompare first.
- *
- * @param other
- * @return
- */
- public int compareTo(DataValue other);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ContainerValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ContainerValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ContainerValue.java
deleted file mode 100644
index 7e23872..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ContainerValue.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-
-
-public interface ContainerValue extends DataValue{
- public BaseArrayValue getAsArray();
- public BaseMapValue getAsMap();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/DataValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/DataValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/DataValue.java
deleted file mode 100644
index b95897b..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/DataValue.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import java.io.IOException;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.ref.rops.DataWriter;
-
-
-public interface DataValue {
-
- public static final DataValue NULL_VALUE = new ScalarValues.NullValue();
-
- public DataValue getValue(PathSegment segment);
- public void addValue(PathSegment segment, DataValue v);
- public void removeValue(PathSegment segment);
- public void write(DataWriter writer) throws IOException;
- public MajorType getDataType();
- public NumericValue getAsNumeric();
- public ContainerValue getAsContainer();
- public StringValue getAsStringValue();
- public BooleanValue getAsBooleanValue();
- public BytesValue getAsBytesValue();
- public boolean equals(DataValue v);
- public boolean equals(Object v);
- public int hashCode();
- public DataValue copy();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/DataValueSet.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/DataValueSet.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/DataValueSet.java
deleted file mode 100644
index 6e592be..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/DataValueSet.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import java.util.Arrays;
-
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-
-public class DataValueSet {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataValueSet.class);
-
- private final DataValue[] values;
- private final BasicEvaluator[] evaluators;
-
- public DataValueSet(BasicEvaluator... evaluators){
- this.values = new DataValue[evaluators.length];
- this.evaluators = evaluators;
- }
-
- private DataValueSet(DataValue[] values){
- this.values = new DataValue[values.length];
- System.arraycopy(values, 0, this.values, 0, values.length);
- this.evaluators = null;
- }
-
- public void grabValues(){
- for(int i =0; i < values.length; i++){
- values[i] = evaluators[i].eval();
- }
- }
-
- public void copyFrom(DataValueSet set){
- System.arraycopy(set.values, 0, values, 0, set.values.length);
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(values);
- return result;
- }
-
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) return true;
- if (obj == null) return false;
- if (getClass() != obj.getClass()) return false;
- DataValueSet other = (DataValueSet) obj;
- if (!Arrays.equals(values, other.values)) return false;
- return true;
- }
-
- @Override
- public String toString() {
- return "DataValueSet [values=" + Arrays.toString(values) + "]";
- }
-
- public DataValueSet cloneValuesOnly(){
- return new DataValueSet(this.values);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/NumericValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/NumericValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/NumericValue.java
deleted file mode 100644
index f32a80e..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/NumericValue.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.ScalarValues.DoubleScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.FloatScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.IntegerScalar;
-import org.apache.drill.exec.ref.values.ScalarValues.LongScalar;
-
-public abstract class NumericValue extends BaseDataValue implements ComparableValue, BasicEvaluator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NumericValue.class);
-
-
- public static enum NumericType {
- // order is important for conversion
- INT, LONG, BIG_INTEGER, FLOAT, DOUBLE, BIG_DECIMAL;
- }
-
- public abstract NumericType getNumericType();
-
- @Override
- public int compareTo(DataValue dv2) {
- NumericValue other = dv2.getAsNumeric();
- NumericType mutual = getMutualType(this, other);
- switch(mutual){
- case BIG_DECIMAL:
- return this.getAsBigDecimal().compareTo(other.getAsBigDecimal());
- case BIG_INTEGER:
- return this.getAsBigInteger().compareTo(other.getAsBigInteger());
- case DOUBLE:
- return Double.compare(this.getAsDouble(), other.getAsDouble());
- case FLOAT:
- return Float.compare(this.getAsFloat(), other.getAsFloat());
- case INT:
- return Integer.compare(this.getAsInt(), other.getAsInt());
- case LONG:
- return Long.compare(this.getAsLong(), other.getAsLong());
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- public int getHashCode(double d){
- Long l = Double.doubleToLongBits(d);
- return (int)(l ^ (l >>> 32));
- }
-
- @Override
- public DataValue eval() {
- return this;
- }
-
- private static NumericType getMutualType(NumericValue... values){
- int ord = 0;
- for(int i =0; i < values.length; i++){
- ord = Math.max(ord, values[i].getNumericType().ordinal());
- }
- return NumericType.values()[ord];
- }
-
-
-
- @Override
- public boolean equals(DataValue v) {
- if(v == null) return false;
- if(Types.isNumericType(v.getDataType())){
- return this.compareTo(v) == 0;
- }else{
- return false;
- }
- }
-
-
- public static NumericValue add(NumericValue... values){
- NumericType mutual = getMutualType(values);
- switch(mutual){
- case BIG_DECIMAL:
- throw new UnsupportedOperationException();
-// BigDecimal bd = new BigDecimal(0);
-// for(int i =0; i < values.length; i++){
-// bd = bd.add(values[i].getAsBigDecimal());
-// }
-// return new BigDecimalScalar(bd);
- case BIG_INTEGER:
- throw new UnsupportedOperationException();
-//
-// BigInteger bi = BigInteger.valueOf(0);
-// for(int i =0; i < values.length; i++){
-// bi = bi.add(values[i].getAsBigInteger());
-// }
-// return new BigIntegerScalar(bi);
- case DOUBLE:
- double d = 0d;
- for(int i =0; i < values.length; i++){
- d += values[i].getAsDouble();
- }
- return new DoubleScalar(d);
- case FLOAT:
- float f = 0f;
- for(int i =0; i < values.length; i++){
- f += values[i].getAsFloat();
- }
- return new FloatScalar(f);
- case INT:
- int x = 0;
- for(int i =0; i < values.length; i++){
- x += values[i].getAsInt();
- }
- return new IntegerScalar(x);
- case LONG:
- int l = 0;
- for(int i =0; i < values.length; i++){
- l += values[i].getAsLong();
- }
- return new LongScalar(l);
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- @Override
- public boolean supportsCompare(DataValue dv2) {
- return Types.isNumericType(dv2.getDataType());
- }
-
-
-
-
- @Override
- public NumericValue getAsNumeric() {
- return this;
- }
-
- public long getAsLong(){
- throw new DrillRuntimeException(String.format("A %s value can not be implicitly cast to a long.", this.getDataType()));
- }
- public int getAsInt(){
- throw new DrillRuntimeException(String.format("A %s value can not be implicitly cast to an int.", this.getDataType()));
- }
- public float getAsFloat(){
- throw new DrillRuntimeException(String.format("A %s value can not be implicitly cast to an float.", this.getDataType()));
- }
- public double getAsDouble(){
- throw new DrillRuntimeException(String.format("A %s value can not be implicitly cast to a double.", this.getDataType()));
- }
- public BigDecimal getAsBigDecimal(){
- throw new DrillRuntimeException(String.format("A %s value can not be implicitly cast to an big decimal.", this.getDataType()));
- }
- public BigInteger getAsBigInteger(){
- throw new DrillRuntimeException(String.format("A %s value can not be implicitly cast to a big integer.", this.getDataType()));
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ScalarValues.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ScalarValues.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ScalarValues.java
deleted file mode 100644
index 9226b94..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ScalarValues.java
+++ /dev/null
@@ -1,560 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Arrays;
-
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.rops.DataWriter;
-import org.apache.hadoop.io.BytesWritable;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-
-
-public final class ScalarValues {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScalarValues.class);
-
- private ScalarValues(){}
-
- public static class StringScalar extends BaseDataValue implements StringValue, ComparableValue, BasicEvaluator {
- private CharSequence seq;
- private MajorType type = MajorType.newBuilder().setMinorType(MinorType.VARCHAR).setMode(DataMode.OPTIONAL).build();
-
- public StringScalar(CharSequence seq){
- this.seq = seq;
- }
-
- @Override
- public int compareTo(DataValue o) {
- CharSequence seq1 = seq;
- CharSequence seq2 = o.getAsStringValue().getString();
- final int len = Math.min(seq1.length(), seq2.length());
- for(int i =0; i < len; i++){
- char c1 = seq1.charAt(i);
- char c2 = seq2.charAt(i);
- if(c1 != c2){
- return c1 - c2;
- }
- }
- return seq1.length() - seq2.length();
- }
-
- @Override
- public boolean supportsCompare(DataValue dv2) {
- return Types.isStringScalarType(dv2.getDataType());
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.writeCharSequence(seq);
- }
-
- @Override
- public MajorType getDataType() {
- return type;
- }
-
- @Override
- public StringValue getAsStringValue() {
- return this;
- }
-
- @Override
- public CharSequence getString() {
- return seq;
- }
-
- @Override
- public boolean isConstant() {
- return true;
- }
-
- public DataValue eval(){
- return this;
- }
-
- @Override
- public String toString() {
- return "StringScalar [seq=" + seq + "]";
- }
-
- @Override
- public boolean equals(DataValue v) {
- if(!v.getDataType().equals(this.getDataType())) return false;
- return seq.equals(v.getAsStringValue().getString());
- }
-
- @Override
- public int hashCode() {
- return seq.hashCode();
- }
-
- @Override
- public DataValue copy() {
- return new StringScalar(seq.toString());
- }
- }
-
-
-
- public static class BooleanScalar extends BaseDataValue implements BooleanValue, BasicEvaluator{
- private boolean b;
- private MajorType type = MajorType.newBuilder().setMinorType(MinorType.BIT).setMode(DataMode.OPTIONAL).build();
-
- public BooleanScalar(boolean b){
- this.b = b;
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.writeBoolean(b);
- }
-
- @Override
- public boolean getBoolean() {
- return b;
- }
-
- @Override
- public BooleanValue getAsBooleanValue() {
- return this;
- }
-
- @Override
- public MajorType getDataType() {
- return type;
- }
-
- @Override
- public boolean isConstant() {
- return true;
- }
-
- @Override
- public DataValue eval() {
- return this;
- }
-
- @Override
- public String toString() {
- return "BooleanScalar [b=" + b + "]";
- }
-
- @Override
- public boolean equals(DataValue v) {
- if(v.getDataType().equals(this.getDataType())) return false;
- return b == v.getAsBooleanValue().getBoolean();
- }
-
-
- @Override
- public int hashCode() {
- return b ? 1 : 0;
- }
-
- @Override
- public DataValue copy() {
- return new BooleanScalar(b);
- }
- }
-
- public static class LongScalar extends NumericValue{
-
- private MajorType type = MajorType.newBuilder().setMinorType(MinorType.BIGINT).setMode(DataMode.OPTIONAL).build();
- long l;
- public LongScalar(long l) {
- this.l = l;
- }
-
- @Override
- public long getAsLong() {
- return l;
- }
-
- @Override
- public float getAsFloat() {
- return l;
- }
-
- @Override
- public double getAsDouble() {
- return l;
- }
-
- @Override
- public BigInteger getAsBigInteger() {
- return BigInteger.valueOf(l);
- }
-
- @Override
- public BigDecimal getAsBigDecimal() {
- return BigDecimal.valueOf(l);
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.writeSInt64(l);
- }
-
- @Override
- public MajorType getDataType() {
- return type;
- }
-
- @Override
- public NumericType getNumericType() {
- return NumericType.LONG;
- }
-
- @Override
- public boolean isConstant() {
- return true;
- }
-
- @Override
- public String toString() {
- return "LongScalar [l=" + l + "]";
- }
-
-
- @Override
- public int hashCode() {
- return getHashCode(l);
- }
-
- @Override
- public DataValue copy() {
- return new LongScalar(l);
- }
- }
-
- public static class IntegerScalar extends NumericValue{
- private MajorType type = MajorType.newBuilder().setMinorType(MinorType.INT).setMode(DataMode.OPTIONAL).build();
- int i;
-
- public IntegerScalar(int i){
- this.i = i;
- }
-
- @Override
- public BigInteger getAsBigInteger() {
- return BigInteger.valueOf(i);
- }
-
- @Override
- public BigDecimal getAsBigDecimal() {
- return BigDecimal.valueOf(i);
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.writeSInt32(i);
- }
-
- @Override
- public MajorType getDataType() {
- return type;
- }
-
- @Override
- public NumericType getNumericType() {
- return NumericType.INT;
- }
-
- @Override
- public long getAsLong() {
- return i;
- }
-
- @Override
- public int getAsInt() {
- return i;
- }
-
- @Override
- public float getAsFloat() {
- return i;
- }
-
- @Override
- public double getAsDouble() {
- return i;
- }
-
- @Override
- public boolean isConstant() {
- return true;
- }
-
- @Override
- public String toString() {
- return "IntegerScalar [i=" + i + "]";
- }
-
-
- @Override
- public int hashCode() {
- return getHashCode(i);
- }
-
- @Override
- public DataValue copy() {
- return new IntegerScalar(i);
- }
- }
-
-
-
- public static class FloatScalar extends NumericValue{
-
- private MajorType type = MajorType.newBuilder().setMinorType(MinorType.FLOAT4).setMode(DataMode.OPTIONAL).build();
- float f;
- public FloatScalar(float f){
- this.f = f;
- }
-
- @Override
- public BigDecimal getAsBigDecimal() {
- return BigDecimal.valueOf(f);
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.writeSFloat32(f);
- }
-
- @Override
- public MajorType getDataType() {
- return type;
- }
-
- @Override
- public NumericType getNumericType() {
- return NumericType.FLOAT;
- }
-
- @Override
- public double getAsDouble() {
- return f;
- }
-
- @Override
- public boolean isConstant() {
- return true;
- }
-
- @Override
- public String toString() {
- return "FloatScalar [f=" + f + "]";
- }
-
- @Override
- public int hashCode() {
- return getHashCode(f);
- }
-
- @Override
- public DataValue copy() {
- return new FloatScalar(f);
- }
- }
-
-
- public static class DoubleScalar extends NumericValue{
- private MajorType type = MajorType.newBuilder().setMinorType(MinorType.FLOAT8).setMode(DataMode.OPTIONAL).build();
- private double d;
-
- public DoubleScalar(double d){
- this.d = d;
- }
-
- @Override
- public MajorType getDataType() {
- return type;
- }
-
- @Override
- public NumericType getNumericType() {
- return NumericType.DOUBLE;
- }
-
-
- @Override
- public double getAsDouble() {
- return d;
- }
-
-
- @Override
- public BigDecimal getAsBigDecimal() {
- return BigDecimal.valueOf(d);
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.writeSFloat64(d);
- }
-
- @Override
- public boolean isConstant() {
- return true;
- }
-
- @Override
- public String toString() {
- return "DoubleScalar [d=" + d + "]";
- }
-
- @Override
- public int hashCode() {
- return getHashCode(d);
- }
-
- @Override
- public DataValue copy() {
- return new DoubleScalar(d);
- }
- }
-
- public static class BytesScalar extends BaseDataValue implements BytesValue{
- private MajorType type = MajorType.newBuilder().setMinorType(MinorType.VARBINARY).setMode(DataMode.OPTIONAL).build();
- private BytesWritable.Comparator comparator = new BytesWritable.Comparator();
- private final static HashFunction HASH = Hashing.murmur3_32();
-
- private byte[] bytes;
- public BytesScalar(byte[] value){
- this.bytes = value;
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.writeBytes(bytes);
- }
-
- @Override
- public boolean supportsCompare(DataValue dv2) {
- return Types.isBytesScalarType(dv2.getDataType());
- }
-
-
- @Override
- public int compareTo(DataValue other) {
- byte[] b1 = bytes;
- byte[] b2 = other.getAsBytesValue().getAsArray();
- return comparator.compare(b1, 0, bytes.length, b2, 0, bytes.length);
- }
-
- @Override
- public MajorType getDataType() {
- return type;
- }
-
- @Override
- public byte[] getAsArray() {
- return bytes;
- }
-
- @Override
- public int getLength() {
- return bytes.length;
- }
-
- @Override
- public byte get(int pos) {
- return bytes[pos];
- }
-
- @Override
- public String toString() {
- return "BytesScalar [bytes=" + Arrays.toString(bytes) + "]";
- }
-
- @Override
- public boolean equals(DataValue v) {
- if(!v.getDataType().equals(this.getDataType())) return false;
- BytesValue other = v.getAsBytesValue();
- if(this.getLength() != other.getLength()) return false;
- for(int i =0; i < this.getLength(); i++){
- if(this.get(i) != other.get(i)) return false;
- }
- return true;
- }
-
- @Override
- public int hashCode() {
- return HASH.hashBytes(bytes).asInt();
- }
-
- @Override
- public DataValue copy() {
- byte[] out = Arrays.copyOf(bytes, bytes.length);
- return new BytesScalar(out);
- }
-
- }
-
-
-
- static class NullValue extends BaseDataValue{
-
- // not sure what to do here...
- MajorType type = MajorType.newBuilder().setMode(DataMode.OPTIONAL).setMinorType(MinorType.LATE).build();
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.writeNullValue();
- }
-
- @Override
- public MajorType getDataType() {
- return type;
- }
-
- @Override
- public String toString() {
- return "NullValue";
- }
-
- @Override
- public boolean equals(DataValue v) {
- // identity since there should be only one.
- return v == this;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- @Override
- public DataValue copy() {
- return this;
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/SimpleArrayValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/SimpleArrayValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/SimpleArrayValue.java
deleted file mode 100644
index 2df0779..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/SimpleArrayValue.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Objects;
-
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.ref.rops.DataWriter;
-
-
-public class SimpleArrayValue extends BaseArrayValue{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleArrayValue.class);
-
- private static int padding = 5;
- private DataValue[] items;
- private int currentLast = -1;
-
-
- public SimpleArrayValue(int size){
- items = new DataValue[size];
- }
-
- public SimpleArrayValue(){
- this(5);
- }
-
- @Override
- public void addToArray(int index, DataValue v) {
- if( !(index < items.length)){
- DataValue[] newItems = new DataValue[index + padding];
- System.arraycopy(items, 0, newItems, 0, items.length);
- this.items = newItems;
- }
- items[index] = v;
- this.currentLast = Math.max(this.currentLast, index);
- }
-
- @Override
- public DataValue getByArrayIndex(int index) {
- if(index < items.length){
- DataValue ret = items[index];
- if(ret == null) return NULL_VALUE;
- return ret;
- }else{
- return NULL_VALUE;
- }
- }
-
- @Override
- protected int getNextIndex() {
- return currentLast+1;
- }
-
- public int size(){
- return currentLast+1;
- }
-
- @Override
- public void write(DataWriter w) throws IOException {
- w.writeArrayStart(currentLast+1);
- for(int i = 0; i <= currentLast; i++){
- w.writeArrayElementStart();
- DataValue v = items[i];
- if(v == null){
- w.writeNullValue();
- }else{
- v.write(w);
- }
- w.writeArrayElementEnd();
- }
- w.writeArrayEnd();
- }
-
-
- @Override
- public void append(BaseArrayValue container) {
- int curLen = size();
- int otherLen = container.size();
-
- // go backwards so the array gets resized first.
- for(int i = otherLen -1; i > -1; i--){
- DataValue v = container.getByArrayIndex(otherLen);
- this.addToArray(i + curLen, v);
- }
-
- }
-
- @Override
- public BaseArrayValue getAsArray() {
- return this;
- }
-
- @Override
- public BaseMapValue getAsMap() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public String toString() {
- return "SimpleArrayValue [items=" + Arrays.toString(items) + ", currentLast=" + currentLast + "]";
- }
-
- @Override
- public boolean equals(DataValue v) {
- if(v.getDataType().getMinorType() == MinorType.REPEATMAP) return false;
- BaseArrayValue other = v.getAsContainer().getAsArray();
- if(this.size() != other.size()) return false;
- for(int i =0; i < this.size(); i++){
- DataValue v1 = this.getByArrayIndex(i);
- DataValue v2 = other.getByArrayIndex(i);
- if(!v1.equals(v2)) return false;
- }
- return true;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash((Object[]) items);
- }
-
- @Override
- public DataValue copy() {
- SimpleArrayValue out = new SimpleArrayValue(this.size());
- for(int i =0; i < this.size(); i++)
- out.addToArray(i, this.getByArrayIndex(i));
-
- return out;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/SimpleMapValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/SimpleMapValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/SimpleMapValue.java
deleted file mode 100644
index 645862f..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/SimpleMapValue.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.exec.ref.exceptions.RecordException;
-import org.apache.drill.exec.ref.rops.DataWriter;
-
-public class SimpleMapValue extends BaseMapValue{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleMapValue.class);
-
- private HashMap<CharSequence, DataValue> map = new HashMap<CharSequence, DataValue>();
-
-
- @Override
- public void setByName(CharSequence name, DataValue v) {
- if( v== null) throw new RecordException(String.format("You attempted to write a null value with the map key of %s.", name), null);
- map.put(name, v);
- }
-
- @Override
- protected DataValue getByName(CharSequence name) {
- return map.get(name);
- }
-
- @Override
- protected void removeByName(CharSequence name) {
- map.remove(name);
- }
-
- @Override
- public void write(DataWriter w) throws IOException {
- w.writeMapStart();
-
- for(Map.Entry<CharSequence, DataValue> e : map.entrySet()){
- DataValue v = e.getValue();
- // skip null values.
- if(v == null) continue;
-// logger.debug("Writing key {}", e.getKey());
- w.writeMapKey(e.getKey());
- w.writeMapValueStart();
- v.write(w);
- w.writeMapValueEnd();
- }
-
- w.writeMapEnd();
- }
-
- @Override
- public Iterator<Entry<CharSequence, DataValue>> iterator() {
- return map.entrySet().iterator();
- }
-
- @Override
- public BaseArrayValue getAsArray() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public BaseMapValue getAsMap() {
- return this;
- }
-
- @Override
- public boolean equals(DataValue v) {
- if(v == null) return false;
- if(v.getDataType().getMode() != DataMode.REPEATED) return false;
- BaseMapValue other = v.getAsContainer().getAsMap();
- for(Entry<CharSequence, DataValue> e : this){
- DataValue v2 = other.getByName(e.getKey());
- if(!e.getValue().equals(v2)) return false;
- }
- return true;
- }
-
- @Override
- public int hashCode() {
- return map.hashCode();
- }
-
- public DataValue copy() {
- SimpleMapValue out = new SimpleMapValue();
- for(Map.Entry<CharSequence, DataValue> entry : map.entrySet()) {
- out.setByName(entry.getKey().toString(), entry.getValue().copy());
- }
- return out;
- }
-
- @Override
- public String toString() {
- final int maxLen = 10;
- return "SimpleMapValue [map=" + (map != null ? toString(map.entrySet(), maxLen) : null) + "]";
- }
-
- private String toString(Collection<?> collection, int maxLen) {
- StringBuilder builder = new StringBuilder();
- builder.append("[");
- int i = 0;
- for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
- if (i > 0) builder.append(", ");
- builder.append(iterator.next());
- }
- builder.append("]");
- return builder.toString();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/StringValue.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/StringValue.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/StringValue.java
deleted file mode 100644
index 9fcee51..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/StringValue.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-public interface StringValue extends DataValue, ComparableValue{
- public abstract CharSequence getString();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ValueReader.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ValueReader.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ValueReader.java
deleted file mode 100644
index 3ddf2c6..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ValueReader.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-
-public class ValueReader {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueReader.class);
-
- public static boolean getBoolean(DataValue v){
- if(v.getDataType().getMinorType() == MinorType.BIT && v.getDataType().getMode() != DataMode.REPEATED){
- return v.getAsBooleanValue().getBoolean();
- }else{
- throw new DrillRuntimeException(String.format("Unable to get boolean. Type was a %s", v.getClass().getCanonicalName()));
- }
- }
-
- public static long getLong(DataValue v){
- if(Types.isNumericType(v.getDataType())){
- return v.getAsNumeric().getAsLong();
- }else{
- throw new DrillRuntimeException(String.format("Unable to get value. %s is not a numeric type.", v.getClass().getCanonicalName()));
- }
- }
- public static double getDouble(DataValue v){
- if(Types.isNumericType(v.getDataType())){
- return v.getAsNumeric().getAsDouble();
- }else{
- throw new DrillRuntimeException(String.format("Unable to get value. %s is not a numeric type.", v.getClass().getCanonicalName()));
- }
- }
- public static CharSequence getChars(DataValue v){
- if(Types.isStringScalarType(v.getDataType())){
- return v.getAsStringValue().getString();
- }else{
- throw new DrillRuntimeException(String.format("Unable to get value. %s is not a StringValue type.", v.getClass().getCanonicalName()));
- }
- }
-
- public static String getString(DataValue v){
- return getChars(v).toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ValueUtils.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ValueUtils.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ValueUtils.java
deleted file mode 100644
index ef1f30d..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/values/ValueUtils.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.values;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.ValueExpressions.CollisionBehavior;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.ref.exceptions.RecordException;
-
-public class ValueUtils {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueUtils.class);
-
- public static DataValue getMergedDataValue(CollisionBehavior behavior, DataValue oldValue, DataValue newValue){
-// logger.debug("Checking merged to see if merged value should be added. Segment {}, oldValue: {}, new value: " + newValue, segment, oldValue);
- if(oldValue == DataValue.NULL_VALUE) return newValue;
-
- switch(behavior){
- case SKIP:
- return oldValue;
- case REPLACE:
- return newValue;
- case FAIL:
- throw new RecordException("Failure while doing query. The destination specified already contains a value and no collision behavior was provdied.", null);
- case OBJECTIFY:
- SimpleMapValue n = new SimpleMapValue();
- n.setByName("old", oldValue);
- n.setByName("new", newValue);
- return n;
- case ARRAYIFY:
- SimpleArrayValue a = new SimpleArrayValue();
- a.addToArray(0, oldValue);
- a.addToArray(1, newValue);
- return a;
- case MERGE_OVERRIDE:
- MajorType oldT = oldValue.getDataType();
- MajorType newT = newValue.getDataType();
- if(oldT.getMinorType() == MinorType.REPEATMAP && newT.getMinorType() == MinorType.REPEATMAP){
- oldValue.getAsContainer().getAsMap().merge(newValue.getAsContainer().getAsMap());
- return oldValue;
- }else if(oldT.getMode() == DataMode.REPEATED && newT.getMode() == DataMode.REPEATED){
- logger.debug("Merging two arrays. {} and {}", oldValue, newValue);
- oldValue.getAsContainer().getAsArray().append(newValue.getAsContainer().getAsArray());
- return oldValue;
- }else if(oldT.getMode() == DataMode.REPEATED || newT.getMode() == DataMode.REPEATED || oldT.getMinorType() == MinorType.REPEATMAP || newT.getMinorType() == MinorType.REPEATMAP){
- throw new RecordException(String.format("Failure while doing query. You requested a merge of values that were incompatibile. Examples include merging an array and a map or merging a map/array with a scalar. Merge Types were %s and %s.", oldT, newT), null);
- }else{
- // scalar type, just override the value.
- return newValue;
- }
- default:
- throw new UnsupportedOperationException();
-
- }
- }
-
- public static DataValue getIntermediateValues(PathSegment missing, DataValue value){
- if(missing == null) return value;
-
- DataValue current = missing.isArray() ? new SimpleArrayValue() : new SimpleMapValue();
- current.addValue(missing, value);
- return current;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/resources/drill-module.conf b/exec/ref/src/main/resources/drill-module.conf
deleted file mode 100644
index c3b50e8..0000000
--- a/exec/ref/src/main/resources/drill-module.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-// This file tells Drill to consider this module when class path scanning.
-// This file can also include any supplementary configuration information.
-// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
-drill.storage.packages += "org.apache.drill.exec.ref.rse"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/resources/logback-test.xml b/exec/ref/src/main/resources/logback-test.xml
deleted file mode 100644
index 3d970a1..0000000
--- a/exec/ref/src/main/resources/logback-test.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
- license agreements. See the NOTICE file distributed with this work for additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
-<configuration>
-
- <appender name="SOCKET" class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
- <Compressing>true</Compressing>
- <ReconnectionDelay>10000</ReconnectionDelay>
- <IncludeCallerData>true</IncludeCallerData>
- <RemoteHosts>localhost</RemoteHosts>
- </appender>
-
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <!-- encoders are assigned the type
- ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
- </encoder>
- </appender>
-
-
- <logger name="org.apache.drill" additivity="false">
- <level value="debug" />
- <appender-ref ref="SOCKET" />
- <appender-ref ref="STDOUT" />
- </logger>
-
- <root>
- <level value="error" />
- <appender-ref ref="SOCKET" />
- <appender-ref ref="STDOUT" />
- </root>
-
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/RSERegistryTest.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/RSERegistryTest.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/RSERegistryTest.java
deleted file mode 100644
index 47db9ec..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/RSERegistryTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.ref.rse.ConsoleRSE;
-import org.apache.drill.exec.ref.rse.RSERegistry;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertSame;
-
-
-public class RSERegistryTest {
-
- @Test
- public void testEnginesWithTheSameNameAreEqual() {
- DrillConfig config = DrillConfig.create();
- RSERegistry rses = new RSERegistry(config);
- StorageEngineConfig hconfig = new ConsoleRSE.ConsoleRSEConfig();
- ConsoleRSE engine = (ConsoleRSE) rses.getEngine(hconfig);
- ConsoleRSE engine2 = (ConsoleRSE) rses.getEngine(hconfig);
- assertSame(engine, engine2);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/RunSimplePlan.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/RunSimplePlan.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/RunSimplePlan.java
deleted file mode 100644
index f4992e0..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/RunSimplePlan.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import static org.junit.Assert.*;
-
-import java.util.Collection;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
-import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
-import org.apache.drill.exec.ref.rse.RSERegistry;
-import org.junit.Test;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-
-public class RunSimplePlan{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RunSimplePlan.class);
-
-
- @Test
- public void parseSimplePlan() throws Throwable{
- DrillConfig config = DrillConfig.create();
- LogicalPlan plan = LogicalPlan.parse(config, Files.toString(FileUtils.getResourceAsFile("/simple_plan.json"), Charsets.UTF_8));
- IteratorRegistry ir = new IteratorRegistry();
- ReferenceInterpreter i = new ReferenceInterpreter(plan, ir, new BasicEvaluatorFactory(ir), new RSERegistry(config));
- i.setup();
- Collection<RunOutcome> outcomes = i.run();
- assertEquals(outcomes.size(), 1);
- assertEquals(outcomes.iterator().next().records, 2);
- }
-
- @Test
- public void joinPlan() throws Throwable{
- DrillConfig config = DrillConfig.create();
- LogicalPlan plan = LogicalPlan.parse(config, Files.toString(FileUtils.getResourceAsFile("/simple_join.json"), Charsets.UTF_8));
- IteratorRegistry ir = new IteratorRegistry();
- ReferenceInterpreter i = new ReferenceInterpreter(plan, ir, new BasicEvaluatorFactory(ir), new RSERegistry(config));
- i.setup();
- Collection<RunOutcome> outcomes = i.run();
- assertEquals(outcomes.size(), 1);
- RunOutcome out = outcomes.iterator().next();
- if(out.outcome != OutcomeType.FAILED && out.exception != null) logger.error("Failure while running {}", out.exception);
- }
-
- @Test
- public void flattenPlan() throws Throwable{
- DrillConfig config = DrillConfig.create();
- LogicalPlan plan = LogicalPlan.parse(config, Files.toString(FileUtils.getResourceAsFile("/simple_plan_flattened.json"), Charsets.UTF_8));
- IteratorRegistry ir = new IteratorRegistry();
- ReferenceInterpreter i = new ReferenceInterpreter(plan, ir, new BasicEvaluatorFactory(ir), new RSERegistry(config));
- i.setup();
- Collection<RunOutcome> outcomes = i.run();
- assertEquals(outcomes.size(), 1);
- RunOutcome out = outcomes.iterator().next();
- if(out.outcome != OutcomeType.FAILED && out.exception != null) logger.error("Failure while running {}", out.exception);
- assertEquals(out.outcome, RunOutcome.OutcomeType.SUCCESS);
- }
-}
[37/38] git commit: Implement storage plugin for INFORMATION_SCHEMA
Posted by ja...@apache.org.
Implement storage plugin for INFORMATION_SCHEMA
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0ca797a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0ca797a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0ca797a4
Branch: refs/heads/master
Commit: 0ca797a41a0fc7f36934c259b4c179f9713e9b8e
Parents: ae041d0
Author: John Morris <jm...@maprtech.com>
Authored: Tue Feb 25 00:31:11 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:18 2014 -0800
----------------------------------------------------------------------
.../apache/drill/exec/ops/FragmentContext.java | 2 +-
.../apache/drill/exec/record/SchemaBuilder.java | 4 +-
.../drill/exec/store/StoragePluginRegistry.java | 9 +-
.../exec/store/ischema/EmptyVectorSet.java | 240 +++++++++++++++++++
.../drill/exec/store/ischema/FixedTable.java | 93 +++++++
.../store/ischema/InfoSchemaBatchCreator.java | 21 ++
.../exec/store/ischema/InfoSchemaConfig.java | 18 ++
.../store/ischema/InfoSchemaDrillTable.java | 23 ++
.../exec/store/ischema/InfoSchemaGroupScan.java | 67 ++++++
.../store/ischema/InfoSchemaStoragePlugin.java | 74 ++++++
.../exec/store/ischema/InfoSchemaSubScan.java | 24 ++
.../exec/store/ischema/InfoSchemaTable.java | 151 ++++++++++++
.../drill/exec/store/ischema/OptiqProvider.java | 234 ++++++++++++++++++
.../drill/exec/store/ischema/PipeProvider.java | 80 +++++++
.../drill/exec/store/ischema/RowProvider.java | 27 +++
.../exec/store/ischema/RowRecordReader.java | 136 +++++++++++
.../drill/exec/store/ischema/SelectedTable.java | 44 ++++
.../drill/exec/store/ischema/VectorSet.java | 38 +++
.../drill/exec/store/ischema/package-info.java | 21 ++
.../drill/exec/store/ischema/OrphanSchema.java | 75 ++++++
.../exec/store/ischema/TestOrphanSchema.java | 141 +++++++++++
.../exec/store/ischema/TestTableProvider.java | 158 ++++++++++++
.../apache/drill/jdbc/test/TestJdbcQuery.java | 9 +
23 files changed, 1684 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 630355e..8462622 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -96,7 +96,7 @@ public class FragmentContext implements Closeable {
}
public SchemaPlus getRootSchema(){
- return null;
+ return context.getStorage().getSchemaFactory().getOrphanedRootSchema();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
index 556e68b..7c6b105 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaBuilder.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.record;
+import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
@@ -31,7 +31,7 @@ import com.google.common.collect.Sets;
* builder will always check that this schema is a equal or more materialized version of the current schema.
*/
public class SchemaBuilder {
- private Set<MaterializedField> fields = Sets.newHashSet();
+ private LinkedHashSet<MaterializedField> fields = Sets.newLinkedHashSet();
private BatchSchema.SelectionVectorMode selectionVectorMode = SelectionVectorMode.NONE;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
index 0e9d1e6..ce15341 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
@@ -46,11 +46,13 @@ import org.apache.drill.exec.planner.logical.StorageEngines;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.dfs.FileSystemPlugin;
import org.apache.drill.exec.store.dfs.FormatPlugin;
+import org.apache.drill.exec.store.ischema.InfoSchemaConfig;
+import org.apache.drill.exec.store.ischema.InfoSchemaStoragePlugin;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
-import com.google.hive12.common.collect.Maps;
+
public class StoragePluginRegistry implements Iterable<Map.Entry<String, StoragePlugin>>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginRegistry.class);
@@ -92,6 +94,7 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage
}
}
+
}
private Map<String, StoragePlugin> createEngines(){
@@ -112,6 +115,8 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage
logger.error("Failure while setting up StoragePlugin with name: '{}'.", config.getKey(), e);
}
}
+ activeEngines.put("INFORMATION_SCHEMA", new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, "INFORMATION_SCHEMA"));
+
return activeEngines;
}
@@ -246,7 +251,7 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage
private class OrphanPlus implements SchemaPlus{
- private HashMap<String, SchemaPlus> schemas = Maps.newHashMap();
+ private HashMap<String, SchemaPlus> schemas = new HashMap();
@Override
public SchemaPlus getParentSchema() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/EmptyVectorSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/EmptyVectorSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/EmptyVectorSet.java
new file mode 100644
index 0000000..5379bc9
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/EmptyVectorSet.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.store.ischema;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.AllocationHelper;
+import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarCharVector;
+
+/**
+ * Manages the value vectors used to implement columns in a record batch.
+ * The vectors themselves are created by subclasses, so this class
+ * handles the generic handling of the vectors.
+ */
+public abstract class EmptyVectorSet implements VectorSet {
+
+ protected List<ValueVector> vectors;
+
+ /**
+ * Prepare to construct a new set of vectors.
+ * The actual vectors will be created by subclasses
+ * by the time our "next" procedure is invoked.
+ */
+ public EmptyVectorSet() {
+ vectors = new ArrayList<ValueVector>();
+ }
+
+ /**
+ * Prepare to read the next batch of rows.
+ * @param maxRows
+ */
+ @Override
+ public void beginBatch(int maxRows) {
+
+ // Allocate memory for each column (value vector)
+ for (ValueVector v: vectors) {
+ AllocationHelper.allocate(v, maxRows, 100); // TODO: later, use configured size
+ }
+ }
+
+
+ /**
+ * Write a row to the value vectors.
+ * This is a routine to "assign generic objects to generic ValueVectors"
+ * which can be overridden to optimize for fixed types of vectors and
+ * fixed types of values.
+ * @param index - the position within the value vectors.
+ * @param row - the objects to write into the vectors
+ * @return true if there was room to write all the values.
+ */
+ @Override
+ public boolean writeRowToVectors(int index, Object[] row) {
+ for (int i=0; i<row.length; i++) {
+ if (!setSafe(vectors.get(i), index, row[i])) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+
+ /**
+ * Signal the end of the current batch.
+ * @param actualRowCount
+ */
+ @Override
+ public void endBatch(int actualRowCount) {
+
+ // Finalize each of the value vectors.
+ for (ValueVector v: vectors) {
+ v.getMutator().setValueCount(actualRowCount);
+ }
+ }
+
+ /**
+ * When everything is done, free up the resources.
+ */
+ @Override
+ public void cleanup() {
+ for (ValueVector v: vectors) {
+ v.close();
+ }
+ }
+
+
+ /**
+ * Make the value vectors visible to whomever needs them.
+ */
+ public List<ValueVector> getValueVectors() {
+ return vectors;
+ }
+
+
+ /**
+ * Estimate how many rows will fit in a given amount of memory.
+ * Perfect estimates are nice, but things work out OK if
+ * the estimates are a bit off.
+ */
+ @Override
+ public int getEstimatedRowCount(int bufSize) {
+ return Math.max(1, bufSize/getEstimatedRowSize());
+ }
+
+
+
+ /**
+ * Estimate the size of an average row. Used for allocating memory.
+ * Override when more information is known about the data.
+ * @return bytes per row.
+ */
+ protected int getEstimatedRowSize() {
+
+ // Add up the sizes of the vectors
+ int size = 0;
+ for (ValueVector v: vectors) {
+ size += TypeHelper.getSize(v.getField().getType());
+ }
+ return size;
+ }
+
+
+ /**
+ * Helper function to create value vectors for a set of columns.
+ * @param names - the names of the fields
+ * @param types - the major types of the fields
+ * @param allocator - a buffer allocator
+ */
+ protected void createVectors(String[] names, MajorType[] types, BufferAllocator allocator) {
+ vectors = new ArrayList<ValueVector>(names.length);
+ for (int i=0; i<names.length; i++) {
+ vectors.add(createVector(names[i], types[i], allocator));
+ }
+ }
+
+
+ /**
+ * Create a value vector for a single column.
+ * @param name - the name of the field
+ * @param type - the type of the field
+ * @param allocator - a buffer allocator
+ * @return the new value vector.
+ */
+ private static ValueVector createVector(String name, MajorType type, BufferAllocator allocator) {
+ return TypeHelper.getNewVector(field(name, type), allocator);
+ }
+
+
+ /**
+ * Helper function to create a MaterializedField, used to create a ValueVector.
+ * @param name - the name of the field
+ * @param majorType - the type of the field
+ * @return the MaterializedField
+ */
+ private static MaterializedField field(String name, MajorType majorType) {
+ return MaterializedField.create(new SchemaPath(name, ExpressionPosition.UNKNOWN), majorType);
+ }
+
+
+ //////////////////////////////////////////////////////////////////
+ //
+ // The following section contains wrappers around ValueVectors.
+ // The wrappers make it easier to create vectors and set values.
+ //
+ // A different approach is to enhance TypeHelper to provide
+ // a uniform way to "setSafe" the common Java types into the type vectors.
+ // (It does that already for some types, but Strings are a particular nuisance.)
+ //
+ // For now, only types used in information schema are implemented.
+ //
+ ///////////////////////////////////////////////////////////////////
+ static final Charset UTF8 = Charset.forName("UTF-8");
+
+
+ // Here are the types used in information schema.
+ public static final MajorType VARCHAR = Types.required(MinorType.VARCHAR);
+ public static final MajorType INT = Types.required(MinorType.INT);
+ //public static final MajorType NULLABLINT = Types.optional(MinorType.INT);
+
+
+ /**
+ * A generic routine to set a Java value into a value vector. It assumes the types are compatible.
+ * When a subclass knows the types of its columns, it should use the strongly typed routines instead.
+ * <P>
+ * Note the value corresponds to what would be received by a varargs procedure.
+ * Also note we are switching on minor type. We really should switch on major type, but it is not an enum or ordinal.
+ * @return true if the value was successfully set.
+ */
+ protected static boolean setSafe(ValueVector vector, int index, Object value) {
+ switch (vector.getField().getType().getMinorType()) {
+ case INT: return setSafe((IntVector)vector, index, (int)value);
+ case VARCHAR: return setSafe((VarCharVector)vector, index, (String)value);
+ default: return false;
+ }
+ }
+
+
+ /**
+ * Strongly typed routines for setting a Java value into a value vector.
+ * @return true if the value was successfully set.
+ */
+ protected static boolean setSafe(VarCharVector v, int index, String string) {
+ return v.getMutator().setSafe(index, string.getBytes(UTF8));
+ }
+
+ protected static boolean setSafe(IntVector v, int index, int value) {
+ return v.getMutator().setSafe(index, value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/FixedTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/FixedTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/FixedTable.java
new file mode 100644
index 0000000..a2f289d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/FixedTable.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.ischema;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.sql.type.SqlTypeFactoryImpl;
+import org.eigenbase.sql.type.SqlTypeName;
+
+/**
+ * A FixedTable represents a table where the fields are always the same name and type.
+ * Since the names and types are unchanging, it is easy to create value vectors during startup
+ * and to use strong types when storing values in the vectors.
+ */
+public class FixedTable extends EmptyVectorSet {
+ String tableName;
+ String[] fieldNames;
+ MajorType[] fieldTypes;
+
+ /* (non-Javadoc)
+ * @see org.apache.drill.exec.store.ischema.VectorSet#createVectors(org.apache.drill.exec.memory.BufferAllocator)
+ */
+ @Override
+ public void createVectors(BufferAllocator allocator) {
+ createVectors(fieldNames, fieldTypes, allocator);
+ }
+
+ /**
+ * Construct a generic table with an unchanging schema.
+ * We leave it to subclasses to define the fields and types.
+ * @param tableName - name of the table
+ * @param fieldNames - names of the fields
+ * @param fieldTypes - major types of the fields
+ */
+ FixedTable(String tableName, String[] fieldNames, MajorType[] fieldTypes) {
+ this.tableName = tableName;
+ this.fieldNames = fieldNames;
+ this.fieldTypes = fieldTypes;
+ }
+
+ public String getName() {
+ return tableName;
+ }
+
+
+
+ /**
+ * Helper function to get the Optiq Schema type from a Drill Type.
+ * Again, we only do it for the information schema types, so it needs to be generalized.
+ * (This probably already exists elsewhere.)
+ */
+
+ static public RelDataType getRelDataType(RelDataTypeFactory typeFactory, MajorType type) {
+ switch (type.getMinorType()) {
+ case INT: return typeFactory.createSqlType(SqlTypeName.INTEGER);
+ case VARCHAR: return typeFactory.createSqlType(SqlTypeName.VARCHAR);
+ default: return null; // TODO - throw exception?
+ }
+ }
+
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+
+ // Convert the array of Drill types to an array of Optiq types
+ RelDataType[] relTypes = new RelDataType[fieldTypes.length];
+ for (int i=0; i<fieldTypes.length; i++) {
+ relTypes[i] = getRelDataType(typeFactory, fieldTypes[i]);
+ }
+
+ // Create a struct type to represent the
+ return typeFactory.createStructType(relTypes, fieldNames);
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java
new file mode 100644
index 0000000..68844e1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java
@@ -0,0 +1,21 @@
+package org.apache.drill.exec.store.ischema;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.physical.impl.ScanBatch;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.store.RecordReader;
+
+public class InfoSchemaBatchCreator implements BatchCreator<InfoSchemaSubScan>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaBatchCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, InfoSchemaSubScan config, List<RecordBatch> children) throws ExecutionSetupException {
+ RecordReader rr = new RowRecordReader(context, config.getTable(), context.getRootSchema());
+ return new ScanBatch(context, Collections.singleton(rr).iterator());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java
new file mode 100644
index 0000000..1447eb3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaConfig.java
@@ -0,0 +1,18 @@
+package org.apache.drill.exec.store.ischema;
+
+import org.apache.drill.common.logical.StoragePluginConfig;
+
+public class InfoSchemaConfig implements StoragePluginConfig{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaConfig.class);
+
+ @Override
+ public int hashCode(){
+ return 1;
+ }
+
+ @Override
+ public boolean equals(Object o){
+ return o instanceof InfoSchemaConfig;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java
new file mode 100644
index 0000000..7b6cb32
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java
@@ -0,0 +1,23 @@
+package org.apache.drill.exec.store.ischema;
+
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+
+public class InfoSchemaDrillTable extends DrillTable{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaDrillTable.class);
+
+ private final SelectedTable table;
+
+ public InfoSchemaDrillTable(String storageEngineName, SelectedTable selection, StoragePluginConfig storageEngineConfig) {
+ super(storageEngineName, selection, storageEngineConfig);
+ this.table = selection;
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ return table.getRowType(typeFactory);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
new file mode 100644
index 0000000..1985d6a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java
@@ -0,0 +1,67 @@
+package org.apache.drill.exec.store.ischema;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
+import org.apache.drill.exec.physical.EndpointAffinity;
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.SubScan;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.hive12.common.base.Preconditions;
+
+@JsonTypeName("info-schema")
+public class InfoSchemaGroupScan extends AbstractGroupScan{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaGroupScan.class);
+
+ private final SelectedTable table;
+
+ @JsonCreator
+ public InfoSchemaGroupScan(@JsonProperty("table") SelectedTable table) {
+ this.table = table;
+ }
+
+ @Override
+ public void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException {
+ Preconditions.checkArgument(endpoints.size() == 1);
+ }
+
+ @Override
+ public SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException {
+ Preconditions.checkArgument(minorFragmentId == 0);
+ return new InfoSchemaSubScan(table);
+ }
+
+ @Override
+ public int getMaxParallelizationWidth() {
+ return 1;
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ return new OperatorCost(1,1,1,1);
+ }
+
+ @Override
+ public Size getSize() {
+ return new Size(1000, 1000);
+ }
+
+ @Override
+ public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
+ return this;
+ }
+
+ @Override
+ public List<EndpointAffinity> getOperatorAffinity() {
+ return Collections.emptyList();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
new file mode 100644
index 0000000..e5ab158
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
@@ -0,0 +1,74 @@
+package org.apache.drill.exec.store.ischema;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.AbstractStoragePlugin;
+import org.apache.drill.exec.store.SchemaHolder;
+
+import com.google.hive12.common.collect.ImmutableMap;
+import com.google.hive12.common.collect.Maps;
+
+public class InfoSchemaStoragePlugin extends AbstractStoragePlugin{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaStoragePlugin.class);
+
+ private final InfoSchemaConfig config;
+ private final DrillbitContext context;
+ private final String name;
+
+ public InfoSchemaStoragePlugin(InfoSchemaConfig config, DrillbitContext context, String name){
+ this.config = config;
+ this.context = context;
+ this.name = name;
+ }
+
+ @Override
+ public boolean supportsRead() {
+ return true;
+ }
+
+ @Override
+ public InfoSchemaGroupScan getPhysicalScan(Scan scan) throws IOException {
+ SelectedTable table = scan.getSelection().getWith(context.getConfig(), SelectedTable.class);
+ return new InfoSchemaGroupScan(table);
+ }
+
+ @Override
+ public Schema createAndAddSchema(SchemaPlus parent) {
+ Schema s = new ISchema(parent);
+ parent.add(s);
+ return s;
+ }
+
+ private class ISchema extends AbstractSchema{
+ private Map<String, InfoSchemaDrillTable> tables;
+ public ISchema(SchemaPlus parent){
+ super(new SchemaHolder(parent), "INFORMATION_SCHEMA");
+ Map<String, InfoSchemaDrillTable> tbls = Maps.newHashMap();
+ for(SelectedTable tbl : SelectedTable.values()){
+ tbls.put(tbl.name(), new InfoSchemaDrillTable("INFORMATION_SCHEMA", tbl, config));
+ }
+ this.tables = ImmutableMap.copyOf(tbls);
+ }
+
+ @Override
+ public DrillTable getTable(String name) {
+ return tables.get(name);
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return tables.keySet();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java
new file mode 100644
index 0000000..20375ca
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaSubScan.java
@@ -0,0 +1,24 @@
+package org.apache.drill.exec.store.ischema;
+
+import org.apache.drill.exec.physical.base.AbstractSubScan;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class InfoSchemaSubScan extends AbstractSubScan{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(InfoSchemaSubScan.class);
+
+
+ private final SelectedTable table;
+
+ @JsonCreator
+ public InfoSchemaSubScan(@JsonProperty("table") SelectedTable table) {
+ this.table = table;
+ }
+
+ public SelectedTable getTable() {
+ return table;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java
new file mode 100644
index 0000000..595488f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTable.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.ischema;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.VarCharVector;
+
+/**
+ * InfoSchemaTable defines the various Information Schema tables.
+ * <p>
+ * All the information schema tables are grouped together for convenience.
+ * For each specific table, the corresponding class:
+ * <p>Declares the table name.
+ * <p>Declares the field names and types.
+ * <p>Optionally defines a typed method to write a row of data to the vectors.
+ * If not defined here, FixedTable will kick in and do the job using
+ * a slower, generic method.
+ */
+public class InfoSchemaTable{
+
+ /**
+ * Layout for the SCHEMATA table.
+ */
+ public static class Schemata extends FixedTable {
+ static final String tableName = "SCHEMATA";
+ static final String[] fieldNames = {"CATALOG_NAME", "SCHEMA_NAME", "SCHEMA_OWNER"};
+ static final MajorType[] fieldTypes = {VARCHAR, VARCHAR, VARCHAR};
+
+ public Schemata() {
+ super(tableName, fieldNames, fieldTypes);
+ }
+
+ // Optional ...
+ public boolean writeRowToVectors(int index, Object[] row) {
+ return
+ setSafe((VarCharVector)vectors.get(0), index, (String)row[0]) &&
+ setSafe((VarCharVector)vectors.get(1), index, (String)row[1]) &&
+ setSafe((VarCharVector)vectors.get(2), index, (String)row[2]);
+ }
+ }
+
+ /**
+ * Layout for the TABLES table.
+ */
+ public static class Tables extends FixedTable {
+ static final String tableName = "TABLES";
+ static final String[] fieldNames = {"TABLE_CATALOG", "TABLE_SCHEMA", "TABLE_NAME", "TABLE_TYPE"};
+ static final MajorType[] fieldTypes = {VARCHAR, VARCHAR, VARCHAR, VARCHAR};
+
+ public Tables() {
+ super(tableName, fieldNames, fieldTypes);
+ }
+
+ // Optional ...
+ public boolean writeRowToVectors(int index, Object[] row) {
+ return
+ setSafe((VarCharVector)vectors.get(0), index, (String)row[0]) &&
+ setSafe((VarCharVector)vectors.get(1), index, (String)row[1]) &&
+ setSafe((VarCharVector)vectors.get(2), index, (String)row[2]) &&
+ setSafe((VarCharVector)vectors.get(3), index, (String)row[3]);
+ }
+ }
+
+
+ /**
+ * Layout for the COLUMNS table.
+ */
+ public static class Columns extends FixedTable {
+ static final String tableName = "COLUMNS";
+ static final String[] fieldNames = {"TABLE_CATALOG", "TABLE_SCHEMA", "TABLE_NAME", "COLUMN_NAME",
+ "ORDINAL_POSITION", "IS_NULLABLE", "DATA_TYPE", "CHARACTER_MAXIMUM_LENGTH",
+ "NUMERIC_PRECISION_RADIX", "NUMERIC_SCALE", "NUMERIC_PRECISION"};
+ static final MajorType[] fieldTypes= { VARCHAR, VARCHAR, VARCHAR, VARCHAR,
+ INT, VARCHAR, VARCHAR, INT,
+ INT, INT, INT};
+ public Columns() {
+ super(tableName, fieldNames, fieldTypes);
+ }
+
+
+ // Optional ...
+ public boolean writeRowToVectors(int index, Object[] row) {
+ return
+ setSafe((VarCharVector)vectors.get(0), index, (String)row[0]) &&
+ setSafe((VarCharVector)vectors.get(1), index, (String)row[1]) &&
+ setSafe((VarCharVector)vectors.get(2), index, (String)row[2]) &&
+ setSafe((VarCharVector)vectors.get(3), index, (String)row[3]) &&
+ setSafe((IntVector)vectors.get(4), index, (int)row[4]) &&
+ setSafe((VarCharVector)vectors.get(5), index, (String)row[5]) &&
+ setSafe((VarCharVector)vectors.get(6), index, (String)row[6]) &&
+ setSafe((IntVector)vectors.get(7), index, (int)row[7]) &&
+ setSafe((IntVector)vectors.get(8), index, (int)row[8]) &&
+ setSafe((IntVector)vectors.get(9), index, (int)row[9]) &&
+ setSafe((IntVector)vectors.get(10), index, (int)row[10]);
+ }
+ }
+
+
+ /**
+ * Layout for the VIEWS table.
+ */
+ static public class Views extends FixedTable {
+ static final String tableName = "VIEWS";
+ static final String[] fieldNames = {"TABLE_CATALOG", "TABLE_SHEMA", "TABLE_NAME", "VIEW_DEFINITION"};
+ static final MajorType[] fieldTypes = {VARCHAR, VARCHAR, VARCHAR, VARCHAR};
+
+ Views() {
+ super(tableName, fieldNames, fieldTypes);
+ }
+
+ // Optional ...
+ public boolean writeRowToVectors(int index, Object[] row) {
+ return setSafe((VarCharVector)vectors.get(0), index, (String)row[0]) &&
+ setSafe((VarCharVector)vectors.get(1), index, (String)row[1]) &&
+ setSafe((VarCharVector)vectors.get(2), index, (String)row[2]) &&
+ setSafe((VarCharVector)vectors.get(3), index, (String)row[3]);
+ }
+ }
+
+
+ /**
+ * Layout for the CATALOGS table.
+ */
+ static public class Catalogs extends FixedTable {
+ static final String tableName = "CATALOGS";
+ static final String[] fieldNames = {"CATALOG_NAME", "CATALOG_DESCRIPTION", "CATALOG_CONNECT"};
+ static final MajorType[] fieldTypes = {VARCHAR, VARCHAR, VARCHAR};
+
+ Catalogs() {
+ super(tableName, fieldNames, fieldTypes);
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
new file mode 100644
index 0000000..795b92c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/OptiqProvider.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.ischema;
+
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+import net.hydromatic.optiq.Table;
+
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeField;
+import org.eigenbase.sql.type.SqlTypeFactoryImpl;
+import org.eigenbase.sql.type.SqlTypeName;
+
+/**
+ * OptiqProvider provides data for the various tables in the information schema.
+ * Each table has its own nested class, keeping them grouped together.
+ * Note "writeRow(...)" must match the values expected by the corresponding table.
+ * <p>
+ * To keep code concise, each class inherits from both an OptiqScanner
+ * and a PipeProvider. Double inheritance is a problem in Java, so this
+ * code needs to be cleaned up. For the moment, OptiqScanner artificially
+ * inherits from PipeProvider.
+ */
+public class OptiqProvider {
+
+ /**
+ * Provide data for TABLES table.
+ */
+ static public class Tables extends Abstract {
+ Tables(SchemaPlus root) {
+ super(root);
+ }
+
+ @Override
+ public boolean visitTableName(String schema, String tableName) {
+ return writeRow("DRILL", schema, tableName, Schema.TableType.TABLE.toString());
+ }
+ }
+
+
+ /**
+ * Provide data for SCHEMATA table.
+ */
+ static public class Schemata extends Abstract {
+ @Override
+ public boolean visitSchema(String schemaName, Schema schema) {
+ if (schemaName != null && schemaName != "") {
+ writeRow("DRILL", schemaName, "<owner>");
+ }
+ return false;
+ }
+
+ Schemata(SchemaPlus root) {
+ super(root);
+ }
+ }
+
+
+
+ /**
+ * Provide data for COLUMNS data.
+ */
+ static public class Columns extends Abstract {
+
+ public Columns(SchemaPlus root) {
+ super(root);
+ }
+
+ @Override
+ public boolean visitField(String schemaName, String tableName, RelDataTypeField field) {
+ String columnName = field.getName();
+ RelDataType type = field.getType();
+ SqlTypeName sqlType = type.getSqlTypeName();
+
+ int position = field.getIndex();
+ String nullable;
+ if (type.isNullable()) nullable = "YES";
+ else nullable = "NO";
+ String sqlTypeName = sqlType.getName();
+ int radix = (sqlType == SqlTypeName.DECIMAL)?10:-1; // TODO: where do we get radix?
+ int charMaxLen = -1; // TODO: where do we get char length?
+ int scale = (sqlType.allowsPrec())?type.getScale(): -1;
+ int precision = (sqlType.allowsScale())?type.getPrecision(): -1;
+
+ writeRow("DRILL", schemaName, tableName, columnName, position, nullable, sqlTypeName, charMaxLen, radix, scale, precision);
+
+ return false;
+ }
+ }
+
+
+
+ /**
+ * Provide data for VIEWS table
+ */
+ public static class Views extends Abstract {
+ public Views(SchemaPlus root) {
+ super(root);
+ }
+ @Override
+ public boolean visitTable(String schemaName, String tableName, Table table) {
+ if (table.getJdbcTableType() == Schema.TableType.VIEW) {
+ writeRow("DRILL", schemaName, tableName, "TODO: GetViewDefinition");
+ }
+ return false;
+ }
+ }
+
+ public static class Catalogs extends Abstract {
+ public Catalogs(SchemaPlus root) {
+ super(root);
+ }
+ @Override
+ public void generateRows() {
+ writeRow("DRILL", "The internal metadata used by Drill", "");
+ }
+ }
+
+
+ /**
+ * An abstract class which helps generate data. It does the actual scanning of an Optiq schema,
+ * but relies on a subclass to provide a "visit" routine to write out data.
+ */
+ public static class Abstract extends OptiqScanner {
+ SchemaPlus root;
+
+ protected Abstract(SchemaPlus root) {
+ this.root = root;
+ }
+
+
+ /**
+ * Start writing out rows.
+ */
+ @Override
+ public void generateRows() {
+
+ // Scan the root schema for subschema, tables, columns.
+ scanSchema(root);
+ }
+ }
+
+
+
+ /**
+ * An OptiqScanner scans the Optiq schema, generating rows for each
+ * schema, table or column. It is intended to be subclassed, where the
+ * subclass does what it needs when visiting a Optiq schema structure.
+ */
+ // We would really prefer multiple inheritance from both OptiqScanner and PipeProvider,
+ // but making one a subclass of the other works for now.
+ // TODO: Refactor to avoid subclassing of what should be an unrelated class.
+ abstract static class OptiqScanner extends PipeProvider {
+
+
+ /**
+ * Each visitor implements at least one of the the following methods.
+ * If the schema visitor returns true, then visit the tables.
+ * If the table visitor returns true, then visit the fields (columns).
+ */
+ public boolean visitSchema(String schemaName, Schema schema){return true;}
+ public boolean visitTableName(String schemaName, String tableName){return true;}
+ public boolean visitTable(String schemaName, String tableName, Table table){return true;}
+ public boolean visitField(String schemaName, String tableName, RelDataTypeField field){return true;}
+
+
+
+ /**
+ * Start scanning an Optiq Schema.
+ * @param root - where to start
+ */
+ protected void scanSchema(Schema root) {
+ scanSchema(root.getName(), root);
+ }
+
+ /**
+ * Recursively scan the schema, invoking the visitor as appropriate.
+ * @param schemaPath - the path to the current schema, so far,
+ * @param schema - the current schema.
+ * @param visitor - the methods to invoke at each entity in the schema.
+ */
+ private void scanSchema(String schemaPath, Schema schema) {
+
+ // If we have an empty schema path, then don't insert a leading dot.
+ String separator;
+ if (schemaPath == "") separator = "";
+ else separator = ".";
+
+ // Recursively scan the subschema.
+ for (String name: schema.getSubSchemaNames()) {
+ scanSchema(schemaPath + separator + name, schema.getSubSchema(name));
+ }
+
+ // Visit this schema and if requested ...
+ if (visitSchema(schemaPath, schema)) {
+
+ // ... do for each of the schema's tables.
+ for (String tableName: schema.getTableNames()) {
+ if(visitTableName(schemaPath, tableName)){
+ Table table = schema.getTable(tableName);
+
+ // Visit the table, and if requested ...
+ if (visitTable(schemaPath, tableName, table)) {
+
+ // ... do for each of the table's fields.
+ RelDataType tableRow = table.getRowType(new SqlTypeFactoryImpl()); // TODO: Is this correct?
+ for (RelDataTypeField field: tableRow.getFieldList()) {
+
+ // Visit the field.
+ visitField(schemaPath, tableName, field);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/PipeProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/PipeProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/PipeProvider.java
new file mode 100644
index 0000000..30d7d76
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/PipeProvider.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.ischema;
+
+import java.util.ArrayList;
+import java.util.ListIterator;
+
+/**
+ * PipeProvider sets up the framework so some subclass can "write" rows
+ * to a an internal pipe, which another class (RowRecordReader) can "read" to
+ * build up a record batch.
+ * <p>
+ * This class helps work around the situation where the rows cannot be conveniently
+ * be generated one at a time by an iterator. Logically, the "writer" writes rows to the pipe,
+ * while a "reader" reads rows from the pipe. The vocabulary implies two separate threads,
+ * but the current implementation is actually just a wrapper around a List.
+ */
+public abstract class PipeProvider implements RowProvider {
+ ArrayList<Object[]> pipe = null;
+ ListIterator<Object[]> iter;
+
+ /**
+ * Method to generate and write rows to the pipe.
+ */
+ abstract void generateRows();
+
+ /**
+ * true if there are rows waiting to be "read".
+ */
+ public boolean hasNext() {
+ if (pipe == null) {
+ pipe = new ArrayList<Object[]>();
+ generateRows();
+ iter = pipe.listIterator();
+ }
+ return iter.hasNext();
+ }
+
+ /**
+ * Read the next row from the pipe.
+ * Should only be called after "hasNext" indicates there are more rows.
+ */
+ public Object[] next() {
+ return iter.next();
+ }
+
+ /**
+ * Sometimes, a row cannot be immediately processed. Put the last row back and re-read it next time.
+ */
+ public void previous() {
+ iter.previous();
+ }
+
+ /**
+ * Write a row to the pipe.
+ * @param values - a varargs list of values, in the same order as the RecordReader's value vectors.
+ * @return true if the row was successfully written to the pipe.
+ */
+ protected boolean writeRow(Object...values) {
+ pipe.add(values);
+ return true;
+ }
+
+}
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RowProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RowProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RowProvider.java
new file mode 100644
index 0000000..c26c836
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RowProvider.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.ischema;
+
+/**
+ * An interface for providing rows of data.
+ */
+public interface RowProvider {
+ public Object[] next(); // Fetch the next row of values
+ public boolean hasNext(); // true if there are rows remaining to fetch.
+ public void previous(); // Put back the last row read, so it can be reread. (Only one row can be put back)
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RowRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RowRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RowRecordReader.java
new file mode 100644
index 0000000..4832132
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RowRecordReader.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.store.ischema;
+
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.vector.ValueVector;
+
+
+
+
+/**
+ * RowRecordReader is a RecordReader which creates RecordBatchs by
+ * reading rows one at a time. The fixed format rows come from a "RowProvider".
+ */
+public class RowRecordReader implements RecordReader {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RowRecordReader.class);
+
+ protected final VectorSet batch;
+ protected final RowProvider provider;
+ protected final FragmentContext context;
+ protected OutputMutator output;
+
+ private int bufSize = 32*1024*1024;
+ private int maxRowCount;
+ /**
+ * Construct a RecordReader which uses rows from a RowProvider and puts them into a set of value vectors.
+ * @param context
+ * @param vectors
+ */
+ public RowRecordReader(FragmentContext context, VectorSet batch, RowProvider provider) {
+ this.context = context;
+ this.provider = provider;
+ this.batch = batch;
+ }
+
+ public RowRecordReader(FragmentContext context, SelectedTable table, SchemaPlus rootSchema){
+ this.context = context;
+ this.provider = table.getProvider(rootSchema);
+ this.batch = table.getFixedTable();
+ }
+
+ /**
+ * Prepare to create record batches.
+ */
+ @Override
+ public void setup(OutputMutator output) throws ExecutionSetupException {
+ this.output = output;
+ batch.createVectors(context.getAllocator());
+
+ // Inform drill of the output columns. They were set up when the vector handler was created.
+ // Note we are currently working with fixed tables.
+ try {
+ for (ValueVector v: batch.getValueVectors()) {
+ output.addField(v);;
+ }
+ output.setNewSchema();
+ } catch (SchemaChangeException e) {
+ throw new ExecutionSetupException("Failure while setting up fields", e);
+ }
+
+ // Estimate the number of records we can hold in a RecordBatch
+ maxRowCount = batch.getEstimatedRowCount(bufSize);
+ }
+
+
+
+ /**
+ * Return the next record batch. An empty batch means end of data.
+ */
+ @Override
+ public int next() {
+
+ // Make note are are starting a new batch of records
+ batch.beginBatch(maxRowCount);
+
+ // Repeat until out of data or vectors are full
+ int actualCount;
+ for (actualCount = 0; actualCount < maxRowCount && provider.hasNext(); actualCount++) {
+
+ // Put the next row into the vectors. If vectors full, try again later.
+ Object[] row = provider.next();
+ if (!batch.writeRowToVectors(actualCount, row)) {
+ provider.previous();
+ break;
+ }
+ }
+
+ // Make note the batch is complete.
+ batch.endBatch(actualCount);
+
+ // Problem if we had a single row which didn't fit.
+ if (actualCount == 0 && provider.hasNext()) {
+ throw new DrillRuntimeException("Row size larger than batch size");
+ }
+
+ // Return the number of rows. 0 means end of data.
+ return actualCount;
+ }
+
+
+
+ /**
+ * Release all resources
+ */
+ public void cleanup() {
+ batch.cleanup();
+ }
+
+
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java
new file mode 100644
index 0000000..3e354f8
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/SelectedTable.java
@@ -0,0 +1,44 @@
+package org.apache.drill.exec.store.ischema;
+
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.exec.store.ischema.InfoSchemaTable.Catalogs;
+import org.apache.drill.exec.store.ischema.InfoSchemaTable.Columns;
+import org.apache.drill.exec.store.ischema.InfoSchemaTable.Schemata;
+import org.apache.drill.exec.store.ischema.InfoSchemaTable.Tables;
+import org.apache.drill.exec.store.ischema.InfoSchemaTable.Views;
+import org.apache.drill.exec.store.ischema.OptiqProvider.OptiqScanner;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+
+public enum SelectedTable{
+ CATALOGS(new Catalogs(), new ScannerFactory(){public OptiqScanner get(SchemaPlus root) {return new OptiqProvider.Catalogs(root);}} ), //
+ SCHEMATA(new Schemata(), new ScannerFactory(){public OptiqScanner get(SchemaPlus root) {return new OptiqProvider.Schemata(root);}} ), //
+ VIEWS(new Views(), new ScannerFactory(){public OptiqScanner get(SchemaPlus root) {return new OptiqProvider.Views(root);}} ), //
+ COLUMNS(new Columns(), new ScannerFactory(){public OptiqScanner get(SchemaPlus root) {return new OptiqProvider.Columns(root);}} ), //
+ TABLES(new Tables(), new ScannerFactory(){public OptiqScanner get(SchemaPlus root) {return new OptiqProvider.Tables(root);}} ); //
+
+ private final FixedTable tableDef;
+ private final ScannerFactory providerFactory;
+
+ private SelectedTable(FixedTable tableDef, ScannerFactory providerFactory) {
+ this.tableDef = tableDef;
+ this.providerFactory = providerFactory;
+ }
+
+ public OptiqScanner getProvider(SchemaPlus root){
+ return providerFactory.get(root);
+ }
+
+ private interface ScannerFactory{
+ public OptiqScanner get(SchemaPlus root);
+ }
+
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ return tableDef.getRowType(typeFactory);
+ }
+
+ public FixedTable getFixedTable(){
+ return tableDef;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/VectorSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/VectorSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/VectorSet.java
new file mode 100644
index 0000000..cb39337
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/VectorSet.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.ischema;
+
+import java.util.List;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.vector.ValueVector;
+
+/**
+ * A collection of value vectors representing the columns in a table.
+ */
+public interface VectorSet {
+ public void cleanup();
+
+ public void beginBatch(int maxRows);
+ public boolean writeRowToVectors(int index, Object[] values);
+ public void endBatch(int actualRows);
+
+ public int getEstimatedRowCount(int bufSize);
+ public void createVectors(BufferAllocator allocator);
+ public List<ValueVector> getValueVectors();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/package-info.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/package-info.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/package-info.java
new file mode 100644
index 0000000..ea38e06
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.drill.exec.store.ischema;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
new file mode 100644
index 0000000..268b844
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/OrphanSchema.java
@@ -0,0 +1,75 @@
+package org.apache.drill.exec.store.ischema;
+
+
+import static org.mockito.Mockito.*;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.junit.Test;
+
+
+import com.codahale.metrics.MetricRegistry;
+/**
+ * OrphanSchema is a stand alone schema tree which is not connected to Optiq.
+ * This class is a refactoring of exec.store.TestOrphanSchema.java. The primary
+ * change is to package a "create()" method for providing a test schema.
+ * For convenient testing, it mocks up the Drillbit context.
+ */
+public class OrphanSchema {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OrphanSchema.class);
+
+ /**
+ * Create an orphan schema to be used for testing.
+ * @return root node of the created schema.
+ */
+ public static SchemaPlus create(){
+
+ final DrillConfig c = DrillConfig.create();
+
+ // Mock up a context which will allow us to create a schema.
+ final DrillbitContext bitContext = mock(DrillbitContext.class);
+ when(bitContext.getMetrics()).thenReturn(new MetricRegistry());
+ when(bitContext.getAllocator()).thenReturn(new TopLevelAllocator());
+ when(bitContext.getConfig()).thenReturn(c);
+
+ // Using the mock context, get the orphan schema.
+ StoragePluginRegistry r = new StoragePluginRegistry(bitContext);
+ SchemaPlus plus = r.getSchemaFactory().getOrphanedRootSchema();
+
+ return plus;
+ }
+
+
+ /**
+ * This test replicates the one in org.apache.drill.exec.server,
+ * but it is refactored to provide a standalone "create()" method.
+ */
+
+ @Test
+ public void test() {
+ printSchema(create(), 0);
+ }
+
+ private static void t(final int t){
+ for(int i =0; i < t; i++) System.out.print('\t');
+ }
+ private static void printSchema(SchemaPlus s, int indent){
+ t(indent);
+ System.out.print("Schema: ");
+ System.out.println(s.getName().equals("") ? "root" : s.getName());
+ for(String table : s.getTableNames()){
+ t(indent + 1);
+ System.out.print("Table: ");
+ System.out.println(table);
+ }
+
+ for(String schema : s.getSubSchemaNames()){
+ SchemaPlus p = s.getSubSchema(schema);
+ printSchema(p, indent + 1);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestOrphanSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestOrphanSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestOrphanSchema.java
new file mode 100644
index 0000000..c29c05f
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestOrphanSchema.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.ischema;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.store.ischema.FixedTable;
+import org.apache.drill.exec.store.ischema.InfoSchemaTable;
+import org.apache.drill.exec.store.ischema.OptiqProvider;
+import org.apache.drill.exec.store.ischema.RowProvider;
+import org.apache.drill.exec.store.ischema.RowRecordReader;
+import org.apache.drill.exec.vector.ValueVector;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Using an orphan schema, create and display the various information schema tables.
+ * An "orphan schema" is a stand alone schema which is not (yet?) connected to Optiq.
+ */
+public class TestOrphanSchema {
+ SchemaPlus root = OrphanSchema.create();
+
+ @Test
+ public void testTables() {
+ displayTable(new InfoSchemaTable.Tables(), new OptiqProvider.Tables(root));
+ }
+
+ @Test
+ public void testSchemata() {
+ displayTable(new InfoSchemaTable.Schemata(), new OptiqProvider.Schemata(root));
+ }
+
+
+ @Test
+ public void testViews() {
+ displayTable(new InfoSchemaTable.Views(), new OptiqProvider.Views(root));
+ }
+
+ @Test
+ public void testCatalogs() {
+ displayTable(new InfoSchemaTable.Catalogs(), new OptiqProvider.Catalogs(root));
+ }
+
+ @Test
+ public void testColumns() {
+ displayTable(new InfoSchemaTable.Columns(), new OptiqProvider.Columns(root));
+ }
+
+
+ private void displayTable(FixedTable table, RowProvider provider) {
+
+ // Set up a mock context
+ FragmentContext context = mock(FragmentContext.class);
+ when(context.getAllocator()).thenReturn(new TopLevelAllocator());
+
+ // Create a RecordReader which reads from the test table.
+ RecordReader reader = new RowRecordReader(context, table, provider);
+
+ // Create an dummy output mutator for the RecordReader.
+ TestOutput output = new TestOutput();
+ try {reader.setup(output);}
+ catch (ExecutionSetupException e) {Assert.fail("reader threw an exception");}
+
+ // print out headers
+ System.out.printf("\n%20s\n", table.getName());
+ System.out.printf("%10s", "RowNumber");
+ for (ValueVector v: table.getValueVectors()) {
+ System.out.printf(" | %16s", v.getField().getName());
+ }
+ System.out.println();
+
+ // Do for each record batch
+ int rowNumber = 0;
+ for (;;) {
+ int count = reader.next();
+ if (count == 0) break;
+
+ // Do for each row in the batch
+ for (int row=0; row<count; row++, rowNumber++) {
+
+ // Display the row
+ System.out.printf("%10d", rowNumber);
+ for (ValueVector v: table.getValueVectors()) {
+ System.out.printf(" | %16s", v.getAccessor().getObject(row));
+ }
+ System.out.println();
+
+ }
+ }
+ }
+
+
+ /**
+ * A dummy OutputMutator so we can examine the contents of the current batch
+ */
+ static class TestOutput implements OutputMutator {
+ List<ValueVector> vectors = new ArrayList<ValueVector>();
+
+ public void addField(ValueVector vector) throws SchemaChangeException {
+ vectors.add(vector);
+ }
+
+ public Object get(int column, int row) {
+ return vectors.get(column).getAccessor().getObject(row);
+ }
+
+ public void removeField(MaterializedField field) {}
+ public void removeAllFields() {}
+ public void setNewSchema() {}
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java
new file mode 100644
index 0000000..c4da32b
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestTableProvider.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.ischema;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.store.ischema.FixedTable;
+import org.apache.drill.exec.store.ischema.PipeProvider;
+import org.apache.drill.exec.store.ischema.RowRecordReader;
+import org.apache.drill.exec.vector.ValueVector;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Using a test table with two columns, create data and verify the values are in the record batch.
+ */
+public class TestTableProvider {
+
+ @Test
+ public void zeroRead() {
+ readTestTable(0);
+ }
+
+ @Test
+ public void oneRead() {
+ readTestTable(1);
+ }
+
+ @Test
+ public void smallRead() {
+ readTestTable(10);
+ }
+
+ @Test
+ public void largeRead() {
+ readTestTable(1024*1024);
+ }
+
+
+ /**
+ * Read record batches from the test table and verify the contents.
+ * @param nrRows - the total number of rows expected.
+ */
+ private void readTestTable(int nrRows) {
+
+ // Mock up a context with a BufferAllocator
+ FragmentContext context = mock(FragmentContext.class);
+ when(context.getAllocator()).thenReturn(new TopLevelAllocator());
+
+ // Create a RecordReader which reads from the test table.
+ RecordReader reader = new RowRecordReader(context, new TestTable(), new TestProvider(nrRows));
+
+ // Create an dummy output mutator for the RecordReader.
+ TestOutput output = new TestOutput();
+ try {reader.setup(output);}
+ catch (ExecutionSetupException e) {Assert.fail("reader threw an exception");}
+
+ // Do for each record batch
+ int rowNumber = 0;
+ for (;;) {
+ int count = reader.next();
+ if (count == 0) break;
+
+ // Do for each row in the batch
+ for (int row=0; row<count; row++, rowNumber++) {
+
+ // Verify the row has an integer and string containing the row number
+ int intValue = (int)output.get(1, row);
+ String strValue = (String)output.get(0, row);
+ Assert.assertEquals(rowNumber, intValue);
+ Assert.assertEquals(rowNumber, Integer.parseInt(strValue));
+ }
+ }
+
+ // Verify we read the correct number of rows.
+ Assert.assertEquals(nrRows, rowNumber);
+ }
+
+
+ /**
+ * Class to define the table we want to create. Two columns - string, integer
+ */
+ static class TestTable extends FixedTable {
+ static final String tableName = "MOCK_TABLE";
+ static final String fieldNames[] = {"STRING_COLUMM", "INTEGER_COLUMN"};
+ static final MajorType fieldTypes[] = {VARCHAR, INT};
+ TestTable() {
+ super(tableName, fieldNames, fieldTypes);
+ }
+ }
+
+
+ /**
+ * Class to generate data for the table
+ */
+ static class TestProvider extends PipeProvider {
+ int maxRows;
+ TestProvider(int maxRows) {
+ this.maxRows = maxRows;
+ }
+ void generateRows() {
+ for (int rowNumber=0; rowNumber<maxRows; rowNumber++) {
+ writeRow(Integer.toString(rowNumber), rowNumber);
+ }
+ }
+ }
+
+
+
+
+ /**
+ * A dummy OutputMutator so we can examine the contents of the current batch
+ */
+ static class TestOutput implements OutputMutator {
+ List<ValueVector> vectors = new ArrayList<ValueVector>();
+
+ public void addField(ValueVector vector) throws SchemaChangeException {
+ vectors.add(vector);
+ }
+
+ public Object get(int column, int row) {
+ return vectors.get(column).getAccessor().getObject(row);
+ }
+
+ public void removeField(MaterializedField field) {}
+ public void removeAllFields() {}
+ public void setNewSchema() {}
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0ca797a4/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index dc80f33..f5081b3 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -64,6 +64,14 @@ public class TestJdbcQuery {
testQuery("select * from cp.`employee.json`");
}
+ @Test
+ public void testInfoSchema() throws Exception{
+ testQuery("select * from INFORMATION_SCHEMA.SCHEMATA");
+ testQuery("select * from INFORMATION_SCHEMA.CATALOGS");
+ testQuery("select * from INFORMATION_SCHEMA.VIEWS");
+ testQuery("select * from INFORMATION_SCHEMA.TABLES");
+ testQuery("select * from INFORMATION_SCHEMA.COLUMNS");
+ }
@Test
public void testCast() throws Exception{
@@ -125,6 +133,7 @@ public class TestJdbcQuery {
System.out.println(String.format("Query completed in %d millis.", watch.elapsed(TimeUnit.MILLISECONDS)));
}
+ System.out.println("\n\n\n");
success = true;
}finally{
if(!success) Thread.sleep(2000);
[21/38] Integrate new SQL changes with Hive storage engine,
moving to automatic file detection. Rename storage engine to storage
plugin. Separate storage plugins from format plugins,
updating Parquet and JSON to format engines. Refactor distribution log
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
deleted file mode 100644
index 71e6283..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec;
-
-import org.apache.drill.exec.client.QuerySubmitter;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.IOException;
-
-/**
- * Created with IntelliJ IDEA.
- * User: sphillips
- * Date: 1/24/14
- * Time: 3:46 PM
- * To change this template use File | Settings | File Templates.
- */
-public class TestPlan {
-
- String location = "/Users/sphillips/hive-lineitem-orderkey";
- String type = "physical";
- String zkQuorum = null;
- boolean local = true;
- int bits = 1;
-
-
- @Test
- @Ignore
- public void testSubmitPlan() throws Exception {
- QuerySubmitter submitter = new QuerySubmitter();
- submitter.submitQuery(location, type, zkQuorum, local, bits);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index 2dd8611..6289509 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -55,7 +55,7 @@ import org.apache.drill.exec.server.BootStrapContext;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.junit.AfterClass;
@@ -304,7 +304,7 @@ public class TestOptiqPlans {
}
};
- StorageEngineRegistry reg = new StorageEngineRegistry(bitContext);
+ StoragePluginRegistry reg = new StoragePluginRegistry(bitContext);
PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),
CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), reg);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
index 493fea6..27fae08 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
@@ -43,7 +43,7 @@ import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.rpc.user.UserServer;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.vector.ValueVector;
import org.junit.AfterClass;
import org.junit.Test;
@@ -119,7 +119,7 @@ public class TestMergeJoin {
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
- PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext));
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext));
PhysicalPlan plan = reader.readPhysicalPlan(
Files.toString(
FileUtils.getResourceAsFile("/join/merge_single_batch.json"), Charsets.UTF_8)
@@ -172,7 +172,7 @@ public class TestMergeJoin {
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
- PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext));
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext));
PhysicalPlan plan = reader.readPhysicalPlan(
Files.toString(
FileUtils.getResourceAsFile("/join/merge_inner_single_batch.json"), Charsets.UTF_8)
@@ -225,7 +225,7 @@ public class TestMergeJoin {
bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
}};
- PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext));
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext));
PhysicalPlan plan = reader.readPhysicalPlan(
Files.toString(
FileUtils.getResourceAsFile("/join/merge_multi_batch.json"), Charsets.UTF_8)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/CachedSingleFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/CachedSingleFileSystem.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/CachedSingleFileSystem.java
new file mode 100644
index 0000000..46a2785
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/CachedSingleFileSystem.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
+import io.netty.buffer.UnpooledByteBufAllocator;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+
+public class CachedSingleFileSystem extends FileSystem{
+
+ private ByteBuf file;
+ private String path;
+
+ public CachedSingleFileSystem(String path) throws IOException{
+ this.path = path;
+ File f = new File(path);
+ long length = f.length();
+ if(length > Integer.MAX_VALUE) throw new UnsupportedOperationException("Cached file system only supports files of less than 2GB.");
+ System.out.println(length);
+ try(InputStream is = new BufferedInputStream(new FileInputStream(path))){
+ byte[] buffer = new byte[64*1024];
+ this.file = UnpooledByteBufAllocator.DEFAULT.directBuffer((int) length);
+ int read;
+ while( (read = is.read(buffer)) > 0){
+ file.writeBytes(buffer, 0, read);
+ }
+ }
+ }
+
+ public void close() throws IOException{
+ file.release();
+ super.close();
+ }
+
+ @Override
+ public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3, short arg4, long arg5,
+ Progressable arg6) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean delete(Path arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean delete(Path arg0, boolean arg1) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public URI getUri() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean mkdirs(Path path, FsPermission arg1) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int arg1) throws IOException {
+ if(!path.toString().equals(this.path)) throw new IOException(String.format("You requested file %s but this cached single file system only has the file %s.", path.toString(), this.path));
+ return new FSDataInputStream(new CachedFSDataInputStream(file.slice()));
+ }
+
+ @Override
+ public boolean rename(Path arg0, Path arg1) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setWorkingDirectory(Path arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+
+ private class CachedFSDataInputStream extends ByteBufInputStream implements Seekable, PositionedReadable{
+ private ByteBuf buf;
+ public CachedFSDataInputStream(ByteBuf buffer) {
+ super(buffer);
+ this.buf = buffer;
+
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return buf.readerIndex();
+ }
+
+ @Override
+ public void seek(long arg0) throws IOException {
+ buf.readerIndex((int) arg0);
+ }
+
+ @Override
+ public boolean seekToNewSource(long arg0) throws IOException {
+ return false;
+ }
+
+ @Override
+ public int read(long pos, byte[] buffer, int offset, int length) throws IOException {
+ ByteBuf local = buf.slice( (int) pos, (int) Math.min( buf.capacity() - pos, length));
+ local.readBytes(buffer, offset, buf.capacity());
+ return buf.capacity();
+ }
+
+ @Override
+ public void readFully(long pos, byte[] buffer) throws IOException {
+ readFully(pos, buffer, 0, buffer.length);
+ }
+
+ @Override
+ public void readFully(long pos, byte[] buffer, int offset, int length) throws IOException {
+ if(length + pos > buf.capacity()) throw new IOException("Read was too big.");
+ read(pos, buffer, offset, length);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
deleted file mode 100644
index 62b9dfa..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-
-import mockit.Expectations;
-import mockit.Injectable;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.TopLevelAllocator;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.store.json.JSONRecordReader;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-
-public class JSONRecordReaderTest {
- private static final Charset UTF_8 = Charset.forName("UTF-8");
-
- private String getResource(String resourceName) {
- return "resource:" + resourceName;
- }
-
- class MockOutputMutator implements OutputMutator {
- List<MaterializedField> removedFields = Lists.newArrayList();
- List<ValueVector> addFields = Lists.newArrayList();
-
- @Override
- public void removeField(MaterializedField field) throws SchemaChangeException {
- removedFields.add(field);
- }
-
- @Override
- public void addField(ValueVector vector) throws SchemaChangeException {
- addFields.add(vector);
- }
-
- @Override
- public void removeAllFields() {
- addFields.clear();
- }
-
- @Override
- public void setNewSchema() throws SchemaChangeException {
- }
-
- List<MaterializedField> getRemovedFields() {
- return removedFields;
- }
-
- List<ValueVector> getAddFields() {
- return addFields;
- }
- }
-
- private <T> void assertField(ValueVector valueVector, int index, MinorType expectedMinorType, T value, String name) {
- UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
- SchemaDefProtos.FieldDef def = metadata.getDef();
- assertEquals(expectedMinorType, def.getMajorType().getMinorType());
- String[] parts = name.split("\\.");
- int expected = parts.length;
- assertEquals(expected, def.getNameList().size());
- for(int i = 0; i < parts.length; ++i) {
- assertEquals(parts[i], def.getName(i).getName());
- }
-
- if (expectedMinorType == MinorType.MAP) {
- return;
- }
-
- T val = (T) valueVector.getAccessor().getObject(index);
- assertValue(value, val);
- }
-
- private void assertValue(Object expected, Object found) {
- if (found instanceof byte[]) {
- assertTrue(Arrays.equals((byte[]) expected, (byte[]) found));
- } else if(found instanceof ArrayList) {
- List expectedArray = (List) expected;
- List foundArray = (List) found;
- assertEquals(expectedArray.size(), foundArray.size());
- for(int i = 0; i < expectedArray.size(); ++i) {
- assertValue(expectedArray.get(i), foundArray.get(i));
- }
- } else {
- assertEquals(expected, found);
- }
- }
-
- @Test
- public void testSameSchemaInSameBatch(@Injectable final FragmentContext context) throws IOException,
- ExecutionSetupException {
- new Expectations() {
- {
- context.getAllocator();
- returns(new TopLevelAllocator());
- }
- };
- JSONRecordReader jr = new JSONRecordReader(context,
- FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null, null);
-
- MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
- jr.setup(mutator);
- assertEquals(2, jr.next());
- assertEquals(3, addFields.size());
- assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
- assertField(addFields.get(1), 0, MinorType.BIT, true, "b");
- assertField(addFields.get(2), 0, MinorType.VARCHAR, "hi!", "c");
- assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
- assertField(addFields.get(1), 1, MinorType.BIT, false, "b");
- assertField(addFields.get(2), 1, MinorType.VARCHAR, "drill!", "c");
-
- assertEquals(0, jr.next());
- assertTrue(mutator.getRemovedFields().isEmpty());
- }
-
- @Test
- public void testChangedSchemaInSameBatch(@Injectable final FragmentContext context) throws IOException,
- ExecutionSetupException {
- new Expectations() {
- {
- context.getAllocator();
- returns(new TopLevelAllocator());
- }
- };
-
- JSONRecordReader jr = new JSONRecordReader(context,
- FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null, null);
-
- MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
-
- jr.setup(mutator);
- assertEquals(3, jr.next());
- assertEquals(7, addFields.size());
- assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
- assertField(addFields.get(1), 0, MinorType.INT, 1, "b");
- assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c");
- assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
- assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1", "str1");
-
- assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
- assertField(addFields.get(1), 1, MinorType.INT, 3, "b");
- assertField(addFields.get(3), 1, MinorType.BIT, false, "bool");
- assertField(addFields.get(4), 1, MinorType.VARCHAR, "test2", "str1");
- assertField(addFields.get(5), 1, MinorType.INT, 4, "d");
-
- assertField(addFields.get(0), 2, MinorType.INT, 12345, "test");
- assertField(addFields.get(2), 2, MinorType.FLOAT4, (float) 5.16, "c");
- assertField(addFields.get(3), 2, MinorType.BIT, true, "bool");
- assertField(addFields.get(5), 2, MinorType.INT, 6, "d");
- assertField(addFields.get(6), 2, MinorType.VARCHAR, "test3", "str2");
- assertTrue(mutator.getRemovedFields().isEmpty());
- assertEquals(0, jr.next());
- }
-
- @Test
- public void testChangedSchemaInTwoBatchesColumnSelect(@Injectable final FragmentContext context) throws IOException,
- ExecutionSetupException {
- new Expectations() {
- {
- context.getAllocator();
- returns(new DirectBufferAllocator());
- }
- };
-
- JSONRecordReader jr = new JSONRecordReader(context,
- FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()),
- 64, null, Arrays.asList(new SchemaPath("test", ExpressionPosition.UNKNOWN))); // batch only fits 1 int
- MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
- List<MaterializedField> removedFields = mutator.getRemovedFields();
-
- jr.setup(mutator);
- assertEquals(1, jr.next());
- assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
- assertTrue(removedFields.isEmpty());
- assertEquals(addFields.size(), 1);
- assertEquals(1, jr.next());
- assertField(addFields.get(0), 0, MinorType.INT, 1234, "test");
- assertEquals(addFields.size(), 1);
- assertTrue(removedFields.isEmpty());
- removedFields.clear();
- assertEquals(1, jr.next());
- assertField(addFields.get(0), 0, MinorType.INT, 12345, "test");
- assertEquals(addFields.size(), 1);
- assertTrue(removedFields.isEmpty());
- assertEquals(0, jr.next());
- }
-
- @Test
- public void testChangedSchemaInTwoBatches(@Injectable final FragmentContext context) throws IOException,
- ExecutionSetupException {
- new Expectations() {
- {
- context.getAllocator();
- returns(new TopLevelAllocator());
- }
- };
-
- JSONRecordReader jr = new JSONRecordReader(context,
- FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()),
- 64, null, null); // batch only fits 1 int
- MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
- List<MaterializedField> removedFields = mutator.getRemovedFields();
-
- jr.setup(mutator);
- assertEquals(1, jr.next());
- assertEquals(5, addFields.size());
- assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
- assertField(addFields.get(1), 0, MinorType.INT, 1, "b");
- assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c");
- assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
- assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1", "str1");
- assertTrue(removedFields.isEmpty());
- assertEquals(1, jr.next());
- assertEquals(6, addFields.size());
- assertField(addFields.get(0), 0, MinorType.INT, 1234, "test");
- assertField(addFields.get(1), 0, MinorType.INT, 3, "b");
- assertField(addFields.get(3), 0, MinorType.BIT, false, "bool");
- assertField(addFields.get(4), 0, MinorType.VARCHAR, "test2", "str1");
- assertField(addFields.get(5), 0, MinorType.INT, 4, "d");
- assertEquals(1, removedFields.size());
- assertEquals("c", removedFields.get(0).getName());
- removedFields.clear();
- assertEquals(1, jr.next());
- assertEquals(7, addFields.size()); // The reappearing of field 'c' is also included
- assertField(addFields.get(0), 0, MinorType.INT, 12345, "test");
- assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
- assertField(addFields.get(5), 0, MinorType.INT, 6, "d");
- assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 5.16, "c");
- assertField(addFields.get(6), 0, MinorType.VARCHAR, "test3", "str2");
- assertEquals(2, removedFields.size());
- Iterables.find(removedFields, new Predicate<MaterializedField>() {
- @Override
- public boolean apply(MaterializedField materializedField) {
- return materializedField.getName().equals("str1");
- }
- });
- Iterables.find(removedFields, new Predicate<MaterializedField>() {
- @Override
- public boolean apply(MaterializedField materializedField) {
- return materializedField.getName().equals("b");
- }
- });
- assertEquals(0, jr.next());
- }
-
- @Test
- public void testNestedFieldInSameBatch(@Injectable final FragmentContext context) throws ExecutionSetupException, IOException {
- new Expectations() {
- {
- context.getAllocator();
- returns(new TopLevelAllocator());
- }
- };
-
- JSONRecordReader jr = new JSONRecordReader(context,
- FileUtils.getResourceAsFile("/scan_json_test_3.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null, null);
-
- MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
- jr.setup(mutator);
- assertEquals(2, jr.next());
- assertEquals(3, addFields.size());
- assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
- assertField(addFields.get(1), 0, MinorType.VARCHAR, "test", "a.b");
- assertField(addFields.get(2), 0, MinorType.BIT, true, "a.a.d");
- assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
- assertField(addFields.get(1), 1, MinorType.VARCHAR, "test2", "a.b");
- assertField(addFields.get(2), 1, MinorType.BIT, false, "a.a.d");
-
- assertEquals(0, jr.next());
- assertTrue(mutator.getRemovedFields().isEmpty());
- }
-
- @Test
- public void testRepeatedFields(@Injectable final FragmentContext context) throws ExecutionSetupException, IOException {
- new Expectations() {
- {
- context.getAllocator();
- returns(new TopLevelAllocator());
- }
- };
-
- JSONRecordReader jr = new JSONRecordReader(context,
- FileUtils.getResourceAsFile("/scan_json_test_4.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null, null);
-
- MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
- jr.setup(mutator);
- assertEquals(2, jr.next());
- assertEquals(7, addFields.size());
- assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
- assertField(addFields.get(1), 0, MinorType.INT, Arrays.asList(1, 2, 3), "test2");
- assertField(addFields.get(2), 0, MinorType.INT, Arrays.asList(4, 5, 6), "test3.a");
- assertField(addFields.get(3), 0, MinorType.INT, Arrays.asList(7, 8, 9), "test3.b");
- assertField(addFields.get(4), 0, MinorType.INT, Arrays.asList(10, 11, 12), "test3.c.d");
- assertField(addFields.get(5), 0, MinorType.FLOAT4, Arrays.<Float>asList((float) 1.1, (float) 1.2, (float) 1.3), "testFloat");
- assertField(addFields.get(6), 0, MinorType.VARCHAR, Arrays.asList("hello", "drill"), "testStr");
- assertField(addFields.get(1), 1, MinorType.INT, Arrays.asList(1, 2), "test2");
- assertField(addFields.get(2), 1, MinorType.INT, Arrays.asList(7, 7, 7, 8), "test3.a");
- assertField(addFields.get(5), 1, MinorType.FLOAT4, Arrays.<Float>asList((float) 2.2, (float) 2.3,(float) 2.4), "testFloat");
-
- assertEquals(0, jr.next());
- assertTrue(mutator.getRemovedFields().isEmpty());
- }
-
- @Test
- public void testRepeatedMissingFields(@Injectable final FragmentContext context) throws ExecutionSetupException, IOException {
- new Expectations() {
- {
- context.getAllocator();
- returns(new TopLevelAllocator());
- }
- };
-
- JSONRecordReader jr = new JSONRecordReader(context,
- FileUtils.getResourceAsFile("/scan_json_test_5.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null, null);
-
- MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
- jr.setup(mutator);
- assertEquals(9, jr.next());
- assertEquals(1, addFields.size());
- assertField(addFields.get(0), 0, MinorType.INT, Arrays.<Integer>asList(), "test");
- assertField(addFields.get(0), 1, MinorType.INT, Arrays.asList(1, 2, 3), "test");
- assertField(addFields.get(0), 2, MinorType.INT, Arrays.<Integer>asList(), "test");
- assertField(addFields.get(0), 3, MinorType.INT, Arrays.<Integer>asList(), "test");
- assertField(addFields.get(0), 4, MinorType.INT, Arrays.asList(4, 5, 6), "test");
- assertField(addFields.get(0), 5, MinorType.INT, Arrays.<Integer>asList(), "test");
- assertField(addFields.get(0), 6, MinorType.INT, Arrays.<Integer>asList(), "test");
- assertField(addFields.get(0), 7, MinorType.INT, Arrays.asList(7, 8, 9), "test");
- assertField(addFields.get(0), 8, MinorType.INT, Arrays.<Integer>asList(), "test");
-
-
- assertEquals(0, jr.next());
- assertTrue(mutator.getRemovedFields().isEmpty());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
deleted file mode 100644
index 6f568fb..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
+++ /dev/null
@@ -1,742 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static parquet.column.Encoding.PLAIN;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
-import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.store.json.JsonSchemaProvider;
-import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import parquet.bytes.BytesInput;
-import parquet.column.ColumnDescriptor;
-import parquet.hadoop.ParquetFileWriter;
-import parquet.hadoop.metadata.CompressionCodecName;
-import parquet.schema.MessageType;
-import parquet.schema.MessageTypeParser;
-
-import com.beust.jcommander.internal.Lists;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import com.google.common.util.concurrent.SettableFuture;
-
-public class ParquetRecordReaderTest {
- org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest.class);
-
- private static final boolean VERBOSE_DEBUG = false;
-
- // { 00000001, 00000010, 00000100, 00001000, 00010000, ... }
- byte[] bitFields = {1, 2, 4, 8, 16, 32, 64, -128};
- byte allBitsTrue = -1;
- byte allBitsFalse = 0;
- int DEFAULT_BYTES_PER_PAGE = 1024 * 1024 * 1;
- static Object[] intVals = {-200, 100, Integer.MAX_VALUE };
- static Object[] longVals = { -5000l, 5000l, Long.MAX_VALUE};
- static Object[] floatVals = { 1.74f, Float.MAX_VALUE, Float.MIN_VALUE};
- static Object[] doubleVals = {100.45d, Double.MAX_VALUE, Double.MIN_VALUE,};
- static Object[] boolVals = {false, false, true};
- static byte[] varLen1 = {50, 51, 52, 53, 54, 55, 56, 57, 58, 59};
- static byte[] varLen2 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
- static byte[] varLen3 = {100, 99, 98};
- static Object[] binVals = { varLen3, varLen2, varLen3};
- static Object[] bin2Vals = { varLen3, varLen2, varLen1};
-
- private void populateFieldInfoMap(ParquetTestProperties props){
- props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
- props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
- props.fields.put("f", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
- props.fields.put("d", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
- props.fields.put("b", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
- props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- }
-
- private void populatePigTPCHCustomerFields(ParquetTestProperties props){
- // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
- props.fields.put("C_CUSTKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
- props.fields.put("C_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
- props.fields.put("C_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
- props.fields.put("C_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
- props.fields.put("C_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
- props.fields.put("C_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("C_MKTSEGMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("C_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- }
-
- private void populatePigTPCHSupplierFields(ParquetTestProperties props){
- // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
- props.fields.put("S_SUPPKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
- props.fields.put("S_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
- props.fields.put("S_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
- props.fields.put("S_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
- props.fields.put("S_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
- props.fields.put("S_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("S_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- }
-
- @Test
- public void testMultipleRowGroups() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(3, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- populateFieldInfoMap(props);
- testParquetFullEngine(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
- }
-
- // TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was
- // generated using pig. Will need to find a good place to keep files like this.
- // For now I will upload it to the JIRA as an attachment.
- @Ignore
- @Test
- public void testNullableColumns() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
- Object[] boolVals = {true, null, null};
- props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
- testParquetFullEngine(false, "/parquet_nullable.json", "/tmp/nullable.parquet", 1, props);
- }
-
- @Ignore
- @Test
- public void testNullableColumnsVarLen() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
- byte[] val = {'b'};
-// Object[] boolVals = { val, null, null};
-// Object[] boolVals = { null, null, null};
- Object[] boolVals = { val, val, val};
- props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
- testParquetFullEngine(false, "/parquet_nullable_varlen.json", "/tmp/nullable.parquet", 1, props);
- }
-
- /**
- * Tests the attribute in a scan node to limit the columns read by a scan.
- *
- * The functionality of selecting all columns is tested in all of the other tests that leave out the attribute.
- * @throws Exception
- */
- @Test
- public void testSelectColumnRead() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- // generate metatdata for a series of test columns, these columns are all generated in the test file
- populateFieldInfoMap(props);
- generateParquetFile("/tmp/test.parquet", props);
- fields.clear();
- // create a new object to describe the dataset expected out of the scan operation
- // the fields added below match those requested in the plan specified in parquet_selective_column_read.json
- // that is used below in the test query
- props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
- props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
- props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
- props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
- testParquetFullEngineEventBased(false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
- }
-
-
- @Test
- public void testMultipleRowGroupsAndReads() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- populateFieldInfoMap(props);
- String readEntries = "";
- // number of times to read the file
- int i = 3;
- for (int j = 0; j < i; j++){
- readEntries += "{path: \"/tmp/test.parquet\"}";
- if (j < i - 1)
- readEntries += ",";
- }
- testParquetFullEngineEventBased(true, "/parquet_scan_screen_read_entry_replace.json", readEntries,
- "/tmp/test.parquet", i, props, true);
- }
-
- // requires binary file generated by pig from TPCH data, also have to disable assert where data is coming in
- @Ignore
- @Test
- public void testMultipleRowGroupsAndReadsPigError() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- populatePigTPCHCustomerFields(props);
-// populatePigTPCHSupplierFields(props);
- String readEntries = "";
- // number of times to read the file
- int i = 1;
- for (int j = 0; j < i; j++){
- readEntries += "{path: \"/tmp/tpc-h/customer\"}";
- if (j < i - 1)
- readEntries += ",";
- }
- testParquetFullEngineEventBased(false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
- "/tmp/test.parquet", i, props, true);
- }
-/*
- @Test
- public void testMultipleRowGroupsEvent() throws Exception {
- HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(10, 30000, DEFAULT_BYTES_PER_PAGE, fields);
- props.fields.put("a", new FieldInfo("a", "asdf", 1, new Object[3], TypeProtos.MinorType.BIGINT, props));
- testParquetFullEngineEventBased(false, "/parquet_scan_screen.json", "/tmp/out", 1, props);
- }
-
-*/
- private class ParquetTestProperties{
- int numberRowGroups;
- int recordsPerRowGroup;
- int bytesPerPage = 1024 * 1024 * 1;
- HashMap<String, FieldInfo> fields = new HashMap<>();
-
- public ParquetTestProperties(int numberRowGroups, int recordsPerRowGroup, int bytesPerPage,
- HashMap<String, FieldInfo> fields){
- this.numberRowGroups = numberRowGroups;
- this.recordsPerRowGroup = recordsPerRowGroup;
- this.bytesPerPage = bytesPerPage;
- this.fields = fields;
- }
-
- }
-
- private static class FieldInfo {
-
- String parquetType;
- String name;
- int bitLength;
- int numberOfPages;
- Object[] values;
- TypeProtos.MinorType type;
-
- FieldInfo(String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type, ParquetTestProperties props){
- this.parquetType = parquetType;
- this.name = name;
- this.bitLength = bitLength;
- this.numberOfPages = Math.max(1, (int) Math.ceil( ((long) props.recordsPerRowGroup) * bitLength / 8.0 / props.bytesPerPage));
- this.values = values;
- // generator is designed to use 3 values
- assert values.length == 3;
- this.type = type;
- }
- }
-
- private String getResource(String resourceName) {
- return "resource:" + resourceName;
- }
-
- public void generateParquetFile(String filename, ParquetTestProperties props) throws Exception {
-
- int currentBooleanByte = 0;
- WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7);
-
- Configuration configuration = new Configuration();
- configuration.set(JsonSchemaProvider.HADOOP_DEFAULT_NAME, "file:///");
- //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
-
- FileSystem fs = FileSystem.get(configuration);
- Path path = new Path(filename);
- if (fs.exists(path)) fs.delete(path, false);
-
-
- String messageSchema = "message m {";
- for (FieldInfo fieldInfo : props.fields.values()) {
- messageSchema += " required " + fieldInfo.parquetType + " " + fieldInfo.name + ";";
- }
- // remove the last semicolon, java really needs a join method for strings...
- // TODO - nvm apparently it requires a semicolon after every field decl, might want to file a bug
- //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
- messageSchema += "}";
-
- MessageType schema = MessageTypeParser.parseMessageType(messageSchema);
-
- CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
- ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
- w.start();
- HashMap<String, Integer> columnValuesWritten = new HashMap();
- int valsWritten;
- for (int k = 0; k < props.numberRowGroups; k++){
- w.startBlock(1);
- currentBooleanByte = 0;
- booleanBitCounter.reset();
-
- for (FieldInfo fieldInfo : props.fields.values()) {
-
- if ( ! columnValuesWritten.containsKey(fieldInfo.name)){
- columnValuesWritten.put((String) fieldInfo.name, 0);
- valsWritten = 0;
- } else {
- valsWritten = columnValuesWritten.get(fieldInfo.name);
- }
-
- String[] path1 = {(String) fieldInfo.name};
- ColumnDescriptor c1 = schema.getColumnDescription(path1);
-
- w.startColumn(c1, props.recordsPerRowGroup, codec);
- int valsPerPage = (int) Math.ceil(props.recordsPerRowGroup / (float) fieldInfo.numberOfPages);
- byte[] bytes;
- // for variable length binary fields
- int bytesNeededToEncodeLength = 4;
- if ((int) fieldInfo.bitLength > 0) {
- bytes = new byte[(int) Math.ceil(valsPerPage * (int) fieldInfo.bitLength / 8.0)];
- } else {
- // the twelve at the end is to account for storing a 4 byte length with each value
- int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength;
- // used for the case where there is a number of values in this row group that is not divisible by 3
- int leftOverBytes = 0;
- if ( valsPerPage % 3 > 0 ) leftOverBytes += ((byte[])fieldInfo.values[1]).length + bytesNeededToEncodeLength;
- if ( valsPerPage % 3 > 1 ) leftOverBytes += ((byte[])fieldInfo.values[2]).length + bytesNeededToEncodeLength;
- bytes = new byte[valsPerPage / 3 * totalValLength + leftOverBytes];
- }
- int bytesPerPage = (int) (valsPerPage * ((int) fieldInfo.bitLength / 8.0));
- int bytesWritten = 0;
- for (int z = 0; z < (int) fieldInfo.numberOfPages; z++, bytesWritten = 0) {
- for (int i = 0; i < valsPerPage; i++) {
- //System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : ""));
- if (fieldInfo.values[0] instanceof Boolean) {
-
- bytes[currentBooleanByte] |= bitFields[booleanBitCounter.val] & ((boolean) fieldInfo.values[valsWritten % 3]
- ? allBitsTrue : allBitsFalse);
- booleanBitCounter.increment();
- if (booleanBitCounter.val == 0) {
- currentBooleanByte++;
- }
- valsWritten++;
- if (currentBooleanByte > bytesPerPage) break;
- } else {
- if (fieldInfo.values[valsWritten % 3] instanceof byte[]){
- System.arraycopy(ByteArrayUtil.toByta(((byte[])fieldInfo.values[valsWritten % 3]).length),
- 0, bytes, bytesWritten, bytesNeededToEncodeLength);
- try{
- System.arraycopy(fieldInfo.values[valsWritten % 3],
- 0, bytes, bytesWritten + bytesNeededToEncodeLength, ((byte[])fieldInfo.values[valsWritten % 3]).length);
- }
- catch (Exception ex){
- Math.min(4, 5);
- }
- bytesWritten += ((byte[])fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength;
- }
- else{
- System.arraycopy( ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]),
- 0, bytes, i * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8);
- }
- valsWritten++;
- }
-
- }
- w.writeDataPage((int) (props.recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length, BytesInput.from(bytes), PLAIN, PLAIN, PLAIN);
- currentBooleanByte = 0;
- }
- w.endColumn();
- columnValuesWritten.remove((String) fieldInfo.name);
- columnValuesWritten.put((String) fieldInfo.name, valsWritten);
- }
-
- w.endBlock();
- }
- w.end(new HashMap<String, String>());
- logger.debug("Finished generating parquet file.");
- }
-
- private class ParquetResultListener implements UserResultsListener {
- private SettableFuture<Void> future = SettableFuture.create();
- int count = 0;
- RecordBatchLoader batchLoader;
-
- int batchCounter = 1;
- HashMap<String, Integer> valuesChecked = new HashMap();
- ParquetTestProperties props;
-
- ParquetResultListener(RecordBatchLoader batchLoader, ParquetTestProperties props){
- this.batchLoader = batchLoader;
- this.props = props;
- }
-
- @Override
- public void submissionFailed(RpcException ex) {
- logger.debug("Submission failed.", ex);
- future.setException(ex);
- }
-
- @Override
- public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
- logger.debug("result arrived in test batch listener.");
- if(result.getHeader().getIsLastChunk()){
- future.set(null);
- }
- int columnValCounter = 0;
- int i = 0;
- FieldInfo currentField;
- count += result.getHeader().getRowCount();
- boolean schemaChanged = false;
- try {
- schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData());
- } catch (SchemaChangeException e) {
- throw new RuntimeException(e);
- }
-
- int recordCount = 0;
- // print headers.
- if (schemaChanged) {
- } // do not believe any change is needed for when the schema changes, with the current mock scan use case
-
- for (VectorWrapper vw : batchLoader) {
- ValueVector vv = vw.getValueVector();
- currentField = props.fields.get(vv.getField().getName());
- if (VERBOSE_DEBUG){
- System.out.println("\n" + (String) currentField.name);
- }
- if ( ! valuesChecked.containsKey(vv.getField().getName())){
- valuesChecked.put(vv.getField().getName(), 0);
- columnValCounter = 0;
- } else {
- columnValCounter = valuesChecked.get(vv.getField().getName());
- }
- for (int j = 0; j < ((BaseDataValueVector)vv).getAccessor().getValueCount(); j++) {
- if (VERBOSE_DEBUG){
- System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
- }
- assertField(vv, j, (TypeProtos.MinorType) currentField.type,
- currentField.values[columnValCounter % 3], (String) currentField.name + "/");
- columnValCounter++;
- }
- if (VERBOSE_DEBUG){
- System.out.println("\n" + ((BaseDataValueVector)vv).getAccessor().getValueCount());
- }
- valuesChecked.remove(vv.getField().getName());
- valuesChecked.put(vv.getField().getName(), columnValCounter);
- }
-
- if (VERBOSE_DEBUG){
- for (i = 0; i < batchLoader.getRecordCount(); i++) {
- recordCount++;
- if (i % 50 == 0){
- System.out.println();
- for (VectorWrapper vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getField().getName(), 20) + " ");
-
- }
- System.out.println();
- System.out.println();
- }
-
- for (VectorWrapper vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getAccessor().getObject(i).toString(), 20) + " ");
- }
- System.out.println(
-
- );
- }
- }
- batchCounter++;
- if(result.getHeader().getIsLastChunk()){
- future.set(null);
- }
- }
-
- public void getResults() throws RpcException{
- try{
- future.get();
- }catch(Throwable t){
- throw RpcException.mapException(t);
- }
- }
-
- @Override
- public void queryIdArrived(QueryId queryId) {
- }
- }
-
-
-
- // specific tests should call this method, but it is not marked as a test itself intentionally
- public void testParquetFullEngineEventBased(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
- testParquetFullEngineEventBased(generateNew, plan, null, filename, numberOfTimesRead, props, true);
- }
-
- // specific tests should call this method, but it is not marked as a test itself intentionally
- public void testParquetFullEngineEventBased(boolean generateNew, String plan, String readEntries, String filename,
- int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props,
- boolean runAsLogicalPlan) throws Exception{
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-
- if (generateNew) generateParquetFile(filename, props);
-
- DrillConfig config = DrillConfig.create();
-
- try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
- bit1.run();
- client.connect();
- RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
- ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props);
- long C = System.nanoTime();
- String planText = Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8);
- // substitute in the string for the read entries, allows reuse of the plan file for several tests
- if (readEntries != null) {
- planText = planText.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries);
- }
- if (runAsLogicalPlan)
- client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener);
- else
- client.runQuery(UserProtos.QueryType.PHYSICAL, planText, resultListener);
- resultListener.getResults();
- for (String s : resultListener.valuesChecked.keySet()) {
- assertEquals("Record count incorrect for column: " + s,
- props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead, (long) resultListener.valuesChecked.get(s));
- logger.debug("Column {}, Values read:{}", s, resultListener.valuesChecked.get(s));
- }
- long D = System.nanoTime();
- System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
- }
- }
-
- // specific tests should call this method, but it is not marked as a test itself intentionally
- public void testParquetFullEngine(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
- testParquetFullEngine(generateNew, plan, null, filename, numberOfTimesRead, props);
- }
-
- // specific tests should call this method, but it is not marked as a test itself intentionally
- public void testParquetFullEngine(boolean generateNew, String plan, String readEntries, String filename,
- int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-
- if (generateNew) generateParquetFile(filename, props);
-
- DrillConfig config = DrillConfig.create();
-
- try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
- long A = System.nanoTime();
- bit1.run();
- long B = System.nanoTime();
- client.connect();
- long C = System.nanoTime();
- List<QueryResultBatch> results;
- // insert a variable number of reads
- if (readEntries != null){
- results = client.runQuery(UserProtos.QueryType.LOGICAL, (Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8).replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries)));
- }
- else{
- results = client.runQuery(UserProtos.QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8));
- }
-// List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
- long D = System.nanoTime();
- System.out.println(String.format("Took %f s to start drillbit", (float)(B-A) / 1E9));
- System.out.println(String.format("Took %f s to connect", (float)(C-B) / 1E9));
- System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
- //List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
- int count = 0;
-// RecordBatchLoader batchLoader = new RecordBatchLoader(new BootStrapContext(config).getAllocator());
- RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
- byte[] bytes;
-
- int batchCounter = 1;
- int columnValCounter = 0;
- int i = 0;
- FieldInfo currentField;
- HashMap<String, Integer> valuesChecked = new HashMap();
- for(QueryResultBatch b : results){
-
- count += b.getHeader().getRowCount();
- boolean schemaChanged = batchLoader.load(b.getHeader().getDef(), b.getData());
-
- int recordCount = 0;
- // print headers.
- if (schemaChanged) {
- } // do not believe any change is needed for when the schema changes, with the current mock scan use case
-
- for (VectorWrapper vw : batchLoader) {
- ValueVector vv = vw.getValueVector();
- currentField = props.fields.get(vv.getField().getName());
- if (VERBOSE_DEBUG){
- System.out.println("\n" + (String) currentField.name);
- }
- if ( ! valuesChecked.containsKey(vv.getField().getName())){
- valuesChecked.put(vv.getField().getName(), 0);
- columnValCounter = 0;
- } else {
- columnValCounter = valuesChecked.get(vv.getField().getName());
- }
- for (int j = 0; j < vv.getAccessor().getValueCount(); j++) {
- if (VERBOSE_DEBUG){
- System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
- }
- assertField(vv, j, currentField.type,
- currentField.values[columnValCounter % 3], currentField.name + "/");
- columnValCounter++;
- }
- if (VERBOSE_DEBUG){
- System.out.println("\n" + vv.getAccessor().getValueCount());
- }
- valuesChecked.remove(vv.getField().getName());
- valuesChecked.put(vv.getField().getName(), columnValCounter);
- }
-
- if (VERBOSE_DEBUG){
- for (i = 0; i < batchLoader.getRecordCount(); i++) {
- recordCount++;
- if (i % 50 == 0){
- System.out.println();
- for (VectorWrapper vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getField().getName(), 20) + " ");
-
- }
- System.out.println();
- System.out.println();
- }
-
- for (VectorWrapper vw : batchLoader) {
- ValueVector v = vw.getValueVector();
- System.out.print(pad(v.getAccessor().getObject(i) + "", 20) + " ");
- }
- System.out.println(
-
- );
- }
- }
- batchCounter++;
- }
- for (String s : valuesChecked.keySet()) {
- assertEquals("Record count incorrect for column: " + s, props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead, (long) valuesChecked.get(s));
- }
- assert valuesChecked.keySet().size() > 0;
- }
- }
-
- public String pad(String value, int length) {
- return pad(value, length, " ");
- }
-
- public String pad(String value, int length, String with) {
- StringBuilder result = new StringBuilder(length);
- result.append(value);
-
- while (result.length() < length) {
- result.insert(0, with);
- }
-
- return result.toString();
- }
-
- class MockOutputMutator implements OutputMutator {
- List<MaterializedField> removedFields = Lists.newArrayList();
- List<ValueVector> addFields = Lists.newArrayList();
-
- @Override
- public void removeField(MaterializedField field) throws SchemaChangeException {
- removedFields.add(field);
- }
-
- @Override
- public void addField(ValueVector vector) throws SchemaChangeException {
- addFields.add(vector);
- }
-
- @Override
- public void removeAllFields() {
- addFields.clear();
- }
-
- @Override
- public void setNewSchema() throws SchemaChangeException {
- }
-
- List<MaterializedField> getRemovedFields() {
- return removedFields;
- }
-
- List<ValueVector> getAddFields() {
- return addFields;
- }
- }
-
- private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) {
- assertField(valueVector, index, expectedMinorType, value, name, 0);
- }
-
- private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
-// UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
-// SchemaDefProtos.FieldDef def = metadata.getDef();
-// assertEquals(expectedMinorType, def.getMajorType().getMinorType());
-// assertEquals(name, def.getNameList().get(0).getName());
-// assertEquals(parentFieldId, def.getParentId());
-
- if (expectedMinorType == TypeProtos.MinorType.MAP) {
- return;
- }
-
- T val = (T) valueVector.getAccessor().getObject(index);
- if (val instanceof String){
- assertEquals(value, val);
- }
- else if (val instanceof byte[]) {
- assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
- } else {
- assertEquals(value, val);
- }
- }
-
- private class WrapAroundCounter {
-
- int maxVal;
- int val;
-
- public WrapAroundCounter(int maxVal) {
- this.maxVal = maxVal;
- }
-
- public int increment() {
- val++;
- if (val > maxVal) {
- val = 0;
- }
- return val;
- }
-
- public void reset() {
- val = 0;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
index 574e847..67b11cb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
@@ -19,15 +19,18 @@ package org.apache.drill.exec.store;
import com.google.common.collect.ImmutableRangeMap;
import com.google.common.collect.Range;
+
import junit.framework.Assert;
import mockit.Injectable;
import mockit.NonStrictExpectations;
+
import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.parquet.ParquetGroupScan;
import org.apache.drill.exec.store.parquet.ParquetRowGroupScan;
-import org.apache.drill.exec.store.parquet.ParquetStorageEngine;
+import org.apache.drill.exec.store.parquet.ParquetFormatPlugin;
+import org.apache.drill.exec.store.schedule.BlockMapBuilder;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -100,60 +103,60 @@ public class TestAffinityCalculator {
return endPoints;
}
- @Test
- public void testSetEndpointBytes(@Injectable final FileSystem fs, @Injectable final FileStatus file) throws Throwable{
- final long blockSize = 256*1024*1024;
- LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups = new LinkedList<>();
- int numberOfHosts = 4;
- int numberOfBlocks = 3;
- String port = "1234";
- String[] hosts = new String[numberOfHosts];
-
- final BlockLocation[] blockLocations = buildBlockLocations(hosts, blockSize);
- final LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = buildEndpoints(numberOfHosts);
- buildRowGroups(rowGroups, numberOfBlocks, blockSize, 3);
-
- new NonStrictExpectations() {{
- fs.getFileBlockLocations(file, 0, 3*blockSize); result = blockLocations;
- fs.getFileStatus(new Path(path)); result = file;
- file.getLen(); result = 3*blockSize;
- }};
-
-
- AffinityCalculator ac = new AffinityCalculator(fs, endPoints);
- for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
- ac.setEndpointBytes(rowGroup);
- }
- ParquetGroupScan.RowGroupInfo rg = rowGroups.get(0);
- Long b = rg.getEndpointBytes().get(endPoints.get(0));
- assertEquals(blockSize,b.longValue());
- b = rg.getEndpointBytes().get(endPoints.get(3));
- assertNull(b);
-
- buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2);
-
- ac = new AffinityCalculator(fs, endPoints);
- for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
- ac.setEndpointBytes(rowGroup);
- }
- rg = rowGroups.get(0);
- b = rg.getEndpointBytes().get(endPoints.get(0));
- assertEquals(blockSize*3/2,b.longValue());
- b = rg.getEndpointBytes().get(endPoints.get(3));
- assertEquals(blockSize / 2, b.longValue());
-
- buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6);
-
- ac = new AffinityCalculator(fs, endPoints);
- for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
- ac.setEndpointBytes(rowGroup);
- }
- rg = rowGroups.get(0);
- b = rg.getEndpointBytes().get(endPoints.get(0));
- assertEquals(blockSize/2,b.longValue());
- b = rg.getEndpointBytes().get(endPoints.get(3));
- assertNull(b);
- }
+// @Test
+// public void testSetEndpointBytes(@Injectable final FileSystem fs, @Injectable final FileStatus file) throws Throwable{
+// final long blockSize = 256*1024*1024;
+// LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups = new LinkedList<>();
+// int numberOfHosts = 4;
+// int numberOfBlocks = 3;
+// String port = "1234";
+// String[] hosts = new String[numberOfHosts];
+//
+// final BlockLocation[] blockLocations = buildBlockLocations(hosts, blockSize);
+// final LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = buildEndpoints(numberOfHosts);
+// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 3);
+//
+// new NonStrictExpectations() {{
+// fs.getFileBlockLocations(file, 0, 3*blockSize); result = blockLocations;
+// fs.getFileStatus(new Path(path)); result = file;
+// file.getLen(); result = 3*blockSize;
+// }};
+//
+//
+// BlockMapBuilder ac = new BlockMapBuilder(fs, endPoints);
+// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
+// ac.setEndpointBytes(rowGroup);
+// }
+// ParquetGroupScan.RowGroupInfo rg = rowGroups.get(0);
+// Long b = rg.getEndpointBytes().get(endPoints.get(0));
+// assertEquals(blockSize,b.longValue());
+// b = rg.getEndpointBytes().get(endPoints.get(3));
+// assertNull(b);
+//
+// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2);
+//
+// ac = new BlockMapBuilder(fs, endPoints);
+// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
+// ac.setEndpointBytes(rowGroup);
+// }
+// rg = rowGroups.get(0);
+// b = rg.getEndpointBytes().get(endPoints.get(0));
+// assertEquals(blockSize*3/2,b.longValue());
+// b = rg.getEndpointBytes().get(endPoints.get(3));
+// assertEquals(blockSize / 2, b.longValue());
+//
+// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6);
+//
+// ac = new BlockMapBuilder(fs, endPoints);
+// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
+// ac.setEndpointBytes(rowGroup);
+// }
+// rg = rowGroups.get(0);
+// b = rg.getEndpointBytes().get(endPoints.get(0));
+// assertEquals(blockSize/2,b.longValue());
+// b = rg.getEndpointBytes().get(endPoints.get(3));
+// assertNull(b);
+// }
@Test
public void testBuildRangeMap() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
new file mode 100644
index 0000000..51a0b0b
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.ValueVector;
+
+import com.google.common.collect.Maps;
+
+public class TestOutputMutator implements OutputMutator, Iterable<VectorWrapper<?>> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestOutputMutator.class);
+
+ private final VectorContainer container = new VectorContainer();
+ private final Map<MaterializedField, ValueVector> fieldVectorMap = Maps.newHashMap();
+
+ public void removeField(MaterializedField field) throws SchemaChangeException {
+ ValueVector vector = fieldVectorMap.remove(field);
+ if (vector == null)
+ throw new SchemaChangeException("Failure attempting to remove an unknown field.");
+ container.remove(vector);
+ vector.close();
+ }
+
+ public void addField(ValueVector vector) {
+ container.add(vector);
+ fieldVectorMap.put(vector.getField(), vector);
+ }
+
+ @Override
+ public void removeAllFields() {
+ for (VectorWrapper<?> vw : container) {
+ vw.clear();
+ }
+ container.clear();
+ fieldVectorMap.clear();
+ }
+
+ @Override
+ public void setNewSchema() throws SchemaChangeException {
+ container.buildSchema(SelectionVectorMode.NONE);
+ }
+
+ public Iterator<VectorWrapper<?>> iterator() {
+ return container.iterator();
+ }
+
+ public void clear(){
+ removeAllFields();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
deleted file mode 100644
index 9c4aeea..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store;
-
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
-import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.vector.ValueVector;
-import org.junit.AfterClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Stopwatch;
-import com.google.common.io.Resources;
-
-public class TestParquetPhysicalPlan {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetPhysicalPlan.class);
-
- public String fileName = "parquet/parquet_scan_filter_union_screen_physical.json";
-
- @Test
- @Ignore
- public void testParseParquetPhysicalPlan() throws Exception {
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
- DrillConfig config = DrillConfig.create();
-
- try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
- bit1.run();
- client.connect();
- List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
- RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
- int count = 0;
- for (QueryResultBatch b : results) {
- System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
- count += b.getHeader().getRowCount();
- loader.load(b.getHeader().getDef(), b.getData());
- for (VectorWrapper vw : loader) {
- System.out.print(vw.getValueVector().getField().getName() + ": ");
- ValueVector vv = vw.getValueVector();
- for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
- Object o = vv.getAccessor().getObject(i);
- if (o instanceof byte[]) {
- System.out.print(" [" + new String((byte[]) o) + "]");
- } else {
- System.out.print(" [" + vv.getAccessor().getObject(i) + "]");
- }
-// break;
- }
- System.out.println();
- }
- }
- client.close();
- System.out.println(String.format("Got %d total results", count));
- }
- }
-
- private class ParquetResultsListener implements UserResultsListener {
- AtomicInteger count = new AtomicInteger();
- private CountDownLatch latch = new CountDownLatch(1);
- @Override
- public void submissionFailed(RpcException ex) {
- logger.error("submission failed", ex);
- latch.countDown();
- }
-
- @Override
- public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
- int rows = result.getHeader().getRowCount();
- System.out.println(String.format("Result batch arrived. Number of records: %d", rows));
- count.addAndGet(rows);
- if (result.getHeader().getIsLastChunk()) latch.countDown();
- result.release();
- }
-
- public int await() throws Exception {
- latch.await();
- return count.get();
- }
-
- @Override
- public void queryIdArrived(QueryId queryId) {
- }
- }
- @Test
- @Ignore
- public void testParseParquetPhysicalPlanRemote() throws Exception {
- DrillConfig config = DrillConfig.create();
-
- try(DrillClient client = new DrillClient(config);){
- client.connect();
- ParquetResultsListener listener = new ParquetResultsListener();
- Stopwatch watch = new Stopwatch();
- watch.start();
- client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8), listener);
- System.out.println(String.format("Got %d total records in %d seconds", listener.await(), watch.elapsed(TimeUnit.SECONDS)));
- client.close();
- }
- }
-
- @AfterClass
- public static void tearDown() throws Exception{
- // pause to get logger to catch up.
- Thread.sleep(1000);
- }
-
-}
[12/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index e4f8e7b..329815d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -17,43 +17,58 @@
*/
package org.apache.drill.exec.work.foreman;
+import io.netty.buffer.ByteBuf;
+
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.FragmentSetupException;
import org.apache.drill.exec.exception.OptimizerException;
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.opt.BasicOptimizer;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.impl.SendingAccountor;
import org.apache.drill.exec.physical.impl.materialize.QueryWritableBatch;
import org.apache.drill.exec.planner.fragment.Fragment;
import org.apache.drill.exec.planner.fragment.MakeFragmentsVisitor;
import org.apache.drill.exec.planner.fragment.PlanningSet;
import org.apache.drill.exec.planner.fragment.SimpleParallelizer;
import org.apache.drill.exec.planner.fragment.StatsCollector;
+import org.apache.drill.exec.planner.sql.DrillSqlWorker;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
import org.apache.drill.exec.proto.UserProtos.QueryResult;
import org.apache.drill.exec.proto.UserProtos.QueryResult.QueryState;
import org.apache.drill.exec.proto.UserProtos.RequestResults;
import org.apache.drill.exec.proto.UserProtos.RunQuery;
+import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.rpc.BaseRpcOutcomeListener;
import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.RpcOutcomeListener;
import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.util.AtomicState;
+import org.apache.drill.exec.vector.VarCharVector;
import org.apache.drill.exec.work.ErrorHelper;
import org.apache.drill.exec.work.QueryWorkUnit;
import org.apache.drill.exec.work.WorkManager.WorkerBee;
+import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
/**
@@ -151,6 +166,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
case LOGICAL:
parseAndRunLogicalPlan(queryRequest.getPlan());
+
break;
case PHYSICAL:
parseAndRunPhysicalPlan(queryRequest.getPlan());
@@ -170,8 +186,18 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
try {
LogicalPlan logicalPlan = context.getPlanReader().readLogicalPlan(json);
+
+ if(logicalPlan.getProperties().resultMode == ResultMode.LOGICAL){
+ fail("Failure running plan. You requested a result mode of LOGICAL and submitted a logical plan. In this case you're output mode must be PHYSICAL or EXEC.", new Exception());
+ }
if(logger.isDebugEnabled()) logger.debug("Logical {}", logicalPlan.unparse(context.getConfig()));
PhysicalPlan physicalPlan = convert(logicalPlan);
+
+ if(logicalPlan.getProperties().resultMode == ResultMode.PHYSICAL){
+ returnPhysical(physicalPlan);
+ return;
+ }
+
if(logger.isDebugEnabled()) logger.debug("Physical {}", context.getConfig().getMapper().writeValueAsString(physicalPlan));
runPhysicalPlan(physicalPlan);
} catch (IOException e) {
@@ -181,6 +207,74 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
}
}
+
+ private void returnLogical(LogicalPlan plan){
+ String jsonPlan = plan.toJsonStringSafe(context.getConfig());
+ sendSingleString("logical", jsonPlan);
+ }
+
+ private void returnPhysical(PhysicalPlan plan){
+ String jsonPlan = plan.unparse(context.getConfig().getMapper().writer());
+ sendSingleString("physical", jsonPlan);
+ }
+
+ private void sendSingleString(String columnName, String value){
+ MaterializedField f = MaterializedField.create(new SchemaPath(columnName, ExpressionPosition.UNKNOWN), Types.required(MinorType.VARCHAR));
+ VarCharVector vector = new VarCharVector(f, bee.getContext().getAllocator());
+ byte[] bytes = value.getBytes(Charsets.UTF_8);
+ vector.allocateNew(bytes.length, 1);
+ vector.getMutator().set(0, bytes);
+ vector.getMutator().setValueCount(1);
+ QueryResult header = QueryResult.newBuilder() //
+ .setQueryId(context.getQueryId()) //
+ .setRowCount(1) //
+ .setDef(RecordBatchDef.newBuilder().addField(vector.getMetadata()).build()) //
+ .setIsLastChunk(false) //
+ .build();
+ QueryWritableBatch b1 = new QueryWritableBatch(header, vector.getBuffers());
+ vector.close();
+
+ QueryResult header2 = QueryResult.newBuilder() //
+ .setQueryId(context.getQueryId()) //
+ .setRowCount(0) //
+ .setDef(RecordBatchDef.getDefaultInstance()) //
+ .setIsLastChunk(true) //
+ .build();
+ QueryWritableBatch b2 = new QueryWritableBatch(header2);
+
+ SingleListener l = new SingleListener();
+ this.initiatingClient.sendResult(l, b1);
+ this.initiatingClient.sendResult(l, b2);
+ l.acct.waitForSendComplete();
+
+ }
+
+
+ class SingleListener implements RpcOutcomeListener<Ack>{
+
+ final SendingAccountor acct;
+
+ public SingleListener(){
+ acct = new SendingAccountor();
+ acct.increment();
+ acct.increment();
+ }
+
+ @Override
+ public void failed(RpcException ex) {
+ acct.decrement();
+ fail("Failure while sending single result.", ex);
+ }
+
+ @Override
+ public void success(Ack value, ByteBuf buffer) {
+ acct.decrement();
+ }
+
+ }
+
+
+
private void parseAndRunPhysicalPlan(String json) {
try {
PhysicalPlan plan = context.getPlanReader().readPhysicalPlan(json);
@@ -192,7 +286,11 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
private void runPhysicalPlan(PhysicalPlan plan) {
+ if(plan.getProperties().resultMode != ResultMode.EXEC){
+ fail(String.format("Failure running plan. You requested a result mode of %s and a physical plan can only be output as EXEC", plan.getProperties().resultMode), new Exception());
+ }
PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
+
MakeFragmentsVisitor makeFragmentsVisitor = new MakeFragmentsVisitor();
Fragment rootFragment;
try {
@@ -218,18 +316,26 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
List<PlanFragment> intermediateFragments = Lists.newArrayList();
// store fragments in distributed grid.
+ logger.debug("Storing fragments");
for (PlanFragment f : work.getFragments()) {
// store all fragments in grid since they are part of handshake.
+
context.getCache().storeFragment(f);
if (f.getLeafFragment()) {
leafFragments.add(f);
} else {
intermediateFragments.add(f);
}
+
+
}
+ logger.debug("Fragments stored.");
+
+ logger.debug("Submitting fragments to run.");
fragmentManager.runFragments(bee, work.getRootFragment(), work.getRootOperator(), initiatingClient, leafFragments, intermediateFragments);
+ logger.debug("Fragments running.");
} catch (ExecutionSetupException | RpcException e) {
@@ -238,11 +344,32 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
}
- private void runSQL(String json) {
- throw new UnsupportedOperationException();
+ private void runSQL(String sql) {
+ try{
+ DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getSchemaFactory(), context.getFunctionRegistry());
+ LogicalPlan plan = sqlWorker.getPlan(sql);
+
+
+ if(plan.getProperties().resultMode == ResultMode.LOGICAL){
+ returnLogical(plan);
+ return;
+ }
+
+ PhysicalPlan physical = convert(plan);
+
+ if(plan.getProperties().resultMode == ResultMode.PHYSICAL){
+ returnPhysical(physical);
+ return;
+ }
+
+ runPhysicalPlan(physical);
+ }catch(Exception e){
+ fail("Failure while parsing sql.", e);
+ }
}
private PhysicalPlan convert(LogicalPlan plan) throws OptimizerException {
+ if(logger.isDebugEnabled()) logger.debug("Converting logical plan {}.", plan.toJsonStringSafe(context.getConfig()));
return new BasicOptimizer(DrillConfig.create(), context).optimize(new BasicOptimizer.BasicOptimizationContext(), plan);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java
index e9302e1..eaf921d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java
@@ -76,18 +76,25 @@ class QueryManager implements FragmentStatusListener{
}
public void runFragments(WorkerBee bee, PlanFragment rootFragment, FragmentRoot rootOperator, UserClientConnection rootClient, List<PlanFragment> leafFragments, List<PlanFragment> intermediateFragments) throws ExecutionSetupException{
+ logger.debug("Setting up fragment runs.");
remainingFragmentCount.set(leafFragments.size()+1);
queryId = rootFragment.getHandle().getQueryId();
workBus = bee.getContext().getWorkBus();
// set up the root fragment first so we'll have incoming buffers available.
{
- FragmentContext rootContext = new FragmentContext(bee.getContext(), rootFragment, rootClient, new FunctionImplementationRegistry(bee.getContext().getConfig()));
+ logger.debug("Setting up root context.");
+ FragmentContext rootContext = new FragmentContext(bee.getContext(), rootFragment, rootClient, bee.getContext().getFunctionImplementationRegistry());
+ logger.debug("Setting up incoming buffers");
IncomingBuffers buffers = new IncomingBuffers(rootOperator, rootContext);
+ logger.debug("Setting buffers on root context.");
rootContext.setBuffers(buffers);
+ logger.debug("Generating Exec tree");
RootExec rootExec = ImplCreator.getExec(rootContext, rootOperator);
+ logger.debug("Exec tree generated.");
// add fragment to local node.
map.put(rootFragment.getHandle(), new FragmentData(rootFragment.getHandle(), null, true));
+ logger.debug("Fragment added to local node.");
rootRunner = new FragmentExecutor(rootContext, rootExec, new RootStatusHandler(rootContext, rootFragment));
RootFragmentManager fragmentManager = new RootFragmentManager(rootFragment.getHandle(), buffers, rootRunner);
@@ -99,6 +106,7 @@ class QueryManager implements FragmentStatusListener{
workBus.setRootFragmentManager(fragmentManager);
}
+
}
// keep track of intermediate fragments (not root or leaf)
@@ -112,6 +120,7 @@ class QueryManager implements FragmentStatusListener{
sendRemoteFragment(f);
}
+ logger.debug("Fragment runs setup is complete.");
}
private void sendRemoteFragment(PlanFragment fragment){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/storage-engines.json b/exec/java-exec/src/main/resources/storage-engines.json
new file mode 100644
index 0000000..d1d0413
--- /dev/null
+++ b/exec/java-exec/src/main/resources/storage-engines.json
@@ -0,0 +1,29 @@
+{
+ "storage":{
+ "parquet-local" :
+ {
+ "type":"parquet",
+ "dfsName" : "file:///"
+ },
+ "parquet-cp" :
+ {
+ "type":"parquet",
+ "dfsName" : "classpath:///"
+ },
+ "jsonl" :
+ {
+ "type":"json",
+ "dfsName" : "file:///"
+ },
+ "json-cp" :
+ {
+ "type":"json",
+ "dfsName" : "classpath:///"
+ },
+ "parquet" :
+ {
+ "type":"parquet",
+ "dfsName" : "file:///"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index b4dc943..454b15f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -171,12 +171,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
}
ValueVector.Accessor accessor = v.getValueVector().getAccessor();
-
- if (v.getField().getType().getMinorType() == TypeProtos.MinorType.VARCHAR) {
- System.out.println(new String((byte[]) accessor.getObject(r), UTF_8));
- } else {
- System.out.print(accessor.getObject(r));
- }
+ System.out.print(accessor.getObject(r));
}
if (!first) System.out.println();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
index bffc427..2be3e8d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
@@ -145,10 +145,10 @@ public class JSONRecordReaderTest {
assertEquals(3, addFields.size());
assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
assertField(addFields.get(1), 0, MinorType.BIT, true, "b");
- assertField(addFields.get(2), 0, MinorType.VARCHAR, "hi!".getBytes(UTF_8), "c");
+ assertField(addFields.get(2), 0, MinorType.VARCHAR, "hi!", "c");
assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
assertField(addFields.get(1), 1, MinorType.BIT, false, "b");
- assertField(addFields.get(2), 1, MinorType.VARCHAR, "drill!".getBytes(UTF_8), "c");
+ assertField(addFields.get(2), 1, MinorType.VARCHAR, "drill!", "c");
assertEquals(0, jr.next());
assertTrue(mutator.getRemovedFields().isEmpty());
@@ -178,19 +178,19 @@ public class JSONRecordReaderTest {
assertField(addFields.get(1), 0, MinorType.INT, 1, "b");
assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c");
assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
- assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1".getBytes(UTF_8), "str1");
+ assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1", "str1");
assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
assertField(addFields.get(1), 1, MinorType.INT, 3, "b");
assertField(addFields.get(3), 1, MinorType.BIT, false, "bool");
- assertField(addFields.get(4), 1, MinorType.VARCHAR, "test2".getBytes(UTF_8), "str1");
+ assertField(addFields.get(4), 1, MinorType.VARCHAR, "test2", "str1");
assertField(addFields.get(5), 1, MinorType.INT, 4, "d");
assertField(addFields.get(0), 2, MinorType.INT, 12345, "test");
assertField(addFields.get(2), 2, MinorType.FLOAT4, (float) 5.16, "c");
assertField(addFields.get(3), 2, MinorType.BIT, true, "bool");
assertField(addFields.get(5), 2, MinorType.INT, 6, "d");
- assertField(addFields.get(6), 2, MinorType.VARCHAR, "test3".getBytes(UTF_8), "str2");
+ assertField(addFields.get(6), 2, MinorType.VARCHAR, "test3", "str2");
assertTrue(mutator.getRemovedFields().isEmpty());
assertEquals(0, jr.next());
}
@@ -220,14 +220,14 @@ public class JSONRecordReaderTest {
assertField(addFields.get(1), 0, MinorType.INT, 1, "b");
assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c");
assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
- assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1".getBytes(UTF_8), "str1");
+ assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1", "str1");
assertTrue(removedFields.isEmpty());
assertEquals(1, jr.next());
assertEquals(6, addFields.size());
assertField(addFields.get(0), 0, MinorType.INT, 1234, "test");
assertField(addFields.get(1), 0, MinorType.INT, 3, "b");
assertField(addFields.get(3), 0, MinorType.BIT, false, "bool");
- assertField(addFields.get(4), 0, MinorType.VARCHAR, "test2".getBytes(UTF_8), "str1");
+ assertField(addFields.get(4), 0, MinorType.VARCHAR, "test2", "str1");
assertField(addFields.get(5), 0, MinorType.INT, 4, "d");
assertEquals(1, removedFields.size());
assertEquals("c", removedFields.get(0).getName());
@@ -238,7 +238,7 @@ public class JSONRecordReaderTest {
assertField(addFields.get(3), 0, MinorType.BIT, true, "bool");
assertField(addFields.get(5), 0, MinorType.INT, 6, "d");
assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 5.16, "c");
- assertField(addFields.get(6), 0, MinorType.VARCHAR, "test3".getBytes(UTF_8), "str2");
+ assertField(addFields.get(6), 0, MinorType.VARCHAR, "test3", "str2");
assertEquals(2, removedFields.size());
Iterables.find(removedFields, new Predicate<MaterializedField>() {
@Override
@@ -274,10 +274,10 @@ public class JSONRecordReaderTest {
assertEquals(2, jr.next());
assertEquals(3, addFields.size());
assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
- assertField(addFields.get(1), 0, MinorType.VARCHAR, "test".getBytes(UTF_8), "a.b");
+ assertField(addFields.get(1), 0, MinorType.VARCHAR, "test", "a.b");
assertField(addFields.get(2), 0, MinorType.BIT, true, "a.a.d");
assertField(addFields.get(0), 1, MinorType.INT, 1234, "test");
- assertField(addFields.get(1), 1, MinorType.VARCHAR, "test2".getBytes(UTF_8), "a.b");
+ assertField(addFields.get(1), 1, MinorType.VARCHAR, "test2", "a.b");
assertField(addFields.get(2), 1, MinorType.BIT, false, "a.a.d");
assertEquals(0, jr.next());
@@ -308,7 +308,7 @@ public class JSONRecordReaderTest {
assertField(addFields.get(3), 0, MinorType.INT, Arrays.asList(7, 8, 9), "test3.b");
assertField(addFields.get(4), 0, MinorType.INT, Arrays.asList(10, 11, 12), "test3.c.d");
assertField(addFields.get(5), 0, MinorType.FLOAT4, Arrays.<Float>asList((float) 1.1, (float) 1.2, (float) 1.3), "testFloat");
- assertField(addFields.get(6), 0, MinorType.VARCHAR, Arrays.asList("hello".getBytes(UTF_8), "drill".getBytes(UTF_8)), "testStr");
+ assertField(addFields.get(6), 0, MinorType.VARCHAR, Arrays.asList("hello", "drill"), "testStr");
assertField(addFields.get(1), 1, MinorType.INT, Arrays.asList(1, 2), "test2");
assertField(addFields.get(2), 1, MinorType.INT, Arrays.asList(7, 7, 7, 8), "test3.a");
assertField(addFields.get(5), 1, MinorType.FLOAT4, Arrays.<Float>asList((float) 2.2, (float) 2.3,(float) 2.4), "testFloat");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
index 733cb1d..8efc762 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
@@ -17,10 +17,13 @@
*/
package org.apache.drill.exec.store;
-import com.beust.jcommander.internal.Lists;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import com.google.common.util.concurrent.SettableFuture;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static parquet.column.Encoding.PLAIN;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.types.TypeProtos;
@@ -34,6 +37,7 @@ import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.server.Drillbit;
@@ -54,10 +58,10 @@ import parquet.hadoop.metadata.CompressionCodecName;
import parquet.schema.MessageType;
import parquet.schema.MessageTypeParser;
-import java.util.*;
-
-import static org.junit.Assert.*;
-import static parquet.column.Encoding.PLAIN;
+import com.beust.jcommander.internal.Lists;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.google.common.util.concurrent.SettableFuture;
public class ParquetRecordReaderTest {
org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest.class);
@@ -368,7 +372,7 @@ public class ParquetRecordReaderTest {
}
@Override
- public void resultArrived(QueryResultBatch result) {
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
logger.debug("result arrived in test batch listener.");
if(result.getHeader().getIsLastChunk()){
future.set(null);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
index b765ed0..9c4aeea 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
@@ -17,9 +17,10 @@
*/
package org.apache.drill.exec.store;
-import com.google.common.base.Charsets;
-import com.google.common.base.Stopwatch;
-import com.google.common.io.Resources;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.client.DrillClient;
@@ -28,6 +29,7 @@ import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.server.Drillbit;
@@ -37,10 +39,9 @@ import org.junit.AfterClass;
import org.junit.Ignore;
import org.junit.Test;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
+import com.google.common.io.Resources;
public class TestParquetPhysicalPlan {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetPhysicalPlan.class);
@@ -93,7 +94,7 @@ public class TestParquetPhysicalPlan {
}
@Override
- public void resultArrived(QueryResultBatch result) {
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
int rows = result.getHeader().getRowCount();
System.out.println(String.format("Result batch arrived. Number of records: %d", rows));
count.addAndGet(rows);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index ab29a9f..52430e1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -41,6 +41,7 @@ import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.server.Drillbit;
@@ -139,7 +140,7 @@ public class ParquetRecordReaderTest {
}
@Override
- public void resultArrived(QueryResultBatch result) {
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
long columnValCounter = 0;
int i = 0;
FieldInfo currentField;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/pom.xml b/exec/pom.xml
index a463015..399f788 100644
--- a/exec/pom.xml
+++ b/exec/pom.xml
@@ -33,6 +33,5 @@
<modules>
<module>bufferl</module>
<module>java-exec</module>
- <module>ref</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/pom.xml
----------------------------------------------------------------------
diff --git a/exec/ref/pom.xml b/exec/ref/pom.xml
deleted file mode 100644
index b1b31dd..0000000
--- a/exec/ref/pom.xml
+++ /dev/null
@@ -1,107 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>exec-parent</artifactId>
- <groupId>org.apache.drill.exec</groupId>
- <version>1.0.0-m2-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>drill-ref</artifactId>
-
- <name>exec/Reference Interpreter</name>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.drill</groupId>
- <artifactId>drill-common</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.carrotsearch</groupId>
- <artifactId>hppc</artifactId>
- <version>0.4.2</version>
- </dependency>
-
- </dependencies>
-
- <profiles>
- <profile>
- <id>default-hadoop</id>
- <activation>
- <property>
- <name>!alt-hadoop</name>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <id>mapr</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <id>cdh4</id>
- <dependencies>
- <dependency>
- <artifactId>hadoop-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <id>hdp</id>
- <dependencies>
- <dependency>
- <artifactId>hadoop-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
-
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/BasicStatusHandle.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/BasicStatusHandle.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/BasicStatusHandle.java
deleted file mode 100644
index 06b79fa..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/BasicStatusHandle.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.drill.exec.ref.rops.SinkROP.StatusHandle;
-
-public class BasicStatusHandle implements StatusHandle{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicStatusHandle.class);
-
- private AtomicBoolean okToContinue = new AtomicBoolean(true);
-
- @Override
- public void progress(long bytes, long records) {
- logger.debug("Progressed {} bytes, {} records.", bytes, records);
- }
-
- @Override
- public boolean okToContinue() {
- return okToContinue.get();
- }
-
- public void markCanceled(){
- okToContinue.set(false);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/ExecRefConstants.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/ExecRefConstants.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/ExecRefConstants.java
deleted file mode 100644
index 447ad81..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/ExecRefConstants.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-public class ExecRefConstants {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExecRefConstants.class);
-
- public static final String CONFIG_DEFAULT = "drill-exec-ref-default.json";
- public static final String CONFIG_OVERRIDE = "drill-exec-ref-site.json";
- public static final String STORAGE_ENGINE_SCAN_PACKAGES = "drill.storage.packages";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/IteratorRegistry.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/IteratorRegistry.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/IteratorRegistry.java
deleted file mode 100644
index 3b73d22..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/IteratorRegistry.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.apache.drill.exec.ref.rops.ROP;
-import org.apache.drill.exec.ref.rops.SinkROP;
-
-import com.google.common.collect.ArrayListMultimap;
-
-public class IteratorRegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IteratorRegistry.class);
-
- ArrayListMultimap<LogicalOperator, ROP> map = ArrayListMultimap.create();
- LinkedList<SinkROP> sinks = new LinkedList<SinkROP>();
-
- public void register(LogicalOperator logOp, ROP referenceOperator){
- map.put(logOp, referenceOperator);
- if(referenceOperator instanceof SinkROP) sinks.add((SinkROP) referenceOperator);
- }
-
- public void remap(LogicalOperator currentId, LogicalOperator newId){
- List<ROP> rops = map.removeAll(currentId);
- map.putAll(newId, rops);
- }
-
- public void swap(LogicalOperator op1, LogicalOperator op2){
- List<ROP> ops1 = map.removeAll(op1);
- List<ROP> ops2 = map.removeAll(op2);
- map.putAll(op1, ops2);
- map.putAll(op2, ops1);
- }
-
- public List<RecordIterator> getOperator(LogicalOperator o){
-// logger.debug("Getting iterator for Logical Operator {}", o);
- if(o == null) throw new SetupException("You requested a Iterator list for a null operator. This doesn't make any sense.");
- List<ROP> refOps = map.get(o);
- List<RecordIterator> iterators = new ArrayList<RecordIterator>(refOps.size());
- for(ROP r : refOps){
- RecordIterator iterator = r.getOutput();
- if(iterator == null) throw new SetupException(String.format("The provided iterator for the reference operator %s is null.", r));
- iterators.add(iterator);
- }
- return iterators;
- }
-
- public List<SinkROP> getSinks(){
- return sinks;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
deleted file mode 100644
index 5e4b3e4..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.Collection;
-
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Store;
-import org.apache.drill.common.logical.data.Union;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.exceptions.SetupException;
-import org.apache.drill.exec.ref.rops.ROP;
-import org.apache.drill.exec.ref.rops.ScanROP;
-import org.apache.drill.exec.ref.rops.StoreROP;
-import org.apache.drill.exec.ref.rops.UnionROP;
-import org.apache.drill.exec.ref.rse.RSERegistry;
-import org.apache.drill.exec.ref.rse.ReferenceStorageEngine;
-import org.apache.drill.exec.ref.rse.ReferenceStorageEngine.ReadEntry;
-
-import com.typesafe.config.Config;
-
-class ROPConverter {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ROPConverter.class);
-
- private LogicalPlan plan;
- private IteratorRegistry registry;
- private EvaluatorFactory builder;
- private RSERegistry engineRegistry;
- private Config config;
-
- public ROPConverter(LogicalPlan plan, IteratorRegistry registry, EvaluatorFactory builder, RSERegistry engineRegistry) {
- this.plan = plan;
- this.registry = registry;
- this.builder = builder;
- this.engineRegistry = engineRegistry;
- }
-
- public void convert(LogicalOperator o) throws SetupException {
-
- try {
- Method m = ROPConverter.class.getMethod("convertSpecific", o.getClass());
- m.invoke(this, o);
- return;
- } catch (NoSuchMethodException e) {
- // noop
- logger.debug("There is no convertSpecific method for type {}. Looking for a generic option...", o.getClass());
- } catch (IllegalAccessException | IllegalArgumentException | SecurityException e) {
- logger.debug("Failure while attempting to run convertSpecific value for class {}", o.getClass().getSimpleName(),
- e);
- } catch (InvocationTargetException e) {
- Throwable c = e.getCause();
- if (c instanceof SetupException) {
- throw (SetupException) c;
- } else {
- throw new RuntimeException("Failure while trying to run convertSpecific conversion of operator type "
- + o.getClass().getSimpleName(), c);
- }
- }
-
- String name = ROP.class.getPackage().getName() + "." + o.getClass().getSimpleName() + "ROP";
- try {
- Class<?> c = Class.forName(name);
- if (!ROP.class.isAssignableFrom(c)) throw new IllegalArgumentException("Class does not derive from ROP.");
- Constructor<?> ctor = c.getConstructor(o.getClass());
- if (ctor == null)
- throw new IllegalArgumentException(
- "Class does not have an available constructor that supports a single argument of the class "
- + o.getClass().getSimpleName());
- ROP r = (ROP) ctor.newInstance(o);
- r.init(registry, builder);
- return;
-
- } catch (NoSuchMethodException | ClassNotFoundException | IllegalAccessException | InstantiationException e) {
- logger.debug("No {} class that accepts a single parameter or type {}.", name, o.getClass().getCanonicalName());
- } catch (InvocationTargetException e) {
- Throwable c = e.getCause();
- if (c instanceof SetupException) throw (SetupException) c;
- throw new SetupException("Failure while trying to run Convert node of type " + o.getClass().getSimpleName(), c);
- }
-
- throw new UnsupportedOperationException("Unable to convert Logical Operator of type "
- + o.getClass().getCanonicalName());
- }
-
- private ReferenceStorageEngine getEngine(String name){
- StorageEngineConfig config = plan.getStorageEngineConfig(name);
- if(config == null) throw new SetupException(String.format("Unable to find define storage engine of name [%s].", name));
- ReferenceStorageEngine engine = engineRegistry.getEngine(config);
- return engine;
- }
-
- public void convertSpecific(Store store) throws SetupException {
- StoreROP rop = new StoreROP(store, getEngine(store.getStorageEngine()));
- rop.init(registry, builder);
- }
-
- public void convertSpecific(Scan scan) throws SetupException {
- StorageEngineConfig engineConfig = plan.getStorageEngineConfig(scan.getStorageEngine());
- ReferenceStorageEngine engine = engineRegistry.getEngine(engineConfig);
- Collection<ReadEntry> readEntries;
- try {
- readEntries = engine.getReadEntries(scan);
- } catch (IOException e1) {
- throw new SetupException("Failure reading input entries.", e1);
- }
-
- switch(readEntries.size()){
- case 0:
- throw new SetupException(String.format("Scan provided did not correspond to any available data.", scan));
- case 1:
- ScanROP scanner = new ScanROP(scan, readEntries.iterator().next(), engine);
- scanner.init(registry, builder);
- return;
- default:
- Union logOp = new Union(null, false);
-
- ROP parentUnion = new UnionROP(logOp);
- ScanROP[] scanners = new ScanROP[readEntries.size()];
- int i = 0;
- for (ReadEntry e : readEntries) {
- scanners[i] = new ScanROP(scan, e, engine);
- scanners[i].init(registry, builder);
- i++;
- }
-
- parentUnion.init(registry, builder);
- registry.swap(logOp, scan); // make it so future things point to the union as the original scans.
- return;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/RecordIterator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/RecordIterator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/RecordIterator.java
deleted file mode 100644
index 2460db6..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/RecordIterator.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import java.util.Iterator;
-
-import org.apache.drill.exec.ref.rops.ROP;
-
-
-public interface RecordIterator{
-
- public enum NextOutcome {NONE_LEFT, INCREMENTED_SCHEMA_UNCHANGED, INCREMENTED_SCHEMA_CHANGED}
- public RecordPointer getRecordPointer(); // called once
- public NextOutcome next();
- public ROP getParent();
-
-
- public static class IteratorWrapper implements RecordIterator{
- final Iterator<RecordPointer> iter;
- final RecordPointer outputRecord;
- final ROP parent;
- public IteratorWrapper(ROP rop, Iterator<RecordPointer> iter, RecordPointer outputRecord) {
- this.iter = iter;
- this.parent = rop;
- this.outputRecord = outputRecord;
- }
-
- @Override
- public NextOutcome next() {
- if(iter.hasNext()) {
- outputRecord.copyFrom(iter.next());
- return NextOutcome.INCREMENTED_SCHEMA_CHANGED;
- }
-
- return NextOutcome.NONE_LEFT;
- }
-
- @Override
- public ROP getParent() {
- return parent;
- }
-
- @Override
- public RecordPointer getRecordPointer() {
- return outputRecord;
- }
-
-
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/RecordPointer.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/RecordPointer.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/RecordPointer.java
deleted file mode 100644
index 3e70fe7..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/RecordPointer.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import java.io.IOException;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.ref.rops.DataWriter;
-import org.apache.drill.exec.ref.values.DataValue;
-
-
-public interface RecordPointer {
- public DataValue getField(SchemaPath field);
- public void addField(SchemaPath field, DataValue value);
- public void addField(PathSegment segment, DataValue value);
- public void removeField(SchemaPath segment);
- public void write(DataWriter writer) throws IOException;
- public RecordPointer copy();
- public void copyFrom(RecordPointer r);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/ReferenceInterpreter.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/ReferenceInterpreter.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/ReferenceInterpreter.java
deleted file mode 100644
index 6859274..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/ReferenceInterpreter.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
-import org.apache.drill.exec.ref.eval.EvaluatorFactory;
-import org.apache.drill.exec.ref.rops.SinkROP;
-import org.apache.drill.exec.ref.rse.RSERegistry;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-
-public class ReferenceInterpreter {
- static final Logger logger = LoggerFactory.getLogger(ReferenceInterpreter.class);
-
- private List<SinkROP> sinks = new ArrayList<SinkROP>();
- private LogicalPlan plan;
- private ROPConverter converter;
- private IteratorRegistry registry;
-
- public ReferenceInterpreter(LogicalPlan p, IteratorRegistry r, EvaluatorFactory builder, RSERegistry rses){
- this.plan = p;
- this.registry = r;
- this.converter = new ROPConverter(p, registry, builder, rses);
- }
-
- /** Generate Reference equivalents to each operation and then collect and store all the sinks.
- * @throws IOException **/
- public void setup() throws IOException{
- for(LogicalOperator op : plan.getSortedOperators()){
- converter.convert(op);
- }
- sinks.addAll(registry.getSinks());
- }
-
- public Collection<RunOutcome> run(){
- Collection<RunOutcome> outcomes = new LinkedList<RunOutcome>();
-
- for(SinkROP r : sinks){
- outcomes.add(r.run(new BasicStatusHandle()));
- }
-
- return outcomes;
- }
-
- public void cleanup(){
-
- }
-
-
- public static void main(String[] args) throws Exception{
- DrillConfig config = DrillConfig.create();
- int arg = 0;
- final BlockingQueue<Object> queue;
- if (arg < args.length && args[arg].equals("--stdout")) {
- ++arg;
- queue = new ArrayBlockingQueue<>(100);
- config.setSinkQueues(0, queue);
- } else {
- queue = null;
- }
- final String jsonFile = args[arg];
- final String planString;
- if (jsonFile.startsWith("inline:")) {
- planString = jsonFile.substring("inline:".length());
- } else {
- planString = Files.toString(new File(jsonFile), Charsets.UTF_8);
- }
- LogicalPlan plan = LogicalPlan.parse(config, planString);
- IteratorRegistry ir = new IteratorRegistry();
- ReferenceInterpreter i = new ReferenceInterpreter(plan, ir, new BasicEvaluatorFactory(ir), new RSERegistry(config));
- i.setup();
- final Object[] result = {null};
- final Thread thread;
- if (queue != null) {
- thread = new Thread(
- new Runnable() {
- @Override
- public void run() {
- try {
- result[0] = run0();
- } catch (Throwable e) {
- result[0] = e;
- }
- }
- private boolean run0() throws IOException {
- for (;;) {
- try {
- Object o = queue.take();
- if (o instanceof RunOutcome.OutcomeType) {
- switch ((RunOutcome.OutcomeType) o) {
- case SUCCESS:
- return true; // end of data
- case CANCELED:
- throw new RuntimeException("canceled");
- case FAILED:
- default:
- throw new RuntimeException("failed");
- }
- } else {
- System.out.write((byte[]) o);
- }
- } catch (InterruptedException e) {
- Thread.interrupted();
- throw new RuntimeException(e);
- }
- }
- }
- });
- thread.start();
- } else {
- thread = null;
- }
- Collection<RunOutcome> outcomes = i.run();
-
- for(RunOutcome outcome : outcomes){
- System.out.println("============");
- System.out.println(outcome);
- if(outcome.outcome == RunOutcome.OutcomeType.FAILED && outcome.exception != null){
- outcome.exception.printStackTrace();
- }
- }
- if (thread != null) {
- thread.join();
- System.out.println("Result: " + result[0]);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/ReferenceTransform.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/ReferenceTransform.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/ReferenceTransform.java
deleted file mode 100644
index e31c09b..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/ReferenceTransform.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import org.apache.drill.exec.ref.rops.ReferenceOperator;
-
-public class ReferenceTransform implements ReferenceOperator{
-
- @Override
- public void setInput(RecordIterator incoming) {
- }
-
- @Override
- public RecordIterator getOutput() {
- return null;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/RunOutcome.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/RunOutcome.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/RunOutcome.java
deleted file mode 100644
index 5a07031..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/RunOutcome.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-public class RunOutcome {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RunOutcome.class);
-
- public enum OutcomeType{
- SUCCESS, FAILED, CANCELED;
- }
-
- public final OutcomeType outcome;
- public final long bytes;
- public final long records;
- public final Throwable exception;
-
- public RunOutcome(OutcomeType outcome, long bytes, long records, Throwable exception) {
- super();
- if(outcome != OutcomeType.SUCCESS) logger.warn("Creating failed outcome.", exception);
- this.outcome = outcome;
- this.bytes = bytes;
- this.records = records;
- this.exception = exception;
- }
-
- public RunOutcome(OutcomeType outcome, long bytes, long records) {
- this(outcome, bytes, records, null);
- }
-
- @Override
- public String toString() {
- return "RunOutcome [outcome=" + outcome + ", bytes=" + bytes + ", records=" + records + ", exception=" + exception
- + "]";
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/UnbackedRecord.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/UnbackedRecord.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/UnbackedRecord.java
deleted file mode 100644
index 29c83d4..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/UnbackedRecord.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import java.io.IOException;
-
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.ValueExpressions.CollisionBehavior;
-import org.apache.drill.exec.ref.rops.DataWriter;
-import org.apache.drill.exec.ref.values.ContainerValue;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.SimpleMapValue;
-import org.apache.drill.exec.ref.values.ValueUtils;
-
-public class UnbackedRecord implements RecordPointer {
-
- private DataValue root = new SimpleMapValue();
-
- public DataValue getField(SchemaPath field) {
- return root.getValue(field.getRootSegment());
- }
-
- public void addField(SchemaPath field, DataValue value) {
- addField(field.getRootSegment(), value);
- }
-
- @Override
- public void addField(PathSegment segment, DataValue value) {
- root.addValue(segment, value);
- }
-
- @Override
- public void removeField(SchemaPath field) {
- root.removeValue(field.getRootSegment());
- }
-
- @Override
- public void write(DataWriter writer) throws IOException {
- writer.startRecord();
- root.write(writer);
- writer.endRecord();
- }
-
- public void merge(DataValue v) {
- if (v instanceof ContainerValue) {
- this.root = ValueUtils.getMergedDataValue(CollisionBehavior.MERGE_OVERRIDE, root, v);
- } else {
- this.root = v;
- }
- }
-
- public void merge(RecordPointer pointer) {
- if (pointer instanceof UnbackedRecord) {
- merge(UnbackedRecord.class.cast(pointer).root);
- } else {
- throw new UnsupportedOperationException(
- String.format("Unable to merge from a record of type %s to an UnbackedRecord.", pointer.getClass().getCanonicalName())
- );
- }
- }
-
- @Override
- public RecordPointer copy() {
- UnbackedRecord r = new UnbackedRecord();
- r.root = this.root.copy();
- return r;
- }
-
- public void clear() {
- root = new SimpleMapValue();
- }
-
- public void setClearAndSetRoot(SchemaPath path, DataValue v) {
- root = new SimpleMapValue();
- root.addValue(path.getRootSegment(), v);
- }
-
- @Override
- public void copyFrom(RecordPointer r) {
- if (r instanceof UnbackedRecord) {
- this.root = ((UnbackedRecord) r).root.copy();
- } else {
- throw new UnsupportedOperationException(String.format("Unable to copy from a record of type %s to an UnbackedRecord.", r.getClass().getCanonicalName()));
- }
- }
-
- @Override
- public String toString() {
- return "UnbackedRecord [root=" + root + "]";
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((root == null) ? 0 : root.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) return true;
- if (obj == null) return false;
- if (getClass() != obj.getClass()) return false;
- UnbackedRecord other = (UnbackedRecord) obj;
- if (root == null) {
- if (other.root != null) return false;
- } else if (!root.equals(other.root)) return false;
- return true;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BaseBasicEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BaseBasicEvaluator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BaseBasicEvaluator.java
deleted file mode 100644
index e6b7e7c..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BaseBasicEvaluator.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval;
-
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-
-public abstract class BaseBasicEvaluator implements BasicEvaluator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseBasicEvaluator.class);
-
- private final boolean isConstant;
- protected final RecordPointer record;
-
- public BaseBasicEvaluator(boolean isConstant, RecordPointer record) {
- super();
- this.record = record;
- this.isConstant = isConstant;
- }
-
- @Override
- public final boolean isConstant() {
- return isConstant;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BasicEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BasicEvaluatorFactory.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BasicEvaluatorFactory.java
deleted file mode 100644
index 13b7729..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BasicEvaluatorFactory.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.expression.visitors.SimpleExprVisitor;
-import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BooleanEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.ConnectedEvaluator;
-import org.apache.drill.exec.ref.eval.fn.agg.AggregatingWrapperEvaluator;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ValueReader;
-
-public class BasicEvaluatorFactory extends EvaluatorFactory{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicEvaluatorFactory.class);
-
- public BasicEvaluatorFactory(IteratorRegistry registry){
-
- }
-
- private SimpleExprVisitor<BasicEvaluator> get(RecordPointer record){
- return new SimpleEvaluationVisitor(record);
- }
-
- @Override
- public BasicEvaluator getBasicEvaluator(RecordPointer inputRecord, LogicalExpression e) {
- return e.accept(get(inputRecord), null);
- }
-
-
-
- @Override
- public AggregatingEvaluator getAggregatingOperator(RecordPointer record, LogicalExpression e) {
- SimpleEvaluationVisitor visitor = new SimpleEvaluationVisitor(record);
- BasicEvaluator b = e.accept(visitor, null);
- return new AggregatingWrapperEvaluator(visitor.getAggregators(), b);
- }
-
- @Override
- public BooleanEvaluator getBooleanEvaluator(RecordPointer record, LogicalExpression e) {
- return new BooleanEvaluatorImpl(e.accept(get(record), null));
- }
-
- @Override
- public ConnectedEvaluator getConnectedEvaluator(RecordPointer record, NamedExpression ne) {
- return new ConnectedEvaluatorImpl(record, ne);
- }
-
-
- private class BooleanEvaluatorImpl implements BooleanEvaluator{
- private BasicEvaluator eval;
-
- public BooleanEvaluatorImpl(BasicEvaluator e){
- this.eval = e;
- }
-
- @Override
- public boolean eval() {
- return ValueReader.getBoolean(eval.eval());
- }
-
- }
-
- private class ConnectedEvaluatorImpl implements ConnectedEvaluator{
- private SchemaPath outputPath;
- private BasicEvaluator eval;
- private RecordPointer record;
-
- public ConnectedEvaluatorImpl(RecordPointer record, NamedExpression e){
- this.outputPath = e.getRef();
- this.record = record;
- this.eval = e.getExpr().accept(get(record), null);
- }
-
- @Override
- public void eval() {
- DataValue val = eval.eval();
- record.addField(outputPath, val);
- }
-
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BoundarySubscriber.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BoundarySubscriber.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BoundarySubscriber.java
deleted file mode 100644
index 5e01fb6..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/BoundarySubscriber.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval;
-
-public interface BoundarySubscriber {
- public void crossedBoundary();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/EvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/EvaluatorFactory.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/EvaluatorFactory.java
deleted file mode 100644
index 084fe70..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/EvaluatorFactory.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.AggregatingEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BooleanEvaluator;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.ConnectedEvaluator;
-
-public abstract class EvaluatorFactory {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EvaluatorFactory.class);
-
- /**
- * Provide an evaluator that returns a true/false value on evaluation.
- * @param e The base logical expression
- * @return true/false of expression.
- */
- public abstract BooleanEvaluator getBooleanEvaluator(RecordPointer record, LogicalExpression e);
-
- /**
- * Provide an evaluator that takes the output value of the named expression evaluation and place that value back into the simple record.
- * @param ne The named expression to work with.
- * @return
- */
- public abstract ConnectedEvaluator getConnectedEvaluator(RecordPointer record, NamedExpression ne);
-
- /**
- * Provide a basic evaluator that returns the data value interested in.
- * @param e
- * @return
- */
- public abstract BasicEvaluator getBasicEvaluator(RecordPointer record, LogicalExpression e);
-
- /**
- * Given a set of named expressions, please provide a single
- * @param expressions
- * @return
- */
- public abstract AggregatingEvaluator getAggregatingOperator(RecordPointer record, LogicalExpression e);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/EvaluatorTypes.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/EvaluatorTypes.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/EvaluatorTypes.java
deleted file mode 100644
index a108f06..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/EvaluatorTypes.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval;
-
-import org.apache.drill.exec.ref.values.DataValue;
-
-public interface EvaluatorTypes {
-
- /**
- * The base form of evaluator. Other evaluators are typically just masks of this type of evaluator.
- */
- public interface BasicEvaluator{
- public DataValue eval();
- public boolean isConstant();
- }
-
- /**
- * An evaluator that returns true or false based on the input record.
- */
- public interface BooleanEvaluator{
- public boolean eval();
- }
-
- /**
- * An evaluator that returns no value. It is expected to write the output to the provided input record instead.
- */
- public interface ConnectedEvaluator{
- public void eval();
- }
-
- /**
- * An evaluator that takes multiple input values and returns a single output value.
- */
- public interface AggregatingEvaluator extends BasicEvaluator{
- public void addRecord();
- }
-
-
- public interface ConstantEvaluator{
- public DataValue eval();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/FieldEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/FieldEvaluator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/FieldEvaluator.java
deleted file mode 100644
index 2b1a218..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/FieldEvaluator.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval;
-
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.DataValue;
-
-public class FieldEvaluator implements BasicEvaluator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FieldEvaluator.class);
-
- private final SchemaPath path;
- private final RecordPointer record;
-
- public FieldEvaluator(SchemaPath p, RecordPointer record){
- this.path = p;
- this.record = record;
- }
-
- @Override
- public DataValue eval() {
- return record.getField(path);
- }
-
- @Override
- public boolean isConstant() {
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/IfEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/IfEvaluator.java b/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/IfEvaluator.java
deleted file mode 100644
index c7f242e..0000000
--- a/exec/ref/src/main/java/org/apache/drill/exec/ref/eval/IfEvaluator.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.eval;
-
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.IfExpression.IfCondition;
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.eval.EvaluatorTypes.BasicEvaluator;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ValueReader;
-
-public class IfEvaluator implements BasicEvaluator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IfEvaluator.class);
-
- private final IfCond[] conditions;
- private final BasicEvaluator elseExpression;
-
- public IfEvaluator(IfExpression expression, ExprVisitor<BasicEvaluator, Void, RuntimeException> evalBuilder, RecordPointer record){
- this.conditions = new IfCond[expression.conditions.size()];
- for(int i =0; i < conditions.length; i++){
- conditions[i] = new IfCond(expression.conditions.get(i), evalBuilder);
- }
- elseExpression = expression.elseExpression.accept(evalBuilder, null);
- }
-
- @Override
- public DataValue eval() {
- for(int i = 0 ; i < conditions.length; i++){
- if(ValueReader.getBoolean(conditions[i].condition.eval())) return conditions[i].valueExpression.eval() ;
- }
- return elseExpression.eval();
- }
-
- public class IfCond{
- private final BasicEvaluator condition;
- private final BasicEvaluator valueExpression;
-
- public IfCond(IfCondition c, ExprVisitor<BasicEvaluator, Void, RuntimeException> evalBuilder){
- this.condition = c.condition.accept(evalBuilder, null);
- this.valueExpression = c.expression.accept(evalBuilder, null);
- }
-
- public boolean matches(RecordPointer r){
- return ValueReader.getBoolean(condition.eval());
- }
-
- public DataValue eval(RecordPointer r){
- return valueExpression.eval();
- }
-
- public boolean isConstant() {
- return condition.isConstant() && valueExpression.isConstant();
- }
- }
-
- @Override
- public boolean isConstant() {
- for(IfCond c : conditions){
- if(!c.isConstant()) return false;
- }
- return elseExpression.isConstant();
- }
-
-
-}
[33/38] git commit: DRILL-377: Use BufferedInputStream in
SpoolingRawBatchBuffer to improve performance
Posted by ja...@apache.org.
DRILL-377: Use BufferedInputStream in SpoolingRawBatchBuffer to improve performance
Signed-off-by: Jacques Nadeau <ja...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/7f7b6be2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/7f7b6be2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/7f7b6be2
Branch: refs/heads/master
Commit: 7f7b6be21af3e2744b4ee867e5da4488f526037b
Parents: ad3ac80
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Feb 13 00:36:59 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/store/LocalSyncableFileSystem.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7f7b6be2/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
index 10a4dc5..8d525f3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java
@@ -136,10 +136,10 @@ public class LocalSyncableFileSystem extends FileSystem {
public class LocalInputStream extends InputStream implements Seekable, PositionedReadable {
- private FileInputStream input;
+ private BufferedInputStream input;
public LocalInputStream(Path path) throws IOException {
- input = new FileInputStream(path.toString());
+ input = new BufferedInputStream(new FileInputStream(path.toString()), 1024*1024);
}
@Override
[15/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 047d3d9..15b2f53 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -1,21 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>drill-root</artifactId>
@@ -23,37 +18,39 @@
<version>1.0.0-m2-incubating-SNAPSHOT</version>
</parent>
- <groupId>org.apache.drill</groupId>
<artifactId>distribution</artifactId>
<packaging>pom</packaging>
<name>Packaging and Distribution Assembly</name>
-
+
<dependencies>
<dependency>
+ <groupId>sqlline</groupId>
+ <artifactId>sqlline</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.drill</groupId>
<artifactId>drill-protocol</artifactId>
<version>${project.version}</version>
+ <classifier>rebuffed</classifier>
</dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
- <artifactId>drill-java-exec</artifactId>
+ <artifactId>drill-buffers</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
- <artifactId>drill-netty-bufferl</artifactId>
- <version>4.0.7.Final</version>
- </dependency>
- <dependency>
- <groupId>org.apache.drill.exec</groupId>
- <artifactId>drill-ref</artifactId>
+ <artifactId>drill-java-exec</artifactId>
<version>${project.version}</version>
+ <classifier>rebuffed</classifier>
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
<artifactId>drill-common</artifactId>
<version>${project.version}</version>
+ <classifier>rebuffed</classifier>
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
@@ -88,9 +85,10 @@
</excludes>
</configuration>
</plugin>
-
+
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
<executions>
<execution>
<id>distro-assembly</id>
@@ -148,7 +146,7 @@
</build>
</profile>
<!-- END SNIPPET: release-profile -->
-
+
<profile>
<id>rpm</id>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index c50df09..60aea67 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -22,28 +22,28 @@
<id>binary-release</id>
<formats>
<format>tar.gz</format>
- <format>dir</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<moduleSets>
- <moduleSet>
- <!-- Enable access to all projects in the current multimodule build! -->
- <useAllReactorProjects>true</useAllReactorProjects>
+ </moduleSets>
+ <dependencySets>
+ <dependencySet>
- <!-- Now, select which projects to include in this module-set. -->
+<!-- Now, select which projects to include in this module-set. -->
<includes>
<include>org.apache.drill:drill-sqlparser:jar</include>
- <include>org.apache.drill.exec:drill-netty-bufferl</include>
- <include>org.apache.drill.exec:drill-ref</include>
+ <include>org.apache.drill.exec:drill-buffers</include>
+ <include>org.apache.drill:drill-protocol:jar:rebuffed</include>
+ <include>org.apache.drill:drill-common:jar:rebuffed</include>
+ <include>org.apache.drill.exec:drill-java-exec:jar:rebuffed</include>
</includes>
- <binaries>
- <outputDirectory>jars</outputDirectory>
- <unpack>false</unpack>
- </binaries>
- </moduleSet>
- </moduleSets>
- <dependencySets>
+
+
+
+ <outputDirectory>jars</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ </dependencySet>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
@@ -51,7 +51,8 @@
<excludes>
<exclude>org.apache.drill</exclude>
<exclude>org.apache.drill.exec</exclude>
- <exclude>com.google.protobuf:protobuf-java:jar:2.5.0</exclude>
+ <exclude>org.hsqldb:hsqldb</exclude> <!-- exclude or sqlline has problems -->
+ <exclude>hsqldb:hsqldb</exclude> <!-- exclude or sqlline has problems -->
</excludes>
<scope>test</scope>
</dependencySet>
@@ -75,22 +76,6 @@
<files>
<file>
- <source>../protocol/target/drill-protocol-${project.version}-rebuffed.jar</source>
- <outputDirectory>jars</outputDirectory>
- </file>
- <file>
- <source>../common/target/drill-common-${project.version}-rebuffed.jar</source>
- <outputDirectory>jars</outputDirectory>
- </file>
- <file>
- <source>../exec/java-exec/target/drill-java-exec-${project.version}-rebuffed.jar</source>
- <outputDirectory>jars</outputDirectory>
- </file>
- <file>
- <source>../contrib/storage-hive/target/storage-hive-${project.version}.jar</source>
- <outputDirectory>jars</outputDirectory>
- </file>
- <file>
<source>src/resources/runbit</source>
<outputDirectory>bin</outputDirectory>
</file>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/distribution/src/resources/sqlline
----------------------------------------------------------------------
diff --git a/distribution/src/resources/sqlline b/distribution/src/resources/sqlline
index c815e8b..c526f03 100755
--- a/distribution/src/resources/sqlline
+++ b/distribution/src/resources/sqlline
@@ -72,7 +72,7 @@ DRILL_SHELL_JAVA_OPTS="$DRILL_SHELL_JAVA_OPTS -Dlog.path=$DRILL_LOG_DIR/sqlline.
if [ -n "$QUERY" ]
then
- echo $QUERY | exec $JAVA $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine "${ARGS[@]}"
+ echo $QUERY | exec $JAVA $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver "${ARGS[@]}"
elif [ -n "$FILE" ]
then
exec $JAVA $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine "${ARGS[@]}" --run=$FILE
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/bufferl/pom.xml
----------------------------------------------------------------------
diff --git a/exec/bufferl/pom.xml b/exec/bufferl/pom.xml
index 834a98f..a8e48ad 100644
--- a/exec/bufferl/pom.xml
+++ b/exec/bufferl/pom.xml
@@ -18,9 +18,7 @@
<version>1.0.0-m2-incubating-SNAPSHOT</version>
</parent>
- <groupId>org.apache.drill.exec</groupId>
- <version>4.0.7.Final</version>
- <artifactId>drill-netty-bufferl</artifactId>
+ <artifactId>drill-buffers</artifactId>
<name>exec/Netty Little Endian Buffers</name>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 9ea8304..a016afc 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -44,12 +44,10 @@
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
- <version>2.9.1</version>
</dependency>
<dependency>
<groupId>xalan</groupId>
<artifactId>xalan</artifactId>
- <version>2.7.1</version>
</dependency>
<dependency>
<groupId>com.sun.codemodel</groupId>
@@ -58,12 +56,12 @@
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
- <artifactId>commons-compiler-jdk</artifactId>
- <version>2.6.1</version>
+ <artifactId>janino</artifactId>
+ <version>2.7.3</version>
</dependency>
<dependency>
<groupId>net.hydromatic</groupId>
- <artifactId>optiq</artifactId>
+ <artifactId>optiq-core</artifactId>
</dependency>
<dependency>
<groupId>org.freemarker</groupId>
@@ -139,8 +137,8 @@
</dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
- <version>4.0.7.Final</version>
- <artifactId>drill-netty-bufferl</artifactId>
+ <version>${project.version}</version>
+ <artifactId>drill-buffers</artifactId>
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
@@ -196,7 +194,7 @@
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
- <version>3.1</version>
+ <version>3.1.4</version>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 3e5a156..5083f74 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -18,6 +18,7 @@ import com.google.common.base.Charsets;
import com.google.common.collect.ObjectArrays;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
import org.apache.commons.lang3.ArrayUtils;
@@ -39,6 +40,8 @@ import java.util.Random;
import java.util.List;
import java.io.Closeable;
+import java.io.InputStream;
+import java.io.InputStreamReader;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
index c357dd6..2d81299 100644
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -155,6 +155,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
return valueCount;
}
+ public boolean isNull(int index){
+ return false;
+ }
+
<#if (type.width > 8)>
public ${minor.javaType!type.javaType} get(int index) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 127c6fd..4677374 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -243,7 +243,12 @@ package org.apache.drill.exec.vector;
</#if> get(int index, int positionIndex) {
return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
}
+
+ public boolean isNull(int index){
+ return false;
+ }
+
public void get(int index, Repeated${minor.class}Holder holder){
holder.start = offsets.getAccessor().get(index);
holder.end = offsets.getAccessor().get(index+1);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/SqlAccessors.java b/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
new file mode 100644
index 0000000..efaf9a6
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/SqlAccessors.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<#list ["", "Nullable"] as mode>
+<#assign name = mode + minor.class?cap_first />
+<#assign javaType = (minor.javaType!type.javaType) />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/accessor/${name}Accessor.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector.accessor;
+
+<#include "/@includes/vv_imports.ftl" />
+
+@SuppressWarnings("unused")
+public class ${name}Accessor extends AbstractSqlAccessor{
+ <#if mode == "Nullable">
+ private static final MajorType TYPE = Types.optional(MinorType.${minor.class?upper_case});
+ <#else>
+ private static final MajorType TYPE = Types.required(MinorType.${minor.class?upper_case});
+ </#if>
+
+ private final ${name}Vector.Accessor ac;
+
+ public ${name}Accessor(${name}Vector vector){
+ this.ac = vector.getAccessor();
+ }
+
+ public Object getObject(int index){
+ return ac.getObject(index);
+ }
+
+ <#if type.major == "VarLen">
+
+ @Override
+ public InputStream getStream(int index){
+ ${name}Holder h = new ${name}Holder();
+ ac.get(index, h);
+ return new ByteBufInputStream(h.buffer.slice(h.start, h.end));
+ }
+
+ @Override
+ public byte[] getBytes(int index){
+ return ac.get(index);
+ }
+
+ <#switch minor.class>
+ <#case "VarBinary">
+ <#break>
+ <#case "VarChar">
+ @Override
+ public InputStreamReader getReader(int index){
+ return new InputStreamReader(getStream(index), Charsets.UTF_8);
+ }
+
+ @Override
+ public String getString(int index){
+ return new String(getBytes(index), Charsets.UTF_8);
+ }
+
+
+ <#break>
+ <#case "Var16Char">
+ @Override
+ public InputStreamReader getReader(int index){
+ return new InputStreamReader(getStream(index), Charsets.UTF_16);
+ }
+
+ @Override
+ public String getString(int index){
+ return new String(getBytes(index), Charsets.UTF_16);
+ }
+
+
+ <#break>
+ <#default>
+ This is uncompilable code
+ </#switch>
+
+ <#else>
+ @Override
+ public ${javaType} get${javaType?cap_first}(int index){
+ return ac.get(index);
+ }
+ </#if>
+
+ @Override
+ public boolean isNull(int index){
+ return false;
+ }
+
+ @Override
+ MajorType getType(){return TYPE;};
+
+}
+
+
+</#list>
+</#list>
+</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
index 783f943..8c72cf7 100644
--- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -23,7 +23,7 @@
package org.apache.drill.exec.expr;
<#include "/@includes/vv_imports.ftl" />
-
+import org.apache.drill.exec.vector.accessor.*;
public class TypeHelper {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
@@ -47,6 +47,25 @@ public class TypeHelper {
throw new UnsupportedOperationException();
}
+ public static SqlAccessor getSqlAccessor(ValueVector vector){
+ switch(vector.getField().getType().getMinorType()){
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case}:
+ switch (vector.getField().getType().getMode()) {
+ case REQUIRED:
+ return new ${minor.class}Accessor((${minor.class}Vector) vector);
+ case OPTIONAL:
+ return new Nullable${minor.class}Accessor((Nullable${minor.class}Vector) vector);
+ case REPEATED:
+ throw new UnsupportedOperationException();
+ }
+ </#list>
+ </#list>
+ }
+ throw new UnsupportedOperationException();
+ }
+
public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
switch (type) {
<#list vv.types as type>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index b059d89..5cd83af 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -225,14 +225,35 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
holder.buffer = data;
}
+
+ <#switch minor.class>
+ <#case "VarChar">
+ public Object getObject(int index) {
+ return new String(get(index), Charsets.UTF_8);
+ }
+ <#break>
+ <#case "Var16Char">
+ public Object getObject(int index) {
+ return new String(get(index), Charsets.UTF_16);
+ }
+ <#break>
+ <#default>
public Object getObject(int index) {
return get(index);
}
+
+ </#switch>
+
+
public int getValueCount() {
return valueCount;
}
+ public boolean isNull(int index){
+ return false;
+ }
+
public UInt${type.width}Vector getOffsetVector(){
return offsetVector;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/cache/HazelCache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/HazelCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/HazelCache.java
index 2f0ab2e..5f370ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/cache/HazelCache.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/cache/HazelCache.java
@@ -35,6 +35,7 @@ import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.hazelcast.config.Config;
+import com.hazelcast.config.MapConfig;
import com.hazelcast.config.SerializerConfig;
import com.hazelcast.core.DuplicateInstanceNameException;
import com.hazelcast.core.Hazelcast;
@@ -132,6 +133,10 @@ public class HazelCache implements DistributedCache {
@Override
public <V extends DrillSerializable> DistributedMap<V> getMap(Class<V> clazz) {
IMap<String, V> imap = this.instance.getMap(clazz.toString());
+ MapConfig myMapConfig = new MapConfig();
+ myMapConfig.setBackupCount(0);
+ myMapConfig.setReadBackupData(true);
+ instance.getConfig().getMapConfigs().put(clazz.toString(), myMapConfig);
return new HCDistributedMapImpl<V>(imap, clazz);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index 74eed94..1297cb3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -21,8 +21,6 @@ import static com.google.common.base.Preconditions.checkState;
import static org.apache.drill.exec.proto.UserProtos.QueryResultsMode.STREAM_FULL;
import static org.apache.drill.exec.proto.UserProtos.RunQuery.newBuilder;
import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.PooledByteBufAllocatorL;
import io.netty.channel.nio.NioEventLoopGroup;
import java.io.Closeable;
@@ -37,11 +35,18 @@ import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.ZKClusterCoordinator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.proto.UserProtos.QueryType;
-import org.apache.drill.exec.rpc.*;
+import org.apache.drill.exec.proto.UserProtos.RpcType;
import org.apache.drill.exec.rpc.BasicClientWithConnection.ServerConnection;
+import org.apache.drill.exec.rpc.ChannelClosedException;
+import org.apache.drill.exec.rpc.DrillRpcFuture;
+import org.apache.drill.exec.rpc.NamedThreadFactory;
+import org.apache.drill.exec.rpc.RpcConnectionHandler;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.rpc.user.UserClient;
import org.apache.drill.exec.rpc.user.UserResultsListener;
@@ -52,7 +57,7 @@ import com.google.common.util.concurrent.SettableFuture;
/**
* Thin wrapper around a UserClient that handles connect/close and transforms String into ByteBuf
*/
-public class DrillClient implements Closeable{
+public class DrillClient implements Closeable, ConnectionThrottle{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillClient.class);
DrillConfig config;
@@ -86,17 +91,28 @@ public class DrillClient implements Closeable{
return config;
}
+ @Override
+ public void setAutoRead(boolean enableAutoRead) {
+ client.setAutoRead(enableAutoRead);
+ }
+
+
+
/**
* Connects the client to a Drillbit server
*
* @throws IOException
*/
- public synchronized void connect() throws RpcException {
+ public void connect() throws RpcException {
+ connect((String) null);
+ }
+
+ public synchronized void connect(String connect) throws RpcException {
if (connected) return;
if (clusterCoordinator == null) {
try {
- this.clusterCoordinator = new ZKClusterCoordinator(this.config);
+ this.clusterCoordinator = new ZKClusterCoordinator(this.config, connect);
this.clusterCoordinator.start(10000);
} catch (Exception e) {
throw new RpcException("Failure setting up ZK for client.", e);
@@ -172,6 +188,11 @@ public class DrillClient implements Closeable{
return listener.getResults();
}
+ public void cancelQuery(QueryId id){
+ client.send(RpcType.CANCEL_QUERY, id, Ack.class);
+ }
+
+
/**
* Submits a Logical plan for direct execution (bypasses parsing)
*
@@ -217,7 +238,7 @@ public class DrillClient implements Closeable{
}
@Override
- public void resultArrived(QueryResultBatch result) {
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
// logger.debug("Result arrived. Is Last Chunk: {}. Full Result: {}", result.getHeader().getIsLastChunk(), result);
results.add(result);
if(result.getHeader().getIsLastChunk()){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
index 7adefdb..b07b3ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
@@ -17,43 +17,33 @@
*/
package org.apache.drill.exec.client;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.internal.Lists;
-import com.google.common.base.Charsets;
-import com.google.common.base.Stopwatch;
-import com.google.common.io.Resources;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.lang.StringUtils;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.ZKClusterCoordinator;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RemoteRpcException;
import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.server.BootStrapContext;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.util.VectorUtil;
-import org.apache.drill.exec.vector.ValueVector;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharsetDecoder;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
public class QuerySubmitter {
@@ -99,42 +89,50 @@ public class QuerySubmitter {
public int submitQuery(String planLocation, String type, String zkQuorum, boolean local, int bits) throws Exception {
DrillConfig config = DrillConfig.create();
- DrillClient client;
- if (local) {
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
- Drillbit[] drillbits = new Drillbit[bits];
- for (int i = 0; i < bits; i++) {
- drillbits[i] = new Drillbit(config, serviceSet);
- drillbits[i].run();
+ DrillClient client = null;
+
+ try{
+ if (local) {
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ Drillbit[] drillbits = new Drillbit[bits];
+ for (int i = 0; i < bits; i++) {
+ drillbits[i] = new Drillbit(config, serviceSet);
+ drillbits[i].run();
+ }
+ client = new DrillClient(config, serviceSet.getCoordinator());
+ } else {
+ ZKClusterCoordinator clusterCoordinator = new ZKClusterCoordinator(config, zkQuorum);
+ clusterCoordinator.start(10000);
+ client = new DrillClient(config, clusterCoordinator);
}
- client = new DrillClient(config, serviceSet.getCoordinator());
- } else {
- ZKClusterCoordinator clusterCoordinator = new ZKClusterCoordinator(config, zkQuorum);
- clusterCoordinator.start(10000);
- client = new DrillClient(config, clusterCoordinator);
- }
- client.connect();
- QueryResultsListener listener = new QueryResultsListener();
- String plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
- UserProtos.QueryType queryType;
- type = type.toLowerCase();
- switch(type) {
- case "logical":
- queryType = UserProtos.QueryType.LOGICAL;
- break;
- case "physical":
- queryType = UserProtos.QueryType.PHYSICAL;
- break;
- default:
- System.out.println("Invalid query type: " + type);
- return -1;
+ client.connect();
+ QueryResultsListener listener = new QueryResultsListener();
+ String plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
+ UserProtos.QueryType queryType;
+ type = type.toLowerCase();
+ switch(type) {
+ case "sql":
+ queryType = UserProtos.QueryType.SQL;
+ break;
+ case "logical":
+ queryType = UserProtos.QueryType.LOGICAL;
+ break;
+ case "physical":
+ queryType = UserProtos.QueryType.PHYSICAL;
+ break;
+ default:
+ System.out.println("Invalid query type: " + type);
+ return -1;
+ }
+ Stopwatch watch = new Stopwatch();
+ watch.start();
+ client.runQuery(queryType, plan, listener);
+ int rows = listener.await();
+ System.out.println(String.format("Got %d record%s in %f seconds", rows, rows > 1 ? "s" : "", (float)watch.elapsed(TimeUnit.MILLISECONDS) / (float)1000));
+ return 0;
+ }finally{
+ if(client != null) client.close();
}
- Stopwatch watch = new Stopwatch();
- watch.start();
- client.runQuery(queryType, plan, listener);
- int rows = listener.await();
- System.out.println(String.format("Got %d record%s in %f seconds", rows, rows > 1 ? "s" : "", (float)watch.elapsed(TimeUnit.MILLISECONDS) / (float)1000));
- return 0;
}
private class QueryResultsListener implements UserResultsListener {
@@ -150,7 +148,7 @@ public class QuerySubmitter {
}
@Override
- public void resultArrived(QueryResultBatch result) {
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
int rows = result.getHeader().getRowCount();
if (result.getData() != null) {
count.addAndGet(rows);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
index 904afb5..080679b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
@@ -112,10 +112,7 @@ class MergeAdapter extends ClassVisitor {
return null;
}
if(arg3 != null){
- System.out.println("a: " + arg3);
arg3 = arg3.replace(set.precompiled.slash, set.generated.slash);
- System.out.println("b: " + arg3);
-
}
// if( (access & Modifier.PUBLIC) == 0){
// access = access ^ Modifier.PUBLIC ^ Modifier.PROTECTED | Modifier.PRIVATE;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 36433ad..cbd722c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -24,16 +24,18 @@ import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.visitors.SimpleExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.record.NullExpression;
import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorAccessible;
import com.google.common.collect.Lists;
-import org.apache.drill.exec.record.VectorAccessible;
-
public class ExpressionTreeMaterializer {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionTreeMaterializer.class);
@@ -45,7 +47,12 @@ public class ExpressionTreeMaterializer {
public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionImplementationRegistry registry) {
LogicalExpression materializedExpr = expr.accept(new MaterializeVisitor(batch, errorCollector), null);
- return ImplicitCastBuilder.injectImplicitCast(materializedExpr, errorCollector, registry);
+ LogicalExpression out = ImplicitCastBuilder.injectImplicitCast(materializedExpr, errorCollector, registry);
+ if(out instanceof NullExpression){
+ return new TypedNullConstant(Types.optional(MinorType.INT));
+ }else{
+ return out;
+ }
}
private static class MaterializeVisitor extends SimpleExprVisitor<LogicalExpression> {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index fd965a7..80a7819 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
+import net.hydromatic.optiq.SchemaPlus;
+
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.compile.ClassTransformer;
import org.apache.drill.exec.compile.QueryClassLoader;
@@ -75,6 +77,7 @@ public class FragmentContext implements Closeable {
this.connection = connection;
this.fragment = fragment;
this.funcRegistry = funcRegistry;
+ logger.debug("Getting initial memory allocation of {}", fragment.getMemInitial());
this.allocator = context.getAllocator().getChildAllocator(fragment.getHandle(), fragment.getMemInitial(), fragment.getMemMax());
}
@@ -91,6 +94,10 @@ public class FragmentContext implements Closeable {
public DrillbitContext getDrillbitContext() {
return context;
}
+
+ public SchemaPlus getRootSchema(){
+ return null;
+ }
/**
* Get this node's identity.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index 256e772..11658e6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -21,9 +21,11 @@ import java.util.Collection;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FunctionRegistry;
import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.planner.sql.DrillSchemaFactory;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.rpc.control.WorkEventBus;
@@ -81,4 +83,13 @@ public class QueryContext {
return workBus;
}
+ public DrillSchemaFactory getSchemaFactory(){
+ return drillbitContext.getSchemaFactory();
+ }
+
+ public FunctionRegistry getFunctionRegistry(){
+ return drillbitContext.getFunctionRegistry();
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index 6e58ec7..cd59428 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -24,7 +24,6 @@ import java.util.Collection;
import java.util.List;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.defs.OrderDef;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.LogicalExpression;
@@ -33,7 +32,6 @@ import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties;
import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.common.logical.data.*;
-import org.apache.drill.common.logical.data.Order.Direction;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
import org.apache.drill.common.types.TypeProtos;
@@ -50,6 +48,8 @@ import org.apache.drill.exec.physical.config.Sort;
import org.apache.drill.exec.physical.config.Limit;
import org.apache.drill.exec.physical.config.StreamingAggregate;
import org.apache.drill.exec.store.StorageEngine;
+import org.eigenbase.rel.RelFieldCollation.Direction;
+import org.eigenbase.rel.RelFieldCollation.NullDirection;
import com.beust.jcommander.internal.Lists;
@@ -123,9 +123,9 @@ public class BasicOptimizer extends Optimizer{
@Override
public PhysicalOperator visitOrder(Order order, Object value) throws OptimizerException {
PhysicalOperator input = order.getInput().accept(this, value);
- List<OrderDef> ods = Lists.newArrayList();
+ List<Ordering> ods = Lists.newArrayList();
for(Ordering o : order.getOrderings()){
- ods.add(OrderDef.create(o));
+ ods.add(o);
}
return new SelectionVectorRemover(new Sort(input, ods, false));
}
@@ -150,13 +150,13 @@ public class BasicOptimizer extends Optimizer{
}
// a collapsing aggregate is a currently implemented as a sort followed by a streaming aggregate.
- List<OrderDef> orderDefs = Lists.newArrayList();
+ List<Ordering> orderDefs = Lists.newArrayList();
List<NamedExpression> keys = Lists.newArrayList();
for(LogicalExpression e : segment.getExprs()){
if( !(e instanceof SchemaPath)) throw new OptimizerException("The basic optimizer doesn't currently support collapsing aggregate where the segment value is something other than a SchemaPath.");
keys.add(new NamedExpression(e, new FieldReference((SchemaPath) e)));
- orderDefs.add(new OrderDef(Direction.ASC, e));
+ orderDefs.add(new Ordering(Direction.Ascending, e, NullDirection.FIRST));
}
Sort sort = new Sort(segment.getInput().accept(this, value), orderDefs, false);
@@ -169,22 +169,22 @@ public class BasicOptimizer extends Optimizer{
@Override
public PhysicalOperator visitJoin(Join join, Object value) throws OptimizerException {
PhysicalOperator leftOp = join.getLeft().accept(this, value);
- List<OrderDef> leftOrderDefs = Lists.newArrayList();
+ List<Ordering> leftOrderDefs = Lists.newArrayList();
for(JoinCondition jc : join.getConditions()){
- leftOrderDefs.add(new OrderDef(Direction.ASC, jc.getLeft()));
+ leftOrderDefs.add(new Ordering(Direction.Ascending, jc.getLeft()));
}
leftOp = new Sort(leftOp, leftOrderDefs, false);
leftOp = new SelectionVectorRemover(leftOp);
PhysicalOperator rightOp = join.getRight().accept(this, value);
- List<OrderDef> rightOrderDefs = Lists.newArrayList();
+ List<Ordering> rightOrderDefs = Lists.newArrayList();
for(JoinCondition jc : join.getConditions()){
- rightOrderDefs.add(new OrderDef(Direction.ASC, jc.getRight()));
+ rightOrderDefs.add(new Ordering(Direction.Ascending, jc.getRight()));
}
rightOp = new Sort(rightOp, rightOrderDefs, false);
rightOp = new SelectionVectorRemover(rightOp);
- MergeJoinPOP mjp = new MergeJoinPOP(leftOp, rightOp, Arrays.asList(join.getConditions()), join.getJointType());
+ MergeJoinPOP mjp = new MergeJoinPOP(leftOp, rightOp, Arrays.asList(join.getConditions()), join.getJoinType());
return new SelectionVectorRemover(mjp);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
index 76916ea..fde88a9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
@@ -21,13 +21,14 @@ import java.util.Iterator;
import java.util.List;
import org.apache.drill.common.logical.data.Join;
-import org.apache.drill.common.logical.data.Join.JoinType;
import org.apache.drill.common.logical.data.JoinCondition;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.physical.base.Size;
+import org.eigenbase.rel.JoinRelType;
+import org.eigenbase.sql.SqlJoinOperator.JoinType;
import com.beust.jcommander.internal.Lists;
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -44,7 +45,7 @@ public class MergeJoinPOP extends AbstractBase{
private final PhysicalOperator left;
private final PhysicalOperator right;
private final List<JoinCondition> conditions;
- private final Join.JoinType joinType;
+ private final JoinRelType joinType;
@Override
public OperatorCost getCost() {
@@ -56,7 +57,7 @@ public class MergeJoinPOP extends AbstractBase{
@JsonProperty("left") PhysicalOperator left,
@JsonProperty("right") PhysicalOperator right,
@JsonProperty("join-conditions") List<JoinCondition> conditions,
- @JsonProperty("join-type") Join.JoinType joinType
+ @JsonProperty("join-type") JoinRelType joinType
) {
this.left = left;
this.right = right;
@@ -93,7 +94,7 @@ public class MergeJoinPOP extends AbstractBase{
return right;
}
- public Join.JoinType getJoinType() {
+ public JoinRelType getJoinType() {
return joinType;
}
@@ -102,12 +103,12 @@ public class MergeJoinPOP extends AbstractBase{
}
public MergeJoinPOP flipIfRight(){
- if(joinType == JoinType.RIGHT){
+ if(joinType == JoinRelType.RIGHT){
List<JoinCondition> flippedConditions = Lists.newArrayList(conditions.size());
for(JoinCondition c : conditions){
flippedConditions.add(c.flip());
}
- return new MergeJoinPOP(right, left, flippedConditions, JoinType.LEFT);
+ return new MergeJoinPOP(right, left, flippedConditions, JoinRelType.LEFT);
}else{
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java
index 667cc33..549c65c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergingReceiverPOP.java
@@ -17,18 +17,18 @@
*/
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.defs.OrderDef;
-import org.apache.drill.common.expression.LogicalExpression;
+import java.util.List;
+
+import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractReceiver;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
// The goal of this operator is to produce outgoing batches with records
// ordered according to the supplied expression. Each incoming batch
@@ -39,12 +39,12 @@ public class MergingReceiverPOP extends AbstractReceiver{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MergingReceiverPOP.class);
private final List<DrillbitEndpoint> senders;
- private final List<OrderDef> orderings;
+ private final List<Ordering> orderings;
@JsonCreator
public MergingReceiverPOP(@JsonProperty("sender-major-fragment") int oppositeMajorFragmentId,
@JsonProperty("senders") List<DrillbitEndpoint> senders,
- @JsonProperty("orderings") List<OrderDef> orderings) {
+ @JsonProperty("orderings") List<Ordering> orderings) {
super(oppositeMajorFragmentId);
this.senders = senders;
this.orderings = orderings;
@@ -78,7 +78,7 @@ public class MergingReceiverPOP extends AbstractReceiver{
return new Size(1,1);
}
- public List<OrderDef> getOrderings() {
+ public List<Ordering> getOrderings() {
return orderings;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java
index f0aa85b..c49509f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionExchange.java
@@ -17,27 +17,27 @@
*/
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.base.Preconditions;
-import org.apache.drill.common.defs.OrderDef;
+import java.util.List;
+
import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.physical.base.AbstractExchange;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.Receiver;
import org.apache.drill.exec.physical.base.Sender;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.Preconditions;
@JsonTypeName("ordered-partition-exchange")
public class OrderedPartitionExchange extends AbstractExchange {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OrderedPartitionExchange.class);
- private final List<OrderDef> orderings;
+ private final List<Ordering> orderings;
private final FieldReference ref;
private int recordsToSample = 10000; // How many records must be received before analyzing
private int samplingFactor = 10; // Will collect SAMPLING_FACTOR * number of partitions to send to distributed cache
@@ -48,7 +48,7 @@ public class OrderedPartitionExchange extends AbstractExchange {
private List<DrillbitEndpoint> receiverLocations;
@JsonCreator
- public OrderedPartitionExchange(@JsonProperty("orderings") List<OrderDef> orderings, @JsonProperty("ref") FieldReference ref,
+ public OrderedPartitionExchange(@JsonProperty("orderings") List<Ordering> orderings, @JsonProperty("ref") FieldReference ref,
@JsonProperty("child") PhysicalOperator child, @JsonProperty("recordsToSample") Integer recordsToSample,
@JsonProperty("samplingFactor") Integer samplingFactor, @JsonProperty("completionFactor") Float completionFactor) {
super(child);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
index de5cf04..55632a2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
@@ -17,24 +17,27 @@
*/
package org.apache.drill.exec.physical.config;
-import com.beust.jcommander.internal.Lists;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.defs.OrderDef;
+import java.util.List;
+
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.*;
-import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.physical.base.AbstractSender;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import java.util.List;
+import com.beust.jcommander.internal.Lists;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("OrderedPartitionSender")
public class OrderedPartitionSender extends AbstractSender {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OrderedPartitionSender.class);
- private final List<OrderDef> orderings;
+ private final List<Ordering> orderings;
private final FieldReference ref;
private final List<DrillbitEndpoint> endpoints;
private final int sendingWidth;
@@ -44,7 +47,7 @@ public class OrderedPartitionSender extends AbstractSender {
private float completionFactor;
@JsonCreator
- public OrderedPartitionSender(@JsonProperty("orderings") List<OrderDef> orderings, @JsonProperty("ref") FieldReference ref, @JsonProperty("child") PhysicalOperator child,
+ public OrderedPartitionSender(@JsonProperty("orderings") List<Ordering> orderings, @JsonProperty("ref") FieldReference ref, @JsonProperty("child") PhysicalOperator child,
@JsonProperty("destinations") List<DrillbitEndpoint> endpoints, @JsonProperty("receiver-major-fragment") int oppositeMajorFragmentId,
@JsonProperty("sending-fragment-width") int sendingWidth, @JsonProperty("recordsToSample") int recordsToSample,
@JsonProperty("samplingFactor") int samplingFactor, @JsonProperty("completionFactor") float completionFactor) {
@@ -70,7 +73,7 @@ public class OrderedPartitionSender extends AbstractSender {
return endpoints;
}
- public List<OrderDef> getOrderings() {
+ public List<Ordering> getOrderings() {
return orderings;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java
index fbeb6df..b8073c1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java
@@ -18,25 +18,25 @@
package org.apache.drill.exec.physical.config;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.defs.OrderDef;
+import java.util.List;
+
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
-import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.physical.base.AbstractExchange;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.Receiver;
import org.apache.drill.exec.physical.base.Sender;
import org.apache.drill.exec.proto.CoordinationProtos;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("single-merge-exchange")
public class SingleMergeExchange extends AbstractExchange {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SingleMergeExchange.class);
- private final List<OrderDef> orderExpr;
+ private final List<Ordering> orderExpr;
// ephemeral for setup tasks
private List<CoordinationProtos.DrillbitEndpoint> senderLocations;
@@ -44,7 +44,7 @@ public class SingleMergeExchange extends AbstractExchange {
@JsonCreator
public SingleMergeExchange(@JsonProperty("child") PhysicalOperator child,
- @JsonProperty("orderings") List<OrderDef> orderExpr) {
+ @JsonProperty("orderings") List<Ordering> orderExpr) {
super(child);
this.orderExpr = orderExpr;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
index 206e7cd..82aa830 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Sort.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.physical.config;
import java.util.List;
-import org.apache.drill.common.defs.OrderDef;
+import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractSingle;
import org.apache.drill.exec.physical.base.PhysicalOperator;
@@ -27,7 +27,6 @@ import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.physical.base.Size;
import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
@@ -35,17 +34,17 @@ import com.fasterxml.jackson.annotation.JsonTypeName;
public class Sort extends AbstractSingle{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Sort.class);
- private final List<OrderDef> orderings;
+ private final List<Ordering> orderings;
private boolean reverse = false;
@JsonCreator
- public Sort(@JsonProperty("child") PhysicalOperator child, @JsonProperty("orderings") List<OrderDef> orderings, @JsonProperty("reverse") boolean reverse) {
+ public Sort(@JsonProperty("child") PhysicalOperator child, @JsonProperty("orderings") List<Ordering> orderings, @JsonProperty("reverse") boolean reverse) {
super(child);
this.orderings = orderings;
this.reverse = reverse;
}
- public List<OrderDef> getOrderings() {
+ public List<Ordering> getOrderings() {
return orderings;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
index aae1a3c..7c8a51c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
@@ -24,6 +24,7 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.record.VectorContainer;
+import org.eigenbase.rel.JoinRelType;
/**
* This join template uses a merge join to combine two ordered streams into a single larger batch. When joining
@@ -91,7 +92,7 @@ public abstract class JoinTemplate implements JoinWorker {
// validate input iterators (advancing to the next record batch if necessary)
if (!status.isRightPositionAllowed()) {
- if (((MergeJoinPOP)status.outputBatch.getPopConfig()).getJoinType() == Join.JoinType.LEFT) {
+ if (((MergeJoinPOP)status.outputBatch.getPopConfig()).getJoinType() == JoinRelType.LEFT) {
// we've hit the end of the right record batch; copy any remaining values from the left batch
while (status.isLeftPositionAllowed()) {
if (!doCopyLeft(status.getLeftPosition(), status.fetchAndIncOutputPos()))
@@ -109,7 +110,7 @@ public abstract class JoinTemplate implements JoinWorker {
case -1:
// left key < right key
- if (((MergeJoinPOP)status.outputBatch.getPopConfig()).getJoinType() == Join.JoinType.LEFT)
+ if (((MergeJoinPOP)status.outputBatch.getPopConfig()).getJoinType() == JoinRelType.LEFT)
if (!doCopyLeft(status.getLeftPosition(), status.fetchAndIncOutputPos())) {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 298b031..bd668e7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -49,6 +49,7 @@ import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
+import org.eigenbase.rel.JoinRelType;
import com.google.common.collect.ImmutableList;
import com.sun.codemodel.JClass;
@@ -112,7 +113,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
private final RecordBatch right;
private final JoinStatus status;
private final JoinCondition condition;
- private final Join.JoinType joinType;
+ private final JoinRelType joinType;
private JoinWorker worker;
public MergeJoinBatchBuilder batchBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java
index 9428b46..3549a33 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java
@@ -17,16 +17,16 @@
*/
package org.apache.drill.exec.physical.impl.join;
-import com.google.common.base.Preconditions;
+import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.logical.data.Join.JoinType;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.impl.BatchCreator;
import org.apache.drill.exec.record.RecordBatch;
+import org.eigenbase.rel.JoinRelType;
-import java.util.List;
+import com.google.common.base.Preconditions;
public class MergeJoinCreator implements BatchCreator<MergeJoinPOP> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MergeJoinCreator.class);
@@ -34,7 +34,7 @@ public class MergeJoinCreator implements BatchCreator<MergeJoinPOP> {
@Override
public RecordBatch getBatch(FragmentContext context, MergeJoinPOP config, List<RecordBatch> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.size() == 2);
- if(config.getJoinType() == JoinType.RIGHT){
+ if(config.getJoinType() == JoinRelType.RIGHT){
return new MergeJoinBatch(config.flipIfRight(), context, children.get(1), children.get(0));
}else{
return new MergeJoinBatch(config, context, children.get(0), children.get(1));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index 43d3b45..72f1ad9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -25,14 +25,13 @@ import java.util.Iterator;
import java.util.List;
import java.util.PriorityQueue;
-import org.apache.drill.common.defs.OrderDef;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.Order.Direction;
+import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.exception.SchemaChangeException;
@@ -60,6 +59,7 @@ import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.allocator.VectorAllocator;
+import org.eigenbase.rel.RelFieldCollation.Direction;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -367,7 +367,7 @@ public class MergingRecordBatch implements RecordBatch {
private MergingReceiverGeneratorBase createMerger() throws SchemaChangeException {
// set up the expression evaluator and code generation
- final List<OrderDef> orderings = config.getOrderings();
+ final List<Ordering> orderings = config.getOrderings();
final ErrorCollector collector = new ErrorCollectorImpl();
final ClassGenerator<MergingReceiverGeneratorBase> cg =
CodeGenerator.getRoot(MergingReceiverGeneratorBase.TEMPLATE_DEFINITION, context.getFunctionRegistry());
@@ -550,11 +550,11 @@ public class MergingRecordBatch implements RecordBatch {
// generate less than/greater than checks (fixing results for ASCending vs. DESCending)
cg.getEvalBlock()._if(out.getValue().eq(JExpr.lit(1)))
._then()
- ._return(JExpr.lit(config.getOrderings().get(comparisonVectorIndex).getDirection() == Direction.ASC ? 1 : -1));
+ ._return(JExpr.lit(config.getOrderings().get(comparisonVectorIndex).getDirection() == Direction.Ascending ? 1 : -1));
cg.getEvalBlock()._if(out.getValue().eq(JExpr.lit(-1)))
._then()
- ._return(JExpr.lit(config.getOrderings().get(comparisonVectorIndex).getDirection() == Direction.ASC ? -1 : 1));
+ ._return(JExpr.lit(config.getOrderings().get(comparisonVectorIndex).getDirection() == Direction.Ascending ? -1 : 1));
++comparisonVectorIndex;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index da8978f..381fbe2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -23,7 +23,6 @@ import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
-import org.apache.drill.common.defs.OrderDef;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
@@ -32,6 +31,7 @@ import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.cache.Counter;
@@ -69,6 +69,7 @@ import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.IntVector;
import org.apache.drill.exec.vector.ValueVector;
+import org.eigenbase.rel.RelFieldCollation.Direction;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -189,7 +190,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
// Project every Nth record to a new vector container, where N = recordsSampled/(samplingFactor * partitions).
// Uses the
- // the expressions from the OrderDefs to populate each column. There is one column for each OrderDef in
+ // the expressions from the Orderings to populate each column. There is one column for each Ordering in
// popConfig.orderings.
VectorContainer containerToCache = new VectorContainer();
@@ -293,11 +294,11 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
VectorContainer allSamplesContainer = new VectorContainer();
containerBuilder.build(context, allSamplesContainer);
- List<OrderDef> orderDefs = Lists.newArrayList();
+ List<Ordering> orderDefs = Lists.newArrayList();
int i = 0;
- for (OrderDef od : popConfig.getOrderings()) {
+ for (Ordering od : popConfig.getOrderings()) {
SchemaPath sp = new SchemaPath("f" + i++, ExpressionPosition.UNKNOWN);
- orderDefs.add(new OrderDef(od.getDirection(), new FieldReference(sp)));
+ orderDefs.add(new Ordering(od.getDirection(), new FieldReference(sp)));
}
// sort the data incoming samples.
@@ -330,8 +331,8 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
/**
* Creates a copier that does a project for every Nth record from a VectorContainer incoming into VectorContainer
- * outgoing. Each OrderDef in orderings generates a column, and evaluation of the expression associated with each
- * OrderDef determines the value of each column. These records will later be sorted based on the values in each
+ * outgoing. Each Ordering in orderings generates a column, and evaluation of the expression associated with each
+ * Ordering determines the value of each column. These records will later be sorted based on the values in each
* column, in the same order as the orderings.
*
* @param sv4
@@ -342,14 +343,14 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
* @throws SchemaChangeException
*/
private SampleCopier getCopier(SelectionVector4 sv4, VectorContainer incoming, VectorContainer outgoing,
- List<OrderDef> orderings) throws SchemaChangeException {
+ List<Ordering> orderings) throws SchemaChangeException {
List<ValueVector> localAllocationVectors = Lists.newArrayList();
final ErrorCollector collector = new ErrorCollectorImpl();
final ClassGenerator<SampleCopier> cg = CodeGenerator.getRoot(SampleCopier.TEMPLATE_DEFINITION,
context.getFunctionRegistry());
int i = 0;
- for (OrderDef od : orderings) {
+ for (Ordering od : orderings) {
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), incoming, collector, context.getFunctionRegistry());
SchemaPath schemaPath = new SchemaPath("f" + i++, ExpressionPosition.UNKNOWN);
TypeProtos.MajorType.Builder builder = TypeProtos.MajorType.newBuilder().mergeFrom(expr.getMajorType())
@@ -521,7 +522,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
cg.setMappingSet(mainMapping);
int count = 0;
- for (OrderDef od : popConfig.getOrderings()) {
+ for (Ordering od : popConfig.getOrderings()) {
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector, context.getFunctionRegistry());
if (collector.hasErrors())
throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
@@ -538,7 +539,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
ClassGenerator.HoldingContainer out = cg.addExpr(f, false);
JConditional jc = cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
- if (od.getDirection() == Order.Direction.ASC) {
+ if (od.getDirection() == Direction.Ascending) {
jc._then()._return(out.getValue());
} else {
jc._then()._return(out.getValue().minus());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 3b8154e..509d13b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -88,6 +88,18 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
}
return ref;
}
+ private boolean isWildcard(NamedExpression ex){
+ LogicalExpression expr = ex.getExpr();
+ LogicalExpression ref = ex.getRef();
+ if(expr instanceof SchemaPath && ref instanceof SchemaPath){
+ PathSegment e = ((SchemaPath) expr).getRootSegment();
+ PathSegment n = ((SchemaPath) ref).getRootSegment();
+ if(e.isNamed() && e.getNameSegment().getPath().equals("*") && n.isNamed() && n.getChild() != null && n.getChild().isNamed() && n.getChild().getNameSegment().getPath().equals("*")){
+ return true;
+ }
+ }
+ return false;
+ }
@Override
protected void setupNewSchema() throws SchemaChangeException{
@@ -99,33 +111,43 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
final ClassGenerator<Projector> cg = CodeGenerator.getRoot(Projector.TEMPLATE_DEFINITION, context.getFunctionRegistry());
- for(int i = 0; i < exprs.size(); i++){
- final NamedExpression namedExpression = exprs.get(i);
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry());
- final MaterializedField outputField = MaterializedField.create(getRef(namedExpression), expr.getMajorType());
- if(collector.hasErrors()){
- throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
- }
-
- // add value vector to transfer if direct reference and this is allowed, otherwise, add to evaluation stack.
- if(expr instanceof ValueVectorReadExpression && incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE){
- ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
- ValueVector vvIn = incoming.getValueAccessorById(vectorRead.getFieldId().getFieldId(), TypeHelper.getValueVectorClass(vectorRead.getMajorType().getMinorType(), vectorRead.getMajorType().getMode())).getValueVector();
- Preconditions.checkNotNull(incoming);
-
- TransferPair tp = vvIn.getTransferPair(getRef(namedExpression));
+ if(exprs.size() == 1 && isWildcard(exprs.get(0))){
+ for(VectorWrapper<?> wrapper : incoming){
+ ValueVector vvIn = wrapper.getValueVector();
+ TransferPair tp = wrapper.getValueVector().getTransferPair(new FieldReference(vvIn.getField().getName()));
transfers.add(tp);
container.add(tp.getTo());
- logger.debug("Added transfer.");
- }else{
- // need to do evaluation.
- ValueVector vector = TypeHelper.getNewVector(outputField, context.getAllocator());
- allocationVectors.add(vector);
- TypedFieldId fid = container.add(vector);
- ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr);
- cg.addExpr(write);
- logger.debug("Added eval.");
}
+ }else{
+ for(int i = 0; i < exprs.size(); i++){
+ final NamedExpression namedExpression = exprs.get(i);
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry());
+ final MaterializedField outputField = MaterializedField.create(getRef(namedExpression), expr.getMajorType());
+ if(collector.hasErrors()){
+ throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
+ }
+
+ // add value vector to transfer if direct reference and this is allowed, otherwise, add to evaluation stack.
+ if(expr instanceof ValueVectorReadExpression && incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE){
+ ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
+ ValueVector vvIn = incoming.getValueAccessorById(vectorRead.getFieldId().getFieldId(), TypeHelper.getValueVectorClass(vectorRead.getMajorType().getMinorType(), vectorRead.getMajorType().getMode())).getValueVector();
+ Preconditions.checkNotNull(incoming);
+
+ TransferPair tp = vvIn.getTransferPair(getRef(namedExpression));
+ transfers.add(tp);
+ container.add(tp.getTo());
+ logger.debug("Added transfer.");
+ }else{
+ // need to do evaluation.
+ ValueVector vector = TypeHelper.getNewVector(outputField, context.getAllocator());
+ allocationVectors.add(vector);
+ TypedFieldId fid = container.add(vector);
+ ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr);
+ cg.addExpr(write);
+ logger.debug("Added eval.");
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
index 4d04735..5fd12c0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
@@ -20,13 +20,11 @@ package org.apache.drill.exec.physical.impl.sort;
import java.io.IOException;
import java.util.List;
-import org.apache.drill.common.defs.OrderDef;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.Order.Direction;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
@@ -46,6 +44,7 @@ import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.eigenbase.rel.RelFieldCollation.Direction;
import com.google.common.collect.ImmutableList;
import com.sun.codemodel.JConditional;
@@ -165,20 +164,20 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
return createNewSorter(this.context, this.popConfig.getOrderings(), this, MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
}
- public static Sorter createNewSorter(FragmentContext context, List<OrderDef> orderings, VectorAccessible batch) throws ClassTransformationException, IOException, SchemaChangeException {
+ public static Sorter createNewSorter(FragmentContext context, List<Ordering> orderings, VectorAccessible batch) throws ClassTransformationException, IOException, SchemaChangeException {
final MappingSet mainMapping = new MappingSet( (String) null, null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
final MappingSet leftMapping = new MappingSet("leftIndex", null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
final MappingSet rightMapping = new MappingSet("rightIndex", null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
return createNewSorter(context, orderings, batch, mainMapping, leftMapping, rightMapping);
}
- public static Sorter createNewSorter(FragmentContext context, List<OrderDef> orderings, VectorAccessible batch, MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping)
+ public static Sorter createNewSorter(FragmentContext context, List<Ordering> orderings, VectorAccessible batch, MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping)
throws ClassTransformationException, IOException, SchemaChangeException{
CodeGenerator<Sorter> cg = CodeGenerator.get(Sorter.TEMPLATE_DEFINITION, context.getFunctionRegistry());
ClassGenerator<Sorter> g = cg.getRoot();
g.setMappingSet(mainMapping);
- for(OrderDef od : orderings){
+ for(Ordering od : orderings){
// first, we rewrite the evaluation stack for each side of the comparison.
ErrorCollector collector = new ErrorCollectorImpl();
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector,context.getFunctionRegistry());
@@ -194,7 +193,7 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
HoldingContainer out = g.addExpr(f, false);
JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
- if(od.getDirection() == Direction.ASC){
+ if(od.getDirection() == Direction.Ascending){
jc._then()._return(out.getValue());
}else{
jc._then()._return(out.getValue().minus());
[05/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
deleted file mode 100644
index 15ca261..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import java.lang.reflect.Method;
-import java.util.Arrays;
-import java.util.List;
-
-import net.hydromatic.linq4j.expressions.BlockBuilder;
-import net.hydromatic.linq4j.expressions.BlockStatement;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.linq4j.expressions.Expressions;
-import net.hydromatic.linq4j.function.Function1;
-import net.hydromatic.linq4j.function.Functions;
-import net.hydromatic.optiq.DataContext;
-import net.hydromatic.optiq.impl.java.JavaTypeFactory;
-import net.hydromatic.optiq.rules.java.EnumerableConvention;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.JavaRowFormat;
-import net.hydromatic.optiq.rules.java.PhysType;
-import net.hydromatic.optiq.rules.java.PhysTypeImpl;
-
-import org.apache.drill.common.util.Hook;
-import org.apache.drill.exec.client.DrillClient;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.SingleRel;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelTraitSet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Relational expression that converts from Drill to Enumerable. At runtime it executes a Drill query and returns the
- * results as an {@link net.hydromatic.linq4j.Enumerable}.
- */
-public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
- private static final Logger logger = LoggerFactory.getLogger(EnumerableDrillRel.class);
-
- private static final Function1<String, Expression> TO_LITERAL = new Function1<String, Expression>() {
- @Override
- public Expression apply(String a0) {
- return Expressions.constant(a0);
- }
- };
-
- private static final Method OF_METHOD;
-
- private PhysType physType;
- private DrillClient client;
-
- static {
- try {
- OF_METHOD = EnumerableDrillFullEngine.class.getMethod("of", String.class, List.class, Class.class,
- DataContext.class);
- // EnumerableDrillFullEngine.class.getMethod("of", String.class, List.class, Class.class);
- } catch (NoSuchMethodException e) {
- logger.error("Exception during query", e);
- throw new RuntimeException(e);
- }
- }
-
- public EnumerableDrillRel(DrillClient client, RelOptCluster cluster, RelTraitSet traitSet, RelNode input) {
- super(cluster, traitSet, input);
- assert getConvention() instanceof EnumerableConvention;
- assert input.getConvention() == DrillRel.CONVENTION;
- this.client = client;
- physType = PhysTypeImpl.of((JavaTypeFactory) cluster.getTypeFactory(), input.getRowType(), JavaRowFormat.ARRAY);
- }
-
- public PhysType getPhysType() {
- return physType;
- }
-
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(.1);
- }
-
- @Override
- public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- return new EnumerableDrillRel(client, getCluster(), traitSet, sole(inputs));
- }
-
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- logger.debug("implementing enumerable");
-
- final DrillImplementor drillImplementor = new DrillImplementor(this.client == null);
- DrillRel input = (DrillRel) getChild();
-
- drillImplementor.go(input);
- String plan = drillImplementor.getJsonString();
- Hook.LOGICAL_PLAN.run(plan);
-
- // not quite sure where this list was supposed to be set earlier, leaving it null got me back the full result set
-
- final List<String> fieldNameList = rowType.getFieldNames();
- // final List<String> fieldNameList = null;
- BlockStatement expr = new BlockBuilder().append(
- Expressions.call(
- OF_METHOD,
- Expressions.constant(plan), //
- Expressions.call(Arrays.class, "asList", //
- Expressions.newArrayInit(String.class, Functions.apply(fieldNameList, TO_LITERAL)) //
- ),
- Expressions.constant(Object.class), //
- Expressions.variable(DataContext.class, "root"))).toBlock();
- final PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(),
- pref.prefer(JavaRowFormat.ARRAY));
- return new Result(expr, physType, JavaRowFormat.ARRAY);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java
deleted file mode 100644
index f35d69d..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import net.hydromatic.optiq.rules.java.EnumerableConvention;
-
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.jdbc.DrillHandler;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.convert.ConverterRule;
-
-/**
- * Rule that converts any Drill relational expression to enumerable format by adding a {@link EnumerableDrillRel}.
- */
-public class EnumerableDrillRule extends ConverterRule {
-
- private final DrillClient client;
-
- public static EnumerableDrillRule getInstance(DrillClient client){
- return new EnumerableDrillRule(EnumerableConvention.INSTANCE, client);
- }
-
- private EnumerableDrillRule(EnumerableConvention outConvention, DrillClient client) {
- super(RelNode.class, DrillRel.CONVENTION, outConvention, "EnumerableDrillRule." + outConvention);
- this.client = client;
- }
-
- @Override
- public boolean isGuaranteed() {
- return true;
- }
-
- @Override
- public RelNode convert(RelNode rel) {
- assert rel.getTraitSet().contains(DrillRel.CONVENTION);
- return new EnumerableDrillRel(client, rel.getCluster(), rel.getTraitSet().replace(getOutConvention()), rel);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchListener.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchListener.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchListener.java
deleted file mode 100644
index 30a31b2..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchListener.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-
-public class BatchListener implements UserResultsListener {
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BatchListener.class);
-
- private volatile RpcException ex;
- private volatile boolean completed = false;
-
- final BlockingQueue<QueryResultBatch> queue = new ArrayBlockingQueue<>(100);
-
- @Override
- public void submissionFailed(RpcException ex) {
- this.ex = ex;
- completed = true;
- System.out.println("Query failed: " + ex);
- }
-
- @Override
- public void resultArrived(QueryResultBatch result) {
- logger.debug("Result arrived {}", result);
- queue.add(result);
- if(result.getHeader().getIsLastChunk()){
- completed = true;
- }
- if (result.getHeader().getErrorCount() > 0) {
- submissionFailed(new RpcException(String.format("%s", result.getHeader().getErrorList())));
- }
- }
-
- public boolean completed(){
- return completed;
- }
-
- public QueryResultBatch getNext() throws RpcException, InterruptedException{
- while(true){
- if(ex != null) throw ex;
- if(completed && queue.isEmpty()){
- return null;
- }else{
- QueryResultBatch q = queue.poll(50, TimeUnit.MILLISECONDS);
- if(q != null) return q;
- }
-
- }
- }
-
- @Override
- public void queryIdArrived(QueryId queryId) {
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
deleted file mode 100644
index 1b58805..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.vector.NullableVarBinaryVector;
-import org.apache.drill.exec.vector.NullableVarCharVector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VarBinaryVector;
-import org.apache.drill.exec.vector.VarCharVector;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-import com.google.common.collect.Maps;
-
-@JsonSerialize(using = BatchLoaderMap.Se.class)
-public class BatchLoaderMap implements Map<String, Object> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BatchLoaderMap.class);
-
- private BatchListener listener;
- private RecordBatchLoader loader;
- private final List<String> requestedFields;
- private final Map<String, ValueVector> fields = Maps.newHashMap();
- private int index;
- private Object[] objArr;
- private JsonHelper helper;
- private boolean loaded;
-
- public BatchLoaderMap(List<String> requestedFields, BatchListener listener, DrillClient client) {
- this.listener = listener;
- this.requestedFields = requestedFields;
- this.objArr = new Object[requestedFields.size()];
- this.loader = new RecordBatchLoader(client.getAllocator());
- this.helper = new JsonHelper(client.getConfig());
- }
-
- private void load(QueryResultBatch batch) throws SchemaChangeException {
- boolean schemaChanged = loader.load(batch.getHeader().getDef(), batch.getData());
- if (schemaChanged) {
- fields.clear();
- for (VectorWrapper<?> v : loader) {
- fields.put(v.getField().getName(), v.getValueVector());
- }
- } else {
- logger.debug("Schema didn't change. {}", batch);
- }
- }
-
- public boolean next() throws SchemaChangeException, RpcException, InterruptedException {
- index++;
- if (index < loader.getRecordCount()) {
- return true;
- } else {
- logger.debug("Starting next query result batch.");
- QueryResultBatch qrb;
- while( (qrb = listener.getNext()) != null && !qrb.hasData()){
- qrb = listener.getNext();
- }
-
- if (qrb == null) {
- logger.debug("No more batches found.");
- index = -1;
- return false;
- } else {
- load(qrb);
- logger.debug("New batch found and loaded. {}", qrb.getHeader().getDef());
- index = 0;
- return true;
- }
-
- }
- }
-
-
-
- public Object getCurrentObject() {
- if(requestedFields.size() == 1){
- if(requestedFields.iterator().next().equals("_MAP")){
- return helper.get(fields, index);
- }else{
- return getObj(fields.get(requestedFields.get(0)));
- }
-
- }
-
- for (int i = 0; i < requestedFields.size(); i++) {
- ValueVector vv = fields.get(requestedFields.get(i));
- if (vv == null) {
- logger.debug("Failure while retrieving field of {}. Didn't exist in returned set of {}", requestedFields.get(i), fields.keySet());
- objArr[i] = null;
- } else {
- objArr[i] = getObj(vv);
- }
- }
- return objArr;
-
- }
-
- private Object getObj(ValueVector vv){
- Object val = vv.getAccessor().getObject(index);
- if(val == null){
- return null;
- }else if(vv instanceof VarBinaryVector || vv instanceof VarCharVector || vv instanceof NullableVarBinaryVector || vv instanceof NullableVarCharVector){
- return new String( (byte[]) val );
- }else{
- return val;
- }
- }
-
- @Override
- public int size() {
- return fields.size();
- }
-
- @Override
- public boolean isEmpty() {
- return fields.isEmpty();
- }
-
- @Override
- public boolean containsKey(Object key) {
- if (!(key instanceof String))
- return false;
- return fields.containsKey(key);
- }
-
- @Override
- public boolean containsValue(Object value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object get(Object key) {
- ValueVector v = fields.get(key);
- Preconditions.checkNotNull(v);
- return v.getAccessor().getObject(index);
- }
-
- @Override
- public Object put(String key, Object value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object remove(Object key) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void putAll(Map<? extends String, ? extends Object> m) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void clear() {
- }
-
- @Override
- public Set<String> keySet() {
- return fields.keySet();
- }
-
- @Override
- public Collection<Object> values() {
- throw new UnsupportedOperationException();
- }
-
- public String toString() {
- try {
- return new ObjectMapper().writeValueAsString(this);
- } catch (JsonProcessingException e) {
- logger.error("Exception during query", e);
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Set<java.util.Map.Entry<String, Object>> entrySet() {
- throw new UnsupportedOperationException();
- }
-
- public static class Se extends StdSerializer<BatchLoaderMap> {
-
- public Se() {
- super(BatchLoaderMap.class);
- }
-
- @Override
- public void serialize(BatchLoaderMap value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
- JsonGenerationException {
- jgen.writeStartObject();
- assert value.index > -1 && value.index < value.loader.getRecordCount();
- for (Map.Entry<String, ValueVector> me : value.fields.entrySet()) {
- jgen.writeObjectField(me.getKey(), me.getValue().getAccessor().getObject(value.index));
- }
- jgen.writeEndObject();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
deleted file mode 100644
index f9f3f33..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.util.List;
-
-import net.hydromatic.linq4j.Enumerator;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.jdbc.DrillTable;
-
-public class DrillFullImpl<E>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFullImpl.class);
-
- private final String plan;
- final DrillConfig config;
- private final List<String> fields;
-
-
- public DrillFullImpl(String plan, DrillConfig config, List<String> fields) {
- super();
- this.plan = plan;
- this.config = config;
- this.fields = fields;
- }
-
- @SuppressWarnings("unchecked")
- public Enumerator<E> enumerator(DrillClient client) {
-
- BatchListener listener = new BatchListener();
-
- // TODO: use a completion service from the container
- QueryRequestRunner runner = new QueryRequestRunner(plan, client, listener);
- runner.start();
-
- return (Enumerator<E>) new ResultEnumerator(listener, client, fields);
-
- }
-
- public class QueryRequestRunner extends Thread{
- final String plan;
- final DrillClient client;
- final BatchListener listener;
-
- public QueryRequestRunner(String plan, DrillClient client, BatchListener listener) {
- super();
- this.setDaemon(true);
- this.plan = plan;
- this.client = client;
- this.listener = listener;
- }
-
- @Override
- public void run() {
- client.runQuery(UserProtos.QueryType.LOGICAL, plan, listener);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillMutableSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillMutableSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillMutableSchema.java
deleted file mode 100644
index dbef38e..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillMutableSchema.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.util.Collection;
-import java.util.Map;
-
-import com.google.common.collect.Multimap;
-
-import net.hydromatic.linq4j.QueryProvider;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.optiq.MutableSchema;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.Table;
-import net.hydromatic.optiq.impl.java.JavaTypeFactory;
-
-public class DrillMutableSchema implements Schema {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMutableSchema.class);
-
- @Override
- public Schema getParentSchema() {
- return null;
- }
-
- @Override
- public String getName() {
- return null;
- }
-
- @Override
- public Collection<TableFunctionInSchema> getTableFunctions(String name) {
- return null;
- }
-
- @Override
- public <E> Table<E> getTable(String name, Class<E> elementType) {
- return null;
- }
-
- @Override
- public Expression getExpression() {
- return null;
- }
-
- @Override
- public QueryProvider getQueryProvider() {
- return null;
- }
-
- @Override
- public Multimap<String, TableFunctionInSchema> getTableFunctions() {
- return null;
- }
-
- @Override
- public Collection<String> getSubSchemaNames() {
- return null;
- }
-
- @Override
- public Map<String, TableInSchema> getTables() {
- return null;
- }
-
- @Override
- public Schema getSubSchema(String name) {
- return null;
- }
-
- @Override
- public JavaTypeFactory getTypeFactory() {
- return null;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
deleted file mode 100644
index 1650aaf..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-
-import net.hydromatic.linq4j.QueryProvider;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.Table;
-import net.hydromatic.optiq.impl.java.JavaTypeFactory;
-
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.drill.jdbc.DrillTable;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-
-public class FileSystemSchema implements Schema{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchema.class);
-
- private ConcurrentMap<String, TableInSchema> tables = Maps.newConcurrentMap();
-
- private final JavaTypeFactory typeFactory;
- private final Schema parentSchema;
- private final String name;
- private final Expression expression;
- private final QueryProvider queryProvider;
- private final SchemaProvider schemaProvider;
- private final DrillClient client;
- private final StorageEngineConfig config;
-
- public FileSystemSchema(DrillClient client, StorageEngineConfig config, SchemaProvider schemaProvider, JavaTypeFactory typeFactory, Schema parentSchema, String name, Expression expression,
- QueryProvider queryProvider) {
- super();
- this.client = client;
- this.typeFactory = typeFactory;
- this.parentSchema = parentSchema;
- this.name = name;
- this.expression = expression;
- this.queryProvider = queryProvider;
- this.schemaProvider = schemaProvider;
- this.config = config;
- }
-
- @Override
- public Schema getSubSchema(String name) {
- return null;
- }
-
- @Override
- public JavaTypeFactory getTypeFactory() {
- return typeFactory;
- }
-
- @Override
- public Schema getParentSchema() {
- return parentSchema;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Expression getExpression() {
- return expression;
- }
-
- @Override
- public QueryProvider getQueryProvider() {
- return queryProvider;
- }
-
-
- @Override
- public Collection<TableFunctionInSchema> getTableFunctions(String name) {
- return Collections.emptyList();
- }
-
- @Override
- public Multimap<String, TableFunctionInSchema> getTableFunctions() {
- return ArrayListMultimap.create();
- }
-
- @Override
- public Collection<String> getSubSchemaNames() {
- return Collections.EMPTY_LIST;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <E> Table<E> getTable(String name, Class<E> elementType) {
- if( !elementType.isAssignableFrom(DrillTable.class)) throw new UnsupportedOperationException();
- TableInfo info = (TableInfo) tables.get(name);
- if(info != null) return (Table<E>) info.table;
- Object selection = schemaProvider.getSelectionBaseOnName(name);
- if(selection == null) return null;
-
- DrillTable table = DrillTable.createTable(client, typeFactory, this, name, null, config, selection);
- info = new TableInfo(name, table);
- TableInfo oldInfo = (TableInfo) tables.putIfAbsent(name, info);
- if(oldInfo != null) return (Table<E>) oldInfo.table;
- return (Table<E>) table;
- }
-
- @Override
- public Map<String, TableInSchema> getTables() {
- return this.tables;
- }
-
- private class TableInfo extends TableInSchema{
-
- final DrillTable table;
-
- public TableInfo(String name, DrillTable table) {
- super(FileSystemSchema.this, name, TableType.TABLE);
- this.table = table;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <E> Table<E> getTable(Class<E> elementType) {
- if( !elementType.isAssignableFrom(DrillTable.class)) throw new UnsupportedOperationException();
- return (Table<E>) table;
- }
-
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
deleted file mode 100644
index 28868f7..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.util.Map;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.vector.ValueVector;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-public class JsonHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonHelper.class);
-
- private ObjectMapper mapper;
-
- JsonHelper(DrillConfig config){
- mapper = config.getMapper();
- }
-
- public JsonNode get(Map<String, ValueVector> vectors, int index){
- final ObjectNode map = mapper.createObjectNode();
- for(Map.Entry<String, ValueVector> e : vectors.entrySet()){
- Object o = e.getValue().getAccessor().getObject(index);
- if(o == null) continue;
- String k = e.getKey().replace("_MAP.", "");
-
- if(o instanceof Integer) map.put(k, (Integer) o);
- else if(o instanceof Long) map.put(k, (Long) o);
- else if(o instanceof Boolean) map.put(k, (Boolean) o);
- else if(o instanceof String) map.put(k, (String) o);
- else if(o instanceof byte[]) map.put(k, new String((byte[]) o));
- else if(o instanceof Float) map.put(k, (Float) o);
- else if(o instanceof Double) map.put(k, (Double) o);
- else map.put(k, "---Unknown Type---");
-
- }
- return map;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
deleted file mode 100644
index 0222254..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.full;
-
-import java.util.List;
-
-import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.optiq.DataContext;
-
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.rpc.RpcException;
-
-/**
- * Enumerator used for full execution engine.
- */
- class ResultEnumerator implements Enumerator<Object> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ResultEnumerator.class);
-
- private final BatchLoaderMap loaderMap;
- private Object current;
-
- public ResultEnumerator(BatchListener listener, DrillClient client, List<String> fields) {
- this.loaderMap = new BatchLoaderMap(fields, listener, client);
- }
-
- public Object current() {
- return current;
- }
-
- public boolean moveNext() {
-
- try {
- boolean succ = loaderMap.next();
- if(succ){
- current = loaderMap.getCurrentObject();
- }
- return succ;
-
- } catch (InterruptedException e) {
- Thread.interrupted();
- logger.error("Exception during query", e);
- throw new RuntimeException(e);
- } catch (RpcException | SchemaChangeException e) {
- logger.error("Exception during query", e);
- throw new RuntimeException(e);
- }
- }
-
- public void reset() {
- throw new UnsupportedOperationException();
- }
-
- public void close(){
-
- }
- }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java b/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
deleted file mode 100644
index f33b54b..0000000
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.sql.client.ref;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import net.hydromatic.linq4j.Enumerator;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.ReferenceInterpreter;
-import org.apache.drill.exec.ref.RunOutcome;
-import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
-import org.apache.drill.exec.ref.rse.RSERegistry;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Lists;
-
-public class DrillRefImpl<E> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRefImpl.class);
-
- private static final ObjectMapper mapper = createMapper();
-
- private final String plan;
- final BlockingQueue<Object> queue;
- final DrillConfig config;
- private final List<String> fields;
-
- public DrillRefImpl(String plan, DrillConfig config, List<String> fields, BlockingQueue<Object> queue) {
- super();
- this.plan = plan;
- this.config = config;
- this.fields = fields;
- this.queue = queue;
- }
-
- private static ObjectMapper createMapper() {
- return new ObjectMapper();
- }
-
- /**
- * Enumerator used for reference interpreter
- */
- private static class JsonEnumerator implements Enumerator {
- private final BlockingQueue<Object> queue;
- private final String holder;
- private final List<String> fields;
- private Object current;
- private Future<Collection<RunOutcome>> futureOutcomes;
-
- public JsonEnumerator(Future<Collection<RunOutcome>> futureOutcomes, BlockingQueue<Object> queue,
- List<String> fields) {
- this.futureOutcomes = futureOutcomes;
- this.queue = queue;
- this.holder = null;
- this.fields = fields;
- }
-
- public void close() {
-
- }
-
- public Object current() {
- return current;
- }
-
- public boolean moveNext() {
- try {
- Object o = queue.take();
- if (o instanceof RunOutcome.OutcomeType) {
- switch ((RunOutcome.OutcomeType) o) {
- case SUCCESS:
- return false; // end of data
- case CANCELED:
- throw new RuntimeException("canceled");
- case FAILED:
- default:
- try {
- Collection<RunOutcome> outcomes = this.futureOutcomes.get();
- List<RunOutcome> l = Lists.newArrayList(outcomes);
- for (int i = 1; i < outcomes.size(); i++) {
- RunOutcome out = l.get(i);
- logger.error("Failure while running query", out, out.exception);
- }
- if (!outcomes.isEmpty()) {
- RunOutcome out = outcomes.iterator().next();
- if (out.exception != null) {
- throw new RuntimeException("Query Failed while running.", out.exception);
- } else {
- throw new RuntimeException("Query Failed while running. " + o);
- }
- }
- } catch (Exception e) {
- throw new RuntimeException("failed", e);
- }
-
- throw new RuntimeException("failed");
- }
- } else {
- Object o1 = parseJson((byte[]) o);
- if (holder != null) {
- o1 = ((Map<String, Object>) o1).get(holder);
- }
- if (fields == null) {
- current = o1;
- } else {
- final Map<String, Object> map = (Map<String, Object>) o1;
- if (fields.size() == 1) {
- current = map.get(fields.get(0));
- } else {
- Object[] os = new Object[fields.size()];
- for (int i = 0; i < os.length; i++) {
- os[i] = map.get(fields.get(i));
- }
- current = os;
- }
- }
- return true;
- }
- } catch (InterruptedException e) {
- Thread.interrupted();
- throw new RuntimeException(e);
- }
- }
-
- public void reset() {
- throw new UnsupportedOperationException();
- }
- }
-
- /**
- * Runs the plan as a background task.
- */
- Future<Collection<RunOutcome>> runRefInterpreterPlan(CompletionService<Collection<RunOutcome>> service) {
- LogicalPlan parsedPlan = LogicalPlan.parse(DrillConfig.create(), plan);
- IteratorRegistry ir = new IteratorRegistry();
- DrillConfig config = DrillConfig.create();
- config.setSinkQueues(0, queue);
- final ReferenceInterpreter i = new ReferenceInterpreter(parsedPlan, ir, new BasicEvaluatorFactory(ir),
- new RSERegistry(config));
- try {
- i.setup();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return service.submit(new Callable<Collection<RunOutcome>>() {
- @Override
- public Collection<RunOutcome> call() throws Exception {
- Collection<RunOutcome> outcomes = i.run();
-
- for (RunOutcome outcome : outcomes) {
- System.out.println("============");
- System.out.println(outcome);
- if (outcome.outcome == RunOutcome.OutcomeType.FAILED && outcome.exception != null) {
- outcome.exception.printStackTrace();
- }
- }
- return outcomes;
- }
- });
- }
-
- public Enumerator<E> enumerator() {
- // TODO: use a completion service from the container
- final ExecutorCompletionService<Collection<RunOutcome>> service = new ExecutorCompletionService<Collection<RunOutcome>>(
- new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingDeque<Runnable>(10)));
-
- // Run the plan using an executor. It runs in a different thread, writing
- // results to our queue.
- //
- // TODO: use the result of task, and check for exceptions
- final Future<Collection<RunOutcome>> task = runRefInterpreterPlan(service);
-
- return new JsonEnumerator(task, queue, fields);
-
- }
-
- /**
- * Converts a JSON document, represented as an array of bytes, into a Java object (consisting of DistributedMap, List, String,
- * Integer, Double, Boolean).
- */
- static Object parseJson(byte[] bytes) {
- try {
- return wrapper(mapper.readTree(bytes));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Converts a JSON node to Java objects ({@link List}, {@link Map}, {@link String}, {@link Integer}, {@link Double},
- * {@link Boolean}.
- */
- static Object wrapper(JsonNode node) {
- switch (node.asToken()) {
- case START_OBJECT:
- return map((ObjectNode) node);
- case START_ARRAY:
- return array((ArrayNode) node);
- case VALUE_STRING:
- return node.asText();
- case VALUE_NUMBER_INT:
- return node.asInt();
- case VALUE_NUMBER_FLOAT:
- return node.asDouble();
- case VALUE_TRUE:
- return Boolean.TRUE;
- case VALUE_FALSE:
- return Boolean.FALSE;
- case VALUE_NULL:
- return null;
- default:
- throw new AssertionError("unexpected: " + node + ": " + node.asToken());
- }
- }
-
- private static List<Object> array(ArrayNode node) {
- final List<Object> list = new ArrayList<>();
- for (JsonNode jsonNode : node) {
- list.add(wrapper(jsonNode));
- }
- return Collections.unmodifiableList(list);
- }
-
- private static SortedMap<String, Object> map(ObjectNode node) {
- // TreeMap makes the results deterministic.
- final TreeMap<String, Object> map = new TreeMap<>();
- final Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
- while (fields.hasNext()) {
- Map.Entry<String, JsonNode> next = fields.next();
- map.put(next.getKey(), wrapper(next.getValue()));
- }
- return Collections.unmodifiableSortedMap(map);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/resources/storage-engines.json b/sqlparser/src/main/resources/storage-engines.json
deleted file mode 100644
index d1d0413..0000000
--- a/sqlparser/src/main/resources/storage-engines.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
- "storage":{
- "parquet-local" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- },
- "parquet-cp" :
- {
- "type":"parquet",
- "dfsName" : "classpath:///"
- },
- "jsonl" :
- {
- "type":"json",
- "dfsName" : "file:///"
- },
- "json-cp" :
- {
- "type":"json",
- "dfsName" : "classpath:///"
- },
- "parquet" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
deleted file mode 100644
index 5e5812c..0000000
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.jdbc.test;
-
-import org.junit.*;
- import org.junit.rules.TestName;
-import org.junit.rules.TestRule;
-import org.junit.rules.Timeout;
-
-import java.io.File;
-
-
-public class FullEngineTest {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FullEngineTest.class);
-
- // Determine if we are in Eclipse Debug mode.
- static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
-
- // Set a timeout unless we're debugging.
- @Rule public TestRule globalTimeout = IS_DEBUG ? new TestName() : new Timeout(100000);
-
- @Test
- @Ignore // since this is a specifically located file.
- public void fullSelectStarEngine() throws Exception {
- JdbcAssert.withFull("parquet-local")
- // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
- .sql("select _MAP['d'] as d, _MAP['b'] as b from \"/tmp/parquet_test_file_many_types\" ").displayResults(50);
- }
-
- @Test
- public void setCPRead() throws Exception {
- JdbcAssert.withFull("json-cp")
- // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
- .sql("select * from \"department.json\" ").displayResults(50);
- }
-
- /**
- * List tables using the system table (metadata.TABLES) which is same as "!tables" from SQLLine
- * @throws Exception
- */
- @Test(timeout=100000) // derby initialization is slow
- @Ignore
- public void listHiveTables() throws Exception {
- JdbcAssert.withFull("hive-derby")
- .sql("select * from \"metadata\".\"TABLES\"")
- .returns(
- "tableCat=null; tableSchem=hive-derby; tableName=testdb1.kv_seq; tableType=TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n" +
- "tableCat=null; tableSchem=hive-derby; tableName=default.all_types; tableType=TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n" +
- "tableCat=null; tableSchem=hive-derby; tableName=default.kv_text; tableType=TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n" +
- "tableCat=null; tableSchem=metadata; tableName=COLUMNS; tableType=SYSTEM_TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n" +
- "tableCat=null; tableSchem=metadata; tableName=TABLES; tableType=SYSTEM_TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n"
- );
- }
-
- @Test(timeout=1000000) // derby initialization is slow
- @Ignore // ignore this until Hive SerDe/InputFormat work is intergrated
- public void selectFromHiveTable() throws Exception {
- JdbcAssert.withFull("hive-derby")
- .sql("select * from \"testdb1\".\"kv\"").displayResults(10);
- }
-
- @AfterClass
- public static void cleanup() {
- // derby creates a derby.log in cwd. Currently there seems to be no way to pass the config
- // property from hive component to derby to disable or write logging in a different location
- // need to delete this file for RAT pass
- File derbyLog = new File("./derby.log");
- if (derbyLog.exists()) {
- derbyLog.delete();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
index 651a509..1d7c7e3 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
@@ -130,15 +130,8 @@ public class JdbcAssert {
this.info = info;
this.connectionFactory = new ConnectionFactory() {
public Connection createConnection() throws Exception {
- String connect = ref ? "jdbc:drillref:" : "jdbc:drill:";
- if (ref) {
- Class.forName("org.apache.drill.jdbc.RefDriver");
- } else {
- Class.forName("org.apache.drill.jdbc.Driver");
- }
-
-
- return DriverManager.getConnection(connect, ModelAndSchema.this.info);
+ Class.forName("org.apache.drill.jdbc.Driver");
+ return DriverManager.getConnection("jdbc:drill:zk=local", ModelAndSchema.this.info);
}
};
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
index 9ae9682..aef4266 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
@@ -21,24 +21,23 @@ import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
-import java.util.Iterator;
-import java.util.List;
import java.util.Map;
-import com.beust.jcommander.internal.Lists;
-import com.beust.jcommander.internal.Maps;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-
-import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.logical.PlanProperties;
import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.logical.PlanProperties;
import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.logical.data.*;
-import org.apache.drill.exec.ref.ReferenceInterpreter;
-import org.apache.drill.exec.ref.rse.ClasspathRSE;
-import org.apache.drill.exec.ref.rse.QueueRSE;
+import org.apache.drill.common.logical.data.CollapsingAggregate;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Store;
+import org.apache.drill.common.logical.data.Union;
import org.apache.drill.jdbc.test.JdbcAssert.TestDataConnection;
+import org.eigenbase.rel.JoinRelType;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -46,10 +45,14 @@ import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
import com.google.common.io.Resources;
/** Unit tests for Drill's JDBC driver. */
+
+@Ignore // ignore for now.
public class JdbcTest {
private static String MODEL;
private static String EXPECTED;
@@ -69,7 +72,6 @@ public class JdbcTest {
* </p>
*/
public static void main(String[] args) throws Exception {
- ReferenceInterpreter.main(args);
}
/** Load driver. */
@@ -82,7 +84,7 @@ public class JdbcTest {
@Test
public void testConnect() throws Exception {
Class.forName("org.apache.drill.jdbc.Driver");
- final Connection connection = DriverManager.getConnection("jdbc:drillref:schema=DONUTS");
+ final Connection connection = DriverManager.getConnection("jdbc:drill:zk=local");
connection.close();
}
@@ -178,9 +180,9 @@ public class JdbcTest {
Assert.assertEquals(PlanProperties.PlanType.APACHE_DRILL_LOGICAL, planProperties.type);
Map<String, StorageEngineConfig> seConfigs = plan.getStorageEngines();
StorageEngineConfig config = seConfigs.get("donuts-json");
- Assert.assertTrue(config != null && config instanceof ClasspathRSE.ClasspathRSEConfig);
+// Assert.assertTrue(config != null && config instanceof ClasspathRSE.ClasspathRSEConfig);
config = seConfigs.get("queue");
- Assert.assertTrue(config != null && config instanceof QueueRSE.QueueRSEConfig);
+// Assert.assertTrue(config != null && config instanceof QueueRSE.QueueRSEConfig);
Scan scan = findOnlyOperator(plan, Scan.class);
Assert.assertEquals("donuts-json", scan.getStorageEngine());
Assert.assertEquals("_MAP", scan.getOutputReference().getPath());
@@ -234,9 +236,9 @@ public class JdbcTest {
Assert.assertEquals(PlanProperties.PlanType.APACHE_DRILL_LOGICAL, planProperties.type);
Map<String, StorageEngineConfig> seConfigs = plan.getStorageEngines();
StorageEngineConfig config = seConfigs.get("donuts-json");
- Assert.assertTrue(config != null && config instanceof ClasspathRSE.ClasspathRSEConfig);
+// Assert.assertTrue(config != null && config instanceof ClasspathRSE.ClasspathRSEConfig);
config = seConfigs.get("queue");
- Assert.assertTrue(config != null && config instanceof QueueRSE.QueueRSEConfig);
+// Assert.assertTrue(config != null && config instanceof QueueRSE.QueueRSEConfig);
Scan scan = findOnlyOperator(plan, Scan.class);
Assert.assertEquals("donuts-json", scan.getStorageEngine());
Assert.assertEquals("_MAP", scan.getOutputReference().getPath());
@@ -331,7 +333,7 @@ public class JdbcTest {
"DEPTID=33; LASTNAME=Steinberg; DEPTID0=33; NAME=Engineering",
"DEPTID=34; LASTNAME=Robinson; DEPTID0=34; NAME=Clerical",
"DEPTID=34; LASTNAME=Smith; DEPTID0=34; NAME=Clerical").planContains(Join.class);
- Assert.assertEquals(Join.JoinType.INNER, join.getJointType());
+ Assert.assertEquals(JoinRelType.INNER, join.getJoinType());
}
@Test
@@ -345,7 +347,7 @@ public class JdbcTest {
"DEPTID=34; LASTNAME=Robinson; DEPTID0=34; NAME=Clerical",
"DEPTID=34; LASTNAME=Smith; DEPTID0=34; NAME=Clerical",
"DEPTID=null; LASTNAME=John; DEPTID0=null; NAME=null").planContains(Join.class);
- Assert.assertEquals(Join.JoinType.LEFT, join.getJointType());
+ Assert.assertEquals(JoinRelType.LEFT, join.getJoinType());
}
/**
@@ -355,7 +357,7 @@ public class JdbcTest {
public void testRightJoin() throws Exception {
Join join = JdbcAssert.withModel(MODEL, "HR").sql("select * from emp right join dept on emp.deptId = dept.deptId")
.returnsUnordered("xx").planContains(Join.class);
- Assert.assertEquals(Join.JoinType.LEFT, join.getJointType());
+ Assert.assertEquals(JoinRelType.LEFT, join.getJoinType());
}
@Test
@@ -370,7 +372,7 @@ public class JdbcTest {
"DEPTID=34; LASTNAME=Smith; DEPTID0=34; NAME=Clerical",
"DEPTID=null; LASTNAME=John; DEPTID0=null; NAME=null",
"DEPTID=null; LASTNAME=null; DEPTID0=35; NAME=Marketing").planContains(Join.class);
- Assert.assertEquals(Join.JoinType.OUTER, join.getJointType());
+ Assert.assertEquals(JoinRelType.FULL, join.getJoinType());
}
/**
@@ -389,7 +391,7 @@ public class JdbcTest {
"DEPTID=33; LASTNAME=Steinberg; NAME=x; DEPTID0=33; NAME0=Engineering",
"DEPTID=34; LASTNAME=Robinson; NAME=x; DEPTID0=34; NAME0=Clerical",
"DEPTID=34; LASTNAME=Smith; NAME=x; DEPTID0=34; NAME0=Clerical").planContains(Join.class);
- Assert.assertEquals(Join.JoinType.INNER, join.getJointType());
+ Assert.assertEquals(JoinRelType.INNER, join.getJoinType());
}
/** Tests that one of the FoodMart tables is present. */
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
new file mode 100644
index 0000000..e6944cb
--- /dev/null
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc.test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.Statement;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.common.util.TestTools;
+import org.apache.drill.jdbc.Driver;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+
+import com.google.common.base.Stopwatch;
+
+public class TestJdbcQuery {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class);
+
+ static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
+
+ // Set a timeout unless we're debugging.
+ @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule();
+
+ static{
+ Driver.load();
+ }
+
+ @Test
+ public void testCast() throws Exception{
+ testQuery("select R_REGIONKEY, cast(R_NAME as varchar(15)) as region, cast(R_COMMENT as varchar(255)) as comment from parquet.`/Users/jnadeau/region.parquet`");
+ }
+ @Test
+ public void testWildcard() throws Exception{
+ testQuery("select * from parquet.`/Users/jnadeau/region.parquet`");
+ }
+
+ @Test
+ public void testLogicalExplain() throws Exception{
+ testQuery("EXPLAIN PLAN WITHOUT IMPLEMENTATION FOR select * from parquet.`/Users/jnadeau/region.parquet`");
+ }
+
+ @Test
+ public void testPhysicalExplain() throws Exception{
+ testQuery("EXPLAIN PLAN FOR select * from parquet.`/Users/jnadeau/region.parquet`");
+ }
+
+ @Test
+ public void checkUnknownColumn() throws Exception{
+ testQuery("select unknownColumn from parquet.`/Users/jnadeau/region.parquet`");
+ }
+
+
+ private void testQuery(String sql) throws Exception{
+ boolean success = false;
+ try (Connection c = DriverManager.getConnection("jdbc:drill:zk=local", null);) {
+ for (int x = 0; x < 1; x++) {
+ Stopwatch watch = new Stopwatch().start();
+ Statement s = c.createStatement();
+ ResultSet r = s.executeQuery(sql);
+ boolean first = true;
+ while (r.next()) {
+ ResultSetMetaData md = r.getMetaData();
+ if (first == true) {
+ for (int i = 1; i <= md.getColumnCount(); i++) {
+ System.out.print(md.getColumnName(i));
+ System.out.print('\t');
+ }
+ System.out.println();
+ first = false;
+ }
+
+ for (int i = 1; i <= md.getColumnCount(); i++) {
+ System.out.print(r.getObject(i));
+ System.out.print('\t');
+ }
+ System.out.println();
+ }
+
+ System.out.println(String.format("Query completed in %d millis.", watch.elapsed(TimeUnit.MILLISECONDS)));
+ }
+
+ success = true;
+ }finally{
+ if(!success) Thread.sleep(2000);
+ }
+
+
+ }
+}
[32/38] git commit: DRILL-366,
DRILL-364: Small fixes due to merge issues
Posted by ja...@apache.org.
DRILL-366, DRILL-364: Small fixes due to merge issues
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d90d5b16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d90d5b16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d90d5b16
Branch: refs/heads/master
Commit: d90d5b165ee3bdb072a8101a3b0acfb8c930c935
Parents: 7716c7e
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Feb 23 17:37:21 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../compile/sig/ConstantExpressionIdentifier.java | 1 +
.../drill/exec/store/dfs/easy/EasyGroupScan.java | 1 +
.../drill/exec/physical/impl/TestCastFunctions.java | 2 +-
.../exec/physical/impl/TestCastVarCharToBigInt.java | 2 +-
.../resources/functions/cast/testCastVarCharNull.json | 14 ++++----------
.../functions/cast/test_cast_varchar_to_bigint.json | 14 ++++----------
6 files changed, 12 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d90d5b16/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
index 4d07a8c..7a48112 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
@@ -26,6 +26,7 @@ import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d90d5b16/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
index 5fcf7a5..a7f556e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
@@ -74,6 +74,7 @@ public class EasyGroupScan extends AbstractGroupScan{
) throws IOException, ExecutionSetupException {
this.formatPlugin = (EasyFormatPlugin<?>) engineRegistry.getFormatPlugin(storageConfig, formatConfig);
+ Preconditions.checkNotNull(formatPlugin, "Unable to load format plugin for provided format config.");
this.selection = new FileSelection(files, true);
this.maxWidth = selection.getFileStatusList(formatPlugin.getFileSystem()).size();
this.ref = ref;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d90d5b16/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
index 4d18815..0659e5b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
@@ -393,7 +393,7 @@ public class TestCastFunctions extends PopUnitTestBase{
client.connect();
List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"), Charsets.UTF_8).replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/jsoninput/input1.json").toURI().toString()));
+ Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"), Charsets.UTF_8).replace("#{TEST_FILE}", "/jsoninput/input1.json"));
QueryResultBatch batch = results.get(0);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d90d5b16/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
index ba8b9da..ac8e5c6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
@@ -78,7 +78,7 @@ public class TestCastVarCharToBigInt extends PopUnitTestBase {
client.connect();
List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
Files.toString(FileUtils.getResourceAsFile("/functions/cast/test_cast_varchar_to_bigint.json"), Charsets.UTF_8)
- .replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/scan_json_test_cast.json").toURI().toString())
+ .replace("#{TEST_FILE}", "/scan_json_test_cast.json")
);
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d90d5b16/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
index 030b696..3ca7ed6 100644
--- a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
+++ b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
@@ -9,16 +9,10 @@
graph:[
{
@id:1,
- pop:"json-scan",
- entries: [
- {
- path : "#{TEST_FILE}"
- }
- ],
- storageengine: {
- "type": "json",
- "dfsName": "file:///"
- }
+ pop:"fs-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "classpath:///"},
+ files:["#{TEST_FILE}"]
}, {
@id:2,
child: 1,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d90d5b16/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json b/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json
index 36439c5..018dbad 100644
--- a/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json
+++ b/exec/java-exec/src/test/resources/functions/cast/test_cast_varchar_to_bigint.json
@@ -9,16 +9,10 @@
graph:[
{
@id:1,
- pop:"json-scan",
- entries: [
- {
- path : "#{TEST_FILE}"
- }
- ],
- storageengine: {
- "type": "json",
- "dfsName": "file:///"
- }
+ pop:"fs-scan",
+ format: {type: "json"},
+ storage:{type: "file", connection: "classpath:///"},
+ files:["#{TEST_FILE}"]
},
{
pop:"project",
[04/38] git commit: DRILL-315: Parquet/JSON selective column read,
parquet performance evaluation code from Jacques. Requested columns
are specified in physical plans,
logical scan operations will be modified by the optimizer as it analyzes the
rest of t
Posted by ja...@apache.org.
DRILL-315: Parquet/JSON selective column read, parquet performance evaluation code from Jacques. Requested columns are specified in physical plans, logical scan operations will be modified by the optimizer as it analyzes the rest of the query graph to decide which columns are needed. This still needs to be added to the optimizer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2dfadc9d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2dfadc9d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2dfadc9d
Branch: refs/heads/master
Commit: 2dfadc9d9945c347fded28ec90af5d0842f91853
Parents: b3460af
Author: Jason Altekruse <al...@gmail.com>
Authored: Wed Feb 12 09:28:07 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:16 2014 -0800
----------------------------------------------------------------------
.../drill/common/expression/SchemaPath.java | 4 +
exec/java-exec/pom.xml | 4 +-
.../drill/exec/store/json/JSONGroupScan.java | 19 ++-
.../drill/exec/store/json/JSONRecordReader.java | 29 ++++-
.../exec/store/json/JSONScanBatchCreator.java | 3 +-
.../exec/store/json/JSONStorageEngine.java | 3 +-
.../drill/exec/store/json/JSONSubScan.java | 16 ++-
.../drill/exec/store/parquet/BitReader.java | 10 +-
.../exec/store/parquet/ColumnDataReader.java | 90 +++++++++++++
.../drill/exec/store/parquet/ColumnReader.java | 64 +++++-----
.../store/parquet/FixedByteAlignedReader.java | 6 +-
.../exec/store/parquet/NullableBitReader.java | 3 +-
.../store/parquet/NullableColumnReader.java | 3 +-
.../parquet/NullableFixedByteAlignedReader.java | 28 +++--
.../exec/store/parquet/PageReadStatus.java | 80 ++++++------
.../exec/store/parquet/ParquetGroupScan.java | 23 +++-
.../exec/store/parquet/ParquetRecordReader.java | 126 +++++++++----------
.../exec/store/parquet/ParquetRowGroupScan.java | 27 +++-
.../store/parquet/ParquetScanBatchCreator.java | 13 +-
.../store/parquet/ParquetStorageEngine.java | 5 +-
.../exec/store/parquet/VarLenBinaryReader.java | 11 +-
.../drill/exec/store/JSONRecordReaderTest.java | 50 +++++++-
.../exec/store/ParquetRecordReaderTest.java | 60 ++++++---
.../test/resources/parquet_nullable_varlen.json | 41 ++++++
.../parquet_selective_column_read.json | 28 +++++
25 files changed, 536 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 7c2dce6..76838dc 100644
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -63,6 +63,10 @@ public class SchemaPath extends LogicalExpressionBase {
this.originalPath = path.originalPath;
this.rootSegment = path.rootSegment;
}
+
+ public SchemaPath(String str){
+ this(str, ExpressionPosition.UNKNOWN);
+ }
public SchemaPath(CharSequence str, ExpressionPosition pos) {
super(pos);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index a016afc..5c6fcfe 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -77,7 +77,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-column</artifactId>
- <version>1.0.1</version>
+ <version>1.2.8</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -92,7 +92,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop</artifactId>
- <version>1.0.1</version>
+ <version>1.2.8</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
index 8988f44..4782d82 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.ReadEntry;
@@ -51,15 +52,18 @@ public class JSONGroupScan extends AbstractGroupScan {
private final OperatorCost cost;
private final Size size;
private final FieldReference ref;
+ private final List<SchemaPath> columns;
@JsonCreator
public JSONGroupScan(@JsonProperty("entries") List<ScanEntry> entries,
@JsonProperty("storageengine") JSONStorageEngineConfig storageEngineConfig,
- @JacksonInject StorageEngineRegistry engineRegistry, @JsonProperty("ref") FieldReference ref) throws ExecutionSetupException {
- this(entries, (JSONStorageEngine) engineRegistry.getEngine(storageEngineConfig), ref);
+ @JacksonInject StorageEngineRegistry engineRegistry, @JsonProperty("ref") FieldReference ref,
+ @JsonProperty("columns") List<SchemaPath> columns) throws ExecutionSetupException {
+ this(entries, (JSONStorageEngine) engineRegistry.getEngine(storageEngineConfig), ref, columns);
}
- public JSONGroupScan(List<ScanEntry> entries, JSONStorageEngine engine, FieldReference ref) {
+ public JSONGroupScan(List<ScanEntry> entries, JSONStorageEngine engine, FieldReference ref,
+ List<SchemaPath> columns) {
this.engine = engine;
this.readEntries = entries;
OperatorCost cost = new OperatorCost(0, 0, 0, 0);
@@ -71,6 +75,7 @@ public class JSONGroupScan extends AbstractGroupScan {
this.cost = cost;
this.size = size;
this.ref = ref;
+ this.columns = columns;
}
@SuppressWarnings("unchecked")
@@ -97,7 +102,7 @@ public class JSONGroupScan extends AbstractGroupScan {
@Override
public SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException{
checkArgument(minorFragmentId < mappings.length, "Mappings length [%s] should be longer than minor fragment id [%s] but it isn't.", mappings.length, minorFragmentId);
- return new JSONSubScan(mappings[minorFragmentId], engine, ref);
+ return new JSONSubScan(mappings[minorFragmentId], engine, ref, columns);
}
@Override
@@ -108,7 +113,7 @@ public class JSONGroupScan extends AbstractGroupScan {
@Override
@JsonIgnore
public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
- return new JSONGroupScan(readEntries, engine, ref);
+ return new JSONGroupScan(readEntries, engine, ref, columns);
}
public static class ScanEntry implements ReadEntry {
@@ -141,6 +146,10 @@ public class JSONGroupScan extends AbstractGroupScan {
return readEntries.size();
}
+ public List<SchemaPath> getColumns() {
+ return columns;
+ }
+
@Override
public OperatorCost getCost() {
return cost;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
index 49aea22..11b972c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
@@ -90,18 +90,22 @@ public class JSONRecordReader implements RecordReader {
private BufferAllocator allocator;
private int batchSize;
private final FieldReference ref;
+ private final List<SchemaPath> columns;
- public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize, FieldReference ref) {
+ public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize,
+ FieldReference ref, List<SchemaPath> columns) {
this.hadoopPath = new Path(inputPath);
this.fileSystem = fileSystem;
this.allocator = fragmentContext.getAllocator();
this.batchSize = batchSize;
valueVectorMap = Maps.newHashMap();
this.ref = ref;
+ this.columns = columns;
}
- public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, FieldReference ref) {
- this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, ref);
+ public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, FieldReference ref,
+ List<SchemaPath> columns) {
+ this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, ref, columns);
}
private JsonParser getParser() {
@@ -209,6 +213,19 @@ public class JSONRecordReader implements RecordReader {
return allocator;
}
+ private boolean fieldSelected(String field){
+ SchemaPath sp = new SchemaPath(field, ExpressionPosition.UNKNOWN);
+ if (this.columns != null && this.columns.size() > 1){
+ for (SchemaPath expr : this.columns){
+ if ( sp.equals(expr)){
+ return true;
+ }
+ }
+ return false;
+ }
+ return true;
+ }
+
public static enum ReadType {
ARRAY(END_ARRAY) {
@Override
@@ -264,6 +281,12 @@ public class JSONRecordReader implements RecordReader {
}
String fieldName = parser.getCurrentName();
+ if ( fieldName != null && ! reader.fieldSelected(fieldName)){
+ // this field was not requested in the query
+ token = parser.nextToken();
+ colIndex += 1;
+ continue;
+ }
MajorType fieldType = JacksonHelper.getFieldType(token, this == ReadType.ARRAY);
ReadType readType = null;
switch (token) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
index a4c00cc..c40cb47 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
@@ -37,7 +37,8 @@ public class JSONScanBatchCreator implements BatchCreator<JSONSubScan> {
List<JSONGroupScan.ScanEntry> entries = config.getReadEntries();
List<RecordReader> readers = Lists.newArrayList();
for (JSONGroupScan.ScanEntry e : entries) {
- readers.add(new JSONRecordReader(context, e.getPath(), config.getStorageEngine().getFileSystem(), config.getRef()));
+ readers.add(new JSONRecordReader(context, e.getPath(), config.getStorageEngine().getFileSystem(), config.getRef(),
+ config.getColumns()));
}
return new ScanBatch(context, readers.iterator());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
index bd3bca5..e4f2070 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.json;
import java.io.IOException;
import java.util.ArrayList;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.Scan;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.AbstractStorageEngine;
@@ -65,7 +66,7 @@ public class JSONStorageEngine extends AbstractStorageEngine {
@Override
public JSONGroupScan getPhysicalScan(Scan scan) throws IOException {
ArrayList<ScanEntry> readEntries = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<ArrayList<ScanEntry>>() {});
- return new JSONGroupScan(readEntries, this, scan.getOutputReference());
+ return new JSONGroupScan(readEntries, this, scan.getOutputReference(), null);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
index a04f906..92f6c0a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.AbstractBase;
@@ -46,15 +47,19 @@ public class JSONSubScan extends AbstractBase implements SubScan {
private final Size size;
private final JSONStorageEngine storageEngine;
private final FieldReference ref;
+ private final List<SchemaPath> columns;
@JsonCreator
public JSONSubScan(@JacksonInject StorageEngineRegistry registry,
@JsonProperty("engineConfig") StorageEngineConfig engineConfig,
- @JsonProperty("readEntries") List<JSONGroupScan.ScanEntry> readEntries, @JsonProperty("ref") FieldReference ref) throws ExecutionSetupException {
- this(readEntries, (JSONStorageEngine) registry.getEngine(engineConfig), ref);
+ @JsonProperty("readEntries") List<JSONGroupScan.ScanEntry> readEntries,
+ @JsonProperty("ref") FieldReference ref,
+ @JsonProperty("columns") List<SchemaPath> columns) throws ExecutionSetupException {
+ this(readEntries, (JSONStorageEngine) registry.getEngine(engineConfig), ref, columns);
}
- JSONSubScan(List<JSONGroupScan.ScanEntry> readEntries, JSONStorageEngine engine, FieldReference ref){
+ JSONSubScan(List<JSONGroupScan.ScanEntry> readEntries, JSONStorageEngine engine, FieldReference ref,
+ List<SchemaPath> columns){
this.readEntries = readEntries;
this.storageEngine = engine;
OperatorCost cost = new OperatorCost(0, 0, 0, 0);
@@ -66,6 +71,7 @@ public class JSONSubScan extends AbstractBase implements SubScan {
this.cost = cost;
this.size = size;
this.ref = ref;
+ this.columns = columns;
}
public FieldReference getRef() {
@@ -109,4 +115,8 @@ public class JSONSubScan extends AbstractBase implements SubScan {
public Iterator<PhysicalOperator> iterator() {
return Iterators.emptyIterator();
}
+
+ public List<SchemaPath> getColumns() {
+ return columns;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
index a3f9503..d0049c7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.BaseDataValueVector;
import org.apache.drill.exec.vector.ValueVector;
import parquet.column.ColumnDescriptor;
@@ -24,11 +25,12 @@ import parquet.hadoop.metadata.ColumnChunkMetaData;
public final class BitReader extends ColumnReader {
- byte currentByte;
- byte nextByte;
-
+ private byte currentByte;
+ private byte nextByte;
+ private byte[] bytes;
+
BitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
- boolean fixedLength, ValueVector v) {
+ boolean fixedLength, ValueVector v) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
new file mode 100644
index 0000000..ec44747
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import parquet.bytes.BytesInput;
+import parquet.format.PageHeader;
+import parquet.format.Util;
+
+public class ColumnDataReader {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.class);
+
+ private final long endPosition;
+ private final FSDataInputStream input;
+
+ public ColumnDataReader(FileSystem fs, Path path, long start, long length) throws IOException{
+ this.input = fs.open(path, 64 * 1024);
+ this.input.seek(start);
+ this.endPosition = start + length;
+ }
+
+ public PageHeader readPageHeader() throws IOException{
+ return Util.readPageHeader(input);
+ }
+
+ public BytesInput getPageAsBytesInput(int pageLength) throws IOException{
+ byte[] b = new byte[pageLength];
+ input.read(b);
+ return new HadoopBytesInput(b);
+ }
+
+ public void clear(){
+ try{
+ input.close();
+ }catch(IOException ex){
+ logger.warn("Error while closing input stream.", ex);
+ }
+ }
+
+ public boolean hasRemainder() throws IOException{
+ return input.getPos() < endPosition;
+ }
+
+ public class HadoopBytesInput extends BytesInput{
+
+ private final byte[] pageBytes;
+
+ public HadoopBytesInput(byte[] pageBytes) {
+ super();
+ this.pageBytes = pageBytes;
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return pageBytes;
+ }
+
+ @Override
+ public long size() {
+ return pageBytes.length;
+ }
+
+ @Override
+ public void writeAllTo(OutputStream out) throws IOException {
+ out.write(pageBytes);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
index 0277d12..94ccbfc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
@@ -18,9 +18,11 @@
package org.apache.drill.exec.store.parquet;
import io.netty.buffer.ByteBuf;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.store.VectorHolder;
import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.hadoop.fs.FSDataInputStream;
import parquet.column.ColumnDescriptor;
import parquet.hadoop.metadata.ColumnChunkMetaData;
import parquet.schema.PrimitiveType;
@@ -28,53 +30,57 @@ import parquet.schema.PrimitiveType;
import java.io.IOException;
public abstract class ColumnReader {
+
+ final ParquetRecordReader parentReader;
+
// Value Vector for this column
- VectorHolder valueVecHolder;
+ final VectorHolder valueVecHolder;
// column description from the parquet library
- ColumnDescriptor columnDescriptor;
+ final ColumnDescriptor columnDescriptor;
// metadata of the column, from the parquet library
- ColumnChunkMetaData columnChunkMetaData;
+ final ColumnChunkMetaData columnChunkMetaData;
// status information on the current page
- PageReadStatus pageReadStatus;
-
- long readPositionInBuffer;
+ final PageReadStatus pageReadStatus;
// quick reference to see if the field is fixed length (as this requires an instanceof)
- boolean isFixedLength;
+ final boolean isFixedLength;
+
// counter for the total number of values read from one or more pages
// when a batch is filled all of these values should be the same for each column
int totalValuesRead;
+
// counter for the values that have been read in this pass (a single call to the next() method)
int valuesReadInCurrentPass;
+
// length of single data value in bits, if the length is fixed
int dataTypeLengthInBits;
int bytesReadInCurrentPass;
- ParquetRecordReader parentReader;
- ByteBuf vectorData;
+ protected ByteBuf vectorData;
// variables for a single read pass
long readStartInBytes = 0, readLength = 0, readLengthInBits = 0, recordsReadInThisIteration = 0;
- byte[] bytes;
- ColumnReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
- boolean fixedLength, ValueVector v){
+ protected ColumnReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) throws ExecutionSetupException {
this.parentReader = parentReader;
- if (allocateSize > 1) valueVecHolder = new VectorHolder(allocateSize, v);
- else valueVecHolder = new VectorHolder(5000, v);
-
- columnDescriptor = descriptor;
+ this.columnDescriptor = descriptor;
this.columnChunkMetaData = columnChunkMetaData;
- isFixedLength = fixedLength;
+ this.isFixedLength = fixedLength;
+
+ if (allocateSize > 1) {
+ valueVecHolder = new VectorHolder(allocateSize, v);
+ } else {
+ valueVecHolder = new VectorHolder(5000, v);
+ }
- pageReadStatus = new PageReadStatus(this, parentReader.getRowGroupIndex(), parentReader.getBufferWithAllData());
- if (parentReader.getRowGroupIndex() != 0) readPositionInBuffer = columnChunkMetaData.getFirstDataPageOffset() - 4;
- else readPositionInBuffer = columnChunkMetaData.getFirstDataPageOffset();
+ this.pageReadStatus = new PageReadStatus(this, parentReader.fileSystem, parentReader.hadoopPath, columnChunkMetaData);
if (columnDescriptor.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
dataTypeLengthInBits = ParquetRecordReader.getTypeLengthInBits(columnDescriptor.getType());
}
+
}
public void readAllFixedFields(long recordsToReadInThisPass, ColumnReader firstColumnStatus) throws IOException {
@@ -82,17 +88,16 @@ public abstract class ColumnReader {
readLength = 0;
readLengthInBits = 0;
recordsReadInThisIteration = 0;
- vectorData = ((BaseValueVector)valueVecHolder.getValueVector()).getData();
+ vectorData = ((BaseValueVector) valueVecHolder.getValueVector()).getData();
do {
// if no page has been read, or all of the records have been read out of a page, read the next one
- if (pageReadStatus.currentPage == null
- || pageReadStatus.valuesRead == pageReadStatus.currentPage.getValueCount()) {
+ if (pageReadStatus.currentPage == null || pageReadStatus.valuesRead == pageReadStatus.currentPage.getValueCount()) {
if (!pageReadStatus.next()) {
break;
}
}
- readField( recordsToReadInThisPass, firstColumnStatus);
+ readField(recordsToReadInThisPass, firstColumnStatus);
valuesReadInCurrentPass += recordsReadInThisIteration;
totalValuesRead += recordsReadInThisIteration;
@@ -102,10 +107,13 @@ public abstract class ColumnReader {
} else {
pageReadStatus.readPosInBytes = readStartInBytes + readLength;
}
- }
- while (valuesReadInCurrentPass < recordsToReadInThisPass && pageReadStatus.currentPage != null);
- valueVecHolder.getValueVector().getMutator().setValueCount(
- valuesReadInCurrentPass);
+ } while (valuesReadInCurrentPass < recordsToReadInThisPass && pageReadStatus.currentPage != null);
+ valueVecHolder.getValueVector().getMutator().setValueCount(valuesReadInCurrentPass);
+ }
+
+ public void clear() {
+ this.valueVecHolder.reset();
+ this.pageReadStatus.clear();
}
protected abstract void readField(long recordsToRead, ColumnReader firstColumnStatus);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java
index 7208cf7..3aae189 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.BaseDataValueVector;
import org.apache.drill.exec.vector.ValueVector;
import parquet.column.ColumnDescriptor;
@@ -24,8 +25,11 @@ import parquet.hadoop.metadata.ColumnChunkMetaData;
public class FixedByteAlignedReader extends ColumnReader {
+ private byte[] bytes;
+
+
FixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
- boolean fixedLength, ValueVector v) {
+ boolean fixedLength, ValueVector v) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
index 7da45cc..1dde7c7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.BaseDataValueVector;
import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.NullableBitVector;
@@ -37,7 +38,7 @@ import java.io.IOException;
public final class NullableBitReader extends ColumnReader {
NullableBitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
- boolean fixedLength, ValueVector v) {
+ boolean fixedLength, ValueVector v) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
index 596108c..5ac9bb1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
import org.apache.drill.exec.vector.ValueVector;
@@ -34,7 +35,7 @@ public abstract class NullableColumnReader extends ColumnReader{
int bitsUsed;
NullableColumnReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
- boolean fixedLength, ValueVector v){
+ boolean fixedLength, ValueVector v) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
index 094e955..bb81024 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
@@ -17,16 +17,18 @@
*/
package org.apache.drill.exec.store.parquet;
-import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.ValueVector;
+
import parquet.column.ColumnDescriptor;
import parquet.hadoop.metadata.ColumnChunkMetaData;
public class NullableFixedByteAlignedReader extends NullableColumnReader {
- NullableFixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
- boolean fixedLength, ValueVector v) {
+ private byte[] bytes;
+
+ NullableFixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
}
@@ -34,15 +36,15 @@ public class NullableFixedByteAlignedReader extends NullableColumnReader {
@Override
protected void readField(long recordsToReadInThisPass, ColumnReader firstColumnStatus) {
- recordsReadInThisIteration = recordsToReadInThisPass;
-
- readStartInBytes = pageReadStatus.readPosInBytes;
- readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
- readLength = (int) Math.ceil(readLengthInBits / 8.0);
+ this.recordsReadInThisIteration = recordsToReadInThisPass;
- bytes = pageReadStatus.pageDataByteArray;
- // vectorData is assigned by the superclass read loop method
- vectorData.writeBytes(bytes,
- (int) readStartInBytes, (int) readLength);
+ // set up metadata
+ this.readStartInBytes = pageReadStatus.readPosInBytes;
+ this.readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
+ this.readLength = (int) Math.ceil(readLengthInBits / 8.0);
+ this.bytes = pageReadStatus.pageDataByteArray;
+
+ // fill in data.
+ vectorData.writeBytes(bytes, (int) readStartInBytes, (int) readLength);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
index 39d636d..c08dcf3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
@@ -17,29 +17,28 @@
*/
package org.apache.drill.exec.store.parquet;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
+import java.io.IOException;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
import parquet.bytes.BytesInput;
import parquet.column.ValuesType;
import parquet.column.page.Page;
-import parquet.column.page.PageReader;
import parquet.column.values.ValuesReader;
import parquet.format.PageHeader;
-
-import java.io.IOException;
-
-import static parquet.format.Util.readPageHeader;
+import parquet.hadoop.metadata.ColumnChunkMetaData;
// class to keep track of the read position of variable length columns
public final class PageReadStatus {
- ColumnReader parentColumnReader;
-
+ private final ColumnReader parentColumnReader;
+ private final ColumnDataReader dataReader;
// store references to the pages that have been uncompressed, but not copied to ValueVectors yet
Page currentPage;
- // buffer to store bytes of current page, set to max size of parquet page
- byte[] pageDataByteArray = new byte[ParquetRecordReader.PARQUET_PAGE_MAX_SIZE];
-
+ // buffer to store bytes of current page
+ byte[] pageDataByteArray;
// read position in the current page, stored in the ByteBuf in ParquetRecordReader called bufferWithAllData
long readPosInBytes;
// bit shift needed for the next page if the last one did not line up with a byte boundary
@@ -50,15 +49,25 @@ public final class PageReadStatus {
// the number of values read out of the last page
int valuesRead;
int byteLength;
- int rowGroupIndex;
+ //int rowGroupIndex;
ValuesReader definitionLevels;
ValuesReader valueReader;
- PageReadStatus(ColumnReader parentStatus, int rowGroupIndex, ByteBuf bufferWithAllData){
+ PageReadStatus(ColumnReader parentStatus, FileSystem fs, Path path, ColumnChunkMetaData columnChunkMetaData) throws ExecutionSetupException{
this.parentColumnReader = parentStatus;
- this.rowGroupIndex = rowGroupIndex;
+
+ long totalByteLength = columnChunkMetaData.getTotalSize();
+ long start = columnChunkMetaData.getFirstDataPageOffset();
+
+ try{
+ this.dataReader = new ColumnDataReader(fs, path, start, totalByteLength);
+ } catch (IOException e) {
+ throw new ExecutionSetupException("Error opening or reading metatdata for parquet file at location: " + path.getName(), e);
+ }
+
}
+
/**
* Grab the next page.
*
@@ -67,27 +76,15 @@ public final class PageReadStatus {
*/
public boolean next() throws IOException {
- int shift = 0;
- if (rowGroupIndex == 0) shift = 0;
- else shift = 4;
- // first ROW GROUP has a different endpoint, because there are for bytes at the beginning of the file "PAR1"
- if (parentColumnReader.readPositionInBuffer + shift == parentColumnReader.columnChunkMetaData.getFirstDataPageOffset() + parentColumnReader.columnChunkMetaData.getTotalSize()){
- return false;
- }
- // TODO - in the JIRA for parquet steven put a stack trace for an error with a row group with 3 values in it
- // the Math.min with the end of the buffer should fix it but now I'm not getting results back, leaving it here for now
- // because it is needed, but there might be a problem with it
- ByteBufInputStream f = new ByteBufInputStream(parentColumnReader.parentReader.getBufferWithAllData().slice(
- (int) parentColumnReader.readPositionInBuffer,
- Math.min(200, parentColumnReader.parentReader.getBufferWithAllData().capacity() - (int) parentColumnReader.readPositionInBuffer)));
- int before = f.available();
- PageHeader pageHeader = readPageHeader(f);
- int length = before - f.available();
- f = new ByteBufInputStream(parentColumnReader.parentReader.getBufferWithAllData().slice(
- (int) parentColumnReader.readPositionInBuffer + length, pageHeader.getCompressed_page_size()));
+ if(!dataReader.hasRemainder()) return false;
+
+ // next, we need to decompress the bytes
+ PageHeader pageHeader = dataReader.readPageHeader();
BytesInput bytesIn = parentColumnReader.parentReader.getCodecFactoryExposer()
- .decompress(BytesInput.from(f, pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size(),
+ .decompress( //
+ dataReader.getPageAsBytesInput(pageHeader.compressed_page_size), //
+ pageHeader.getUncompressed_page_size(), //
parentColumnReader.columnChunkMetaData.getCodec());
currentPage = new Page(
bytesIn,
@@ -98,7 +95,6 @@ public final class PageReadStatus {
ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.encoding)
);
- parentColumnReader.readPositionInBuffer += pageHeader.compressed_page_size + length;
byteLength = pageHeader.uncompressed_page_size;
if (currentPage == null) {
@@ -106,12 +102,14 @@ public final class PageReadStatus {
}
// if the buffer holding each page's data is not large enough to hold the current page, re-allocate, with a little extra space
- if (pageHeader.getUncompressed_page_size() > pageDataByteArray.length) {
- pageDataByteArray = new byte[pageHeader.getUncompressed_page_size() + 100];
- }
+// if (pageHeader.getUncompressed_page_size() > pageDataByteArray.length) {
+// pageDataByteArray = new byte[pageHeader.getUncompressed_page_size() + 100];
+// }
// TODO - would like to get this into the mainline, hopefully before alpha
pageDataByteArray = currentPage.getBytes().toByteArray();
+ readPosInBytes = 0;
+ valuesRead = 0;
if (parentColumnReader.columnDescriptor.getMaxDefinitionLevel() != 0){
definitionLevels = currentPage.getDlEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.DEFINITION_LEVEL);
valueReader = currentPage.getValueEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.VALUES);
@@ -120,8 +118,10 @@ public final class PageReadStatus {
readPosInBytes = endOfDefinitionLevels;
}
- readPosInBytes = 0;
- valuesRead = 0;
return true;
}
+
+ public void clear(){
+ this.dataReader.clear();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index cb05273..aa01115 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -35,6 +35,9 @@ import com.codahale.metrics.Timer;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.data.NamedExpression;
import org.apache.drill.exec.metrics.DrillMetrics;
import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.physical.OperatorCost;
@@ -81,6 +84,8 @@ public class ParquetGroupScan extends AbstractGroupScan {
private List<RowGroupInfo> rowGroupInfos;
private Stopwatch watch = new Stopwatch();
+ private List<SchemaPath> columns;
+
public List<ReadEntryWithPath> getEntries() {
return entries;
}
@@ -103,9 +108,11 @@ public class ParquetGroupScan extends AbstractGroupScan {
public ParquetGroupScan(@JsonProperty("entries") List<ReadEntryWithPath> entries,
@JsonProperty("storageengine") ParquetStorageEngineConfig storageEngineConfig,
@JacksonInject StorageEngineRegistry engineRegistry,
- @JsonProperty("ref") FieldReference ref
+ @JsonProperty("ref") FieldReference ref,
+ @JsonProperty("columns") List<SchemaPath> columns
)throws IOException, ExecutionSetupException {
engineRegistry.init(DrillConfig.create());
+ this.columns = columns;
this.storageEngine = (ParquetStorageEngine) engineRegistry.getEngine(storageEngineConfig);
this.availableEndpoints = storageEngine.getContext().getBits();
this.fs = storageEngine.getFileSystem();
@@ -116,9 +123,12 @@ public class ParquetGroupScan extends AbstractGroupScan {
calculateEndpointBytes();
}
- public ParquetGroupScan(List<ReadEntryWithPath> entries,
- ParquetStorageEngine storageEngine, FieldReference ref) throws IOException {
+ public ParquetGroupScan(List<ReadEntryWithPath> entries, //
+ ParquetStorageEngine storageEngine, //
+ FieldReference ref, //
+ List<SchemaPath> columns) throws IOException {
this.storageEngine = storageEngine;
+ this.columns = columns;
this.engineConfig = storageEngine.getEngineConfig();
this.availableEndpoints = storageEngine.getContext().getBits();
this.fs = storageEngine.getFileSystem();
@@ -362,7 +372,8 @@ public class ParquetGroupScan extends AbstractGroupScan {
logger.debug("minorFragmentId: {} Path: {} RowGroupIndex: {}",minorFragmentId, rg.getPath(),rg.getRowGroupIndex());
}
Preconditions.checkArgument(!mappings.get(minorFragmentId).isEmpty(), String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId));
- return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId), ref);
+ return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId), ref,
+ columns);
}
@@ -375,6 +386,10 @@ public class ParquetGroupScan extends AbstractGroupScan {
return rowGroupInfos.size();
}
+ public List<SchemaPath> getColumns() {
+ return columns;
+ }
+
@Override
public OperatorCost getCost() {
//TODO Figure out how to properly calculate cost
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
index 9a6b7ff..1e6c31a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.store.parquet;
import static com.google.common.base.Preconditions.checkArgument;
-import io.netty.buffer.ByteBuf;
import java.io.IOException;
import java.util.ArrayList;
@@ -39,7 +38,6 @@ import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.vector.ValueVector;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -58,7 +56,8 @@ public class ParquetRecordReader implements RecordReader {
private static final int NUMBER_OF_VECTORS = 1;
private static final long DEFAULT_BATCH_LENGTH = 256 * 1024 * NUMBER_OF_VECTORS; // 256kb
private static final long DEFAULT_BATCH_LENGTH_IN_BITS = DEFAULT_BATCH_LENGTH * 8; // 256kb
-
+ private static final char DEFAULT_RECORDS_TO_READ_IF_NOT_FIXED_WIDTH = 32*1024;
+
// TODO - should probably find a smarter way to set this, currently 1 megabyte
private static final int VAR_LEN_FIELD_LENGTH = 1024 * 1024 * 1;
public static final int PARQUET_PAGE_MAX_SIZE = 1024 * 1024 * 1;
@@ -73,18 +72,18 @@ public class ParquetRecordReader implements RecordReader {
private int bitWidthAllFixedFields;
private boolean allFieldsFixedLength;
private int recordsPerBatch;
- private ByteBuf bufferWithAllData;
private final FieldReference ref;
private long totalRecords;
private long rowGroupOffset;
private List<ColumnReader> columnStatuses;
- private FileSystem fileSystem;
- private BufferAllocator allocator;
+ FileSystem fileSystem;
+ BufferAllocator allocator;
private long batchSize;
- private Path hadoopPath;
+ Path hadoopPath;
private VarLenBinaryReader varLengthReader;
private ParquetMetadata footer;
+ private List<SchemaPath> columns;
public CodecFactoryExposer getCodecFactoryExposer() {
return codecFactoryExposer;
@@ -96,16 +95,17 @@ public class ParquetRecordReader implements RecordReader {
public ParquetRecordReader(FragmentContext fragmentContext,
String path, int rowGroupIndex, FileSystem fs,
- CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref) throws ExecutionSetupException {
- this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer, ref);
+ CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref,
+ List<SchemaPath> columns) throws ExecutionSetupException {
+ this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer, ref,
+ columns);
}
-
public ParquetRecordReader(FragmentContext fragmentContext, long batchSize,
String path, int rowGroupIndex, FileSystem fs,
- CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref) throws ExecutionSetupException {
+ CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref,
+ List<SchemaPath> columns) throws ExecutionSetupException {
this.allocator = fragmentContext.getAllocator();
-
hadoopPath = new Path(path);
fileSystem = fs;
this.ref = ref;
@@ -113,10 +113,7 @@ public class ParquetRecordReader implements RecordReader {
this.rowGroupIndex = rowGroupIndex;
this.batchSize = batchSize;
this.footer = footer;
- }
-
- public ByteBuf getBufferWithAllData() {
- return bufferWithAllData;
+ this.columns = columns;
}
public int getRowGroupIndex() {
@@ -149,22 +146,42 @@ public class ParquetRecordReader implements RecordReader {
}
}
+ private boolean fieldSelected(MaterializedField field){
+ // TODO - not sure if this is how we want to represent this
+ // for now it makes the existing tests pass, simply selecting
+ // all available data if no columns are provided
+ if (this.columns != null){
+ for (SchemaPath expr : this.columns){
+ if ( field.matches(expr)){
+ return true;
+ }
+ }
+ return false;
+ }
+ return true;
+ }
+
@Override
public void setup(OutputMutator output) throws ExecutionSetupException {
-
columnStatuses = new ArrayList<>();
-
totalRecords = footer.getBlocks().get(rowGroupIndex).getRowCount();
-
List<ColumnDescriptor> columns = footer.getFileMetaData().getSchema().getColumns();
allFieldsFixedLength = true;
ColumnDescriptor column;
ColumnChunkMetaData columnChunkMetaData;
+ int columnsToScan = 0;
+ MaterializedField field;
// loop to add up the length of the fixed width columns and build the schema
for (int i = 0; i < columns.size(); ++i) {
column = columns.get(i);
+ field = MaterializedField.create(toFieldName(column.getPath()),
+ toMajorType(column.getType(), getDataMode(column)));
+ if ( ! fieldSelected(field)){
+ continue;
+ }
+ columnsToScan++;
// sum the lengths of all of the fixed length fields
if (column.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
// There is not support for the fixed binary type yet in parquet, leaving a task here as a reminder
@@ -178,10 +195,13 @@ public class ParquetRecordReader implements RecordReader {
} else {
allFieldsFixedLength = false;
}
-
}
rowGroupOffset = footer.getBlocks().get(rowGroupIndex).getColumns().get(0).getFirstDataPageOffset();
+ // none of the columns in the parquet file matched the request columns from the query
+ if (columnsToScan == 0){
+ return;
+ }
if (allFieldsFixedLength) {
recordsPerBatch = (int) Math.min(batchSize / bitWidthAllFixedFields, footer.getBlocks().get(0).getColumns().get(0).getValueCount());
}
@@ -190,12 +210,14 @@ public class ParquetRecordReader implements RecordReader {
ArrayList<VarLenBinaryReader.NullableVarLengthColumn> nullableVarLengthColumns = new ArrayList<>();
// initialize all of the column read status objects
boolean fieldFixedLength = false;
- MaterializedField field;
for (int i = 0; i < columns.size(); ++i) {
column = columns.get(i);
columnChunkMetaData = footer.getBlocks().get(0).getColumns().get(i);
field = MaterializedField.create(toFieldName(column.getPath()),
toMajorType(column.getType(), getDataMode(column)));
+ // the field was not requested to be read
+ if ( ! fieldSelected(field)) continue;
+
fieldFixedLength = column.getType() != PrimitiveType.PrimitiveTypeName.BINARY;
ValueVector v = TypeHelper.getNewVector(field, allocator);
if (column.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
@@ -213,10 +235,8 @@ public class ParquetRecordReader implements RecordReader {
} catch (SchemaChangeException e) {
throw new ExecutionSetupException(e);
}
-
-
- output.removeAllFields();
+ output.removeAllFields();
try {
for (ColumnReader crs : columnStatuses) {
output.addField(crs.valueVecHolder.getValueVector());
@@ -232,48 +252,6 @@ public class ParquetRecordReader implements RecordReader {
throw new ExecutionSetupException("Error setting up output mutator.", e);
}
- // the method for reading into a ByteBuf from a stream copies all of the data into a giant buffer
- // here we do the same thing in a loop to not initialize so much on heap
-
- // TODO - this should be replaced by an enhancement in Hadoop 2.0 that will allow reading
- // directly into a ByteBuf passed into the reading method
- int totalByteLength = 0;
- long start = 0;
- if (rowGroupIndex == 0){
- totalByteLength = 4;
- }
- else{
- start = rowGroupOffset;
- }
- // TODO - the methods for get total size and get total uncompressed size seem to have the opposite results of
- // what they should
- // I found the bug in the mainline and made a issue for it, hopefully it will be fixed soon
- for (ColumnReader crs : columnStatuses){
- totalByteLength += crs.columnChunkMetaData.getTotalSize();
- }
- for (VarLenBinaryReader.VarLengthColumn r : varLengthReader.columns){
- totalByteLength += r.columnChunkMetaData.getTotalSize();
- }
- for (VarLenBinaryReader.NullableVarLengthColumn r : varLengthReader.nullableColumns){
- totalByteLength += r.columnChunkMetaData.getTotalSize();
- }
- int bufferSize = 64*1024;
- long totalBytesWritten = 0;
- int validBytesInCurrentBuffer;
- byte[] buffer = new byte[bufferSize];
-
- try (FSDataInputStream inputStream = fileSystem.open(hadoopPath)) {
- bufferWithAllData = allocator.buffer(totalByteLength);
- inputStream.seek(start);
- while (totalBytesWritten < totalByteLength){
- validBytesInCurrentBuffer = (int) Math.min(bufferSize, totalByteLength - totalBytesWritten);
- inputStream.read(buffer, 0 , validBytesInCurrentBuffer);
- bufferWithAllData.writeBytes(buffer, 0 , (int) validBytesInCurrentBuffer);
- totalBytesWritten += validBytesInCurrentBuffer;
- }
- } catch (IOException e) {
- throw new ExecutionSetupException("Error opening or reading metatdata for parquet file at location: " + hadoopPath.getName());
- }
}
private SchemaPath toFieldName(String[] paths) {
@@ -317,7 +295,7 @@ public class ParquetRecordReader implements RecordReader {
*/
private boolean createFixedColumnReader(boolean fixedLength, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, int allocateSize, ValueVector v)
- throws SchemaChangeException {
+ throws SchemaChangeException, ExecutionSetupException {
// if the column is required
if (descriptor.getMaxDefinitionLevel() == 0){
if (columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.BOOLEAN){
@@ -371,8 +349,7 @@ public class ParquetRecordReader implements RecordReader {
if (allFieldsFixedLength) {
recordsToRead = Math.min(recordsPerBatch, firstColumnStatus.columnChunkMetaData.getValueCount() - firstColumnStatus.totalValuesRead);
} else {
- // arbitrary
- recordsToRead = 4000;
+ recordsToRead = DEFAULT_RECORDS_TO_READ_IF_NOT_FIXED_WIDTH;
// going to incorporate looking at length of values and copying the data into a single loop, hopefully it won't
// get too complicated
@@ -504,8 +481,17 @@ public class ParquetRecordReader implements RecordReader {
@Override
public void cleanup() {
+ for (ColumnReader column : columnStatuses) {
+ column.clear();
+ }
columnStatuses.clear();
- bufferWithAllData.release();
+
+ for (VarLenBinaryReader.VarLengthColumn r : varLengthReader.columns){
+ r.clear();
+ }
+ for (VarLenBinaryReader.NullableVarLengthColumn r : varLengthReader.nullableColumns){
+ r.clear();
+ }
varLengthReader.columns.clear();
varLengthReader.nullableColumns.clear();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
index 9a94ec2..b3ce9b4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
@@ -23,8 +23,9 @@ import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.exception.SetupException;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.ReadEntryFromHDFS;
import org.apache.drill.exec.physical.base.AbstractBase;
@@ -51,22 +52,31 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
private final ParquetStorageEngine parquetStorageEngine;
private final List<RowGroupReadEntry> rowGroupReadEntries;
private final FieldReference ref;
+ private final List<SchemaPath> columns;
@JsonCreator
- public ParquetRowGroupScan(@JacksonInject StorageEngineRegistry registry, @JsonProperty("engineConfig") StorageEngineConfig engineConfig,
- @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries, @JsonProperty("ref") FieldReference ref) throws ExecutionSetupException {
+ public ParquetRowGroupScan(@JacksonInject StorageEngineRegistry registry,
+ @JsonProperty("engineConfig") StorageEngineConfig engineConfig,
+ @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries,
+ @JsonProperty("ref") FieldReference ref,
+ @JsonProperty("columns") List<SchemaPath> columns
+ ) throws ExecutionSetupException {
parquetStorageEngine = (ParquetStorageEngine) registry.getEngine(engineConfig);
this.rowGroupReadEntries = rowGroupReadEntries;
this.engineConfig = engineConfig;
this.ref = ref;
+ this.columns = columns;
}
public ParquetRowGroupScan(ParquetStorageEngine engine, ParquetStorageEngineConfig config,
- List<RowGroupReadEntry> rowGroupReadEntries, FieldReference ref) {
+ List<RowGroupReadEntry> rowGroupReadEntries, FieldReference ref,
+ List<SchemaPath> columns
+ ) {
parquetStorageEngine = engine;
engineConfig = config;
this.rowGroupReadEntries = rowGroupReadEntries;
this.ref = ref;
+ this.columns = columns;
}
public List<RowGroupReadEntry> getRowGroupReadEntries() {
@@ -108,9 +118,10 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
}
@Override
- public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
+ public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.isEmpty());
- return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries, ref);
+ return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries,
+ ref, columns);
}
@Override
@@ -118,6 +129,10 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
return Iterators.emptyIterator();
}
+ public List<SchemaPath> getColumns() {
+ return columns;
+ }
+
public static class RowGroupReadEntry extends ReadEntryFromHDFS {
private int rowGroupIndex;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index f73ee84..966a16b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -20,7 +20,9 @@ package org.apache.drill.exec.store.parquet;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
@@ -48,6 +50,8 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
public RecordBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan, List<RecordBatch> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.isEmpty());
List<RecordReader> readers = Lists.newArrayList();
+ // keep footers in a map to avoid re-reading them
+ Map<String, ParquetMetadata> footers = new HashMap<String, ParquetMetadata>();
for(ParquetRowGroupScan.RowGroupReadEntry e : rowGroupScan.getRowGroupReadEntries()){
/*
Here we could store a map from file names to footers, to prevent re-reading the footer for each row group in a file
@@ -57,12 +61,17 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
These fields will be added to the constructor below
*/
try {
+ if ( ! footers.containsKey(e.getPath())){
+ footers.put(e.getPath(),
+ ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath())));
+ }
readers.add(
new ParquetRecordReader(
context, e.getPath(), e.getRowGroupIndex(), rowGroupScan.getStorageEngine().getFileSystem(),
rowGroupScan.getStorageEngine().getCodecFactoryExposer(),
- ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath())),
- rowGroupScan.getRef()
+ footers.get(e.getPath()),
+ rowGroupScan.getRef(),
+ rowGroupScan.getColumns()
)
);
} catch (IOException e1) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
index 45b9cc1..ad9756e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
@@ -27,6 +27,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.common.logical.data.Scan;
import org.apache.drill.exec.ops.FragmentContext;
@@ -58,7 +59,6 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
private final DrillbitContext context;
static final ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
private CodecFactoryExposer codecFactoryExposer;
- final ParquetMetadata footer;
private final ParquetSchemaProvider schemaProvider;
private final ParquetStorageEngineConfig engineConfig;
@@ -67,7 +67,6 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
this.schemaProvider = new ParquetSchemaProvider(configuration, context.getConfig());
codecFactoryExposer = new CodecFactoryExposer(schemaProvider.conf);
this.engineConfig = configuration;
- this.footer = null;
}
public Configuration getHadoopConfig() {
@@ -99,7 +98,7 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
ArrayList<ReadEntryWithPath> readEntries = scan.getSelection().getListWith(new ObjectMapper(),
new TypeReference<ArrayList<ReadEntryWithPath>>() {});
- return new ParquetGroupScan( Lists.newArrayList(readEntries), this, scan.getOutputReference());
+ return new ParquetGroupScan( readEntries, this, scan.getOutputReference(), null);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
index 276a4df..0321838 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.vector.NullableVarBinaryVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
@@ -43,7 +44,7 @@ public class VarLenBinaryReader {
public static class VarLengthColumn extends ColumnReader {
- VarLengthColumn(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) {
+ VarLengthColumn(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
}
@@ -58,7 +59,7 @@ public class VarLenBinaryReader {
int nullsRead;
boolean currentValNull = false;
- NullableVarLengthColumn(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) {
+ NullableVarLengthColumn(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
}
@@ -118,6 +119,12 @@ public class VarLenBinaryReader {
lengthVarFieldsInCurrentRecord += columnReader.dataTypeLengthInBits;
}
for (NullableVarLengthColumn columnReader : nullableColumns) {
+ // check to make sure there is capacity for the next value (for nullables this is a check to see if there is
+ // still space in the nullability recording vector)
+ if (recordsReadInCurrentPass == columnReader.valueVecHolder.getValueVector().getValueCapacity()){
+ rowGroupFinished = true;
+ break;
+ }
if (columnReader.pageReadStatus.currentPage == null
|| columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) {
columnReader.totalValuesRead += columnReader.pageReadStatus.valuesRead;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
index 2be3e8d..62b9dfa 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
@@ -33,6 +33,9 @@ import mockit.Expectations;
import mockit.Injectable;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.exception.SchemaChangeException;
@@ -136,7 +139,7 @@ public class JSONRecordReaderTest {
};
JSONRecordReader jr = new JSONRecordReader(context,
FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null);
+ FileSystem.getLocal(new Configuration()), null, null);
MockOutputMutator mutator = new MockOutputMutator();
List<ValueVector> addFields = mutator.getAddFields();
@@ -166,7 +169,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context,
FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null);
+ FileSystem.getLocal(new Configuration()), null, null);
MockOutputMutator mutator = new MockOutputMutator();
List<ValueVector> addFields = mutator.getAddFields();
@@ -196,6 +199,41 @@ public class JSONRecordReaderTest {
}
@Test
+ public void testChangedSchemaInTwoBatchesColumnSelect(@Injectable final FragmentContext context) throws IOException,
+ ExecutionSetupException {
+ new Expectations() {
+ {
+ context.getAllocator();
+ returns(new DirectBufferAllocator());
+ }
+ };
+
+ JSONRecordReader jr = new JSONRecordReader(context,
+ FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
+ FileSystem.getLocal(new Configuration()),
+ 64, null, Arrays.asList(new SchemaPath("test", ExpressionPosition.UNKNOWN))); // batch only fits 1 int
+ MockOutputMutator mutator = new MockOutputMutator();
+ List<ValueVector> addFields = mutator.getAddFields();
+ List<MaterializedField> removedFields = mutator.getRemovedFields();
+
+ jr.setup(mutator);
+ assertEquals(1, jr.next());
+ assertField(addFields.get(0), 0, MinorType.INT, 123, "test");
+ assertTrue(removedFields.isEmpty());
+ assertEquals(addFields.size(), 1);
+ assertEquals(1, jr.next());
+ assertField(addFields.get(0), 0, MinorType.INT, 1234, "test");
+ assertEquals(addFields.size(), 1);
+ assertTrue(removedFields.isEmpty());
+ removedFields.clear();
+ assertEquals(1, jr.next());
+ assertField(addFields.get(0), 0, MinorType.INT, 12345, "test");
+ assertEquals(addFields.size(), 1);
+ assertTrue(removedFields.isEmpty());
+ assertEquals(0, jr.next());
+ }
+
+ @Test
public void testChangedSchemaInTwoBatches(@Injectable final FragmentContext context) throws IOException,
ExecutionSetupException {
new Expectations() {
@@ -208,7 +246,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context,
FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
FileSystem.getLocal(new Configuration()),
- 64, null); // batch only fits 1 int
+ 64, null, null); // batch only fits 1 int
MockOutputMutator mutator = new MockOutputMutator();
List<ValueVector> addFields = mutator.getAddFields();
List<MaterializedField> removedFields = mutator.getRemovedFields();
@@ -266,7 +304,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context,
FileUtils.getResourceAsFile("/scan_json_test_3.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null);
+ FileSystem.getLocal(new Configuration()), null, null);
MockOutputMutator mutator = new MockOutputMutator();
List<ValueVector> addFields = mutator.getAddFields();
@@ -295,7 +333,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context,
FileUtils.getResourceAsFile("/scan_json_test_4.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null);
+ FileSystem.getLocal(new Configuration()), null, null);
MockOutputMutator mutator = new MockOutputMutator();
List<ValueVector> addFields = mutator.getAddFields();
@@ -328,7 +366,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context,
FileUtils.getResourceAsFile("/scan_json_test_5.json").toURI().toString(),
- FileSystem.getLocal(new Configuration()), null);
+ FileSystem.getLocal(new Configuration()), null, null);
MockOutputMutator mutator = new MockOutputMutator();
List<ValueVector> addFields = mutator.getAddFields();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
index 8efc762..6f568fb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
@@ -151,6 +151,32 @@ public class ParquetRecordReaderTest {
testParquetFullEngine(false, "/parquet_nullable_varlen.json", "/tmp/nullable.parquet", 1, props);
}
+ /**
+ * Tests the attribute in a scan node to limit the columns read by a scan.
+ *
+ * The functionality of selecting all columns is tested in all of the other tests that leave out the attribute.
+ * @throws Exception
+ */
+ @Test
+ public void testSelectColumnRead() throws Exception {
+ HashMap<String, FieldInfo> fields = new HashMap<>();
+ ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ // generate metatdata for a series of test columns, these columns are all generated in the test file
+ populateFieldInfoMap(props);
+ generateParquetFile("/tmp/test.parquet", props);
+ fields.clear();
+ // create a new object to describe the dataset expected out of the scan operation
+ // the fields added below match those requested in the plan specified in parquet_selective_column_read.json
+ // that is used below in the test query
+ props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
+ props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
+ props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
+ props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
+ props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
+ testParquetFullEngineEventBased(false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
+ }
+
+
@Test
public void testMultipleRowGroupsAndReads() throws Exception {
HashMap<String, FieldInfo> fields = new HashMap<>();
@@ -165,7 +191,7 @@ public class ParquetRecordReaderTest {
readEntries += ",";
}
testParquetFullEngineEventBased(true, "/parquet_scan_screen_read_entry_replace.json", readEntries,
- "/tmp/test.parquet", i, props);
+ "/tmp/test.parquet", i, props, true);
}
// requires binary file generated by pig from TPCH data, also have to disable assert where data is coming in
@@ -185,18 +211,18 @@ public class ParquetRecordReaderTest {
readEntries += ",";
}
testParquetFullEngineEventBased(false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
- "/tmp/test.parquet", i, props);
+ "/tmp/test.parquet", i, props, true);
}
-
+/*
@Test
public void testMultipleRowGroupsEvent() throws Exception {
HashMap<String, FieldInfo> fields = new HashMap<>();
- ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
- populateFieldInfoMap(props);
- testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+ ParquetTestProperties props = new ParquetTestProperties(10, 30000, DEFAULT_BYTES_PER_PAGE, fields);
+ props.fields.put("a", new FieldInfo("a", "asdf", 1, new Object[3], TypeProtos.MinorType.BIGINT, props));
+ testParquetFullEngineEventBased(false, "/parquet_scan_screen.json", "/tmp/out", 1, props);
}
-
+*/
private class ParquetTestProperties{
int numberRowGroups;
int recordsPerRowGroup;
@@ -462,14 +488,17 @@ public class ParquetRecordReaderTest {
}
}
+
+
// specific tests should call this method, but it is not marked as a test itself intentionally
public void testParquetFullEngineEventBased(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
- testParquetFullEngine(generateNew, plan, null, filename, numberOfTimesRead, props);
+ testParquetFullEngineEventBased(generateNew, plan, null, filename, numberOfTimesRead, props, true);
}
// specific tests should call this method, but it is not marked as a test itself intentionally
public void testParquetFullEngineEventBased(boolean generateNew, String plan, String readEntries, String filename,
- int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
+ int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props,
+ boolean runAsLogicalPlan) throws Exception{
RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
if (generateNew) generateParquetFile(filename, props);
@@ -482,12 +511,15 @@ public class ParquetRecordReaderTest {
RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props);
long C = System.nanoTime();
- if (readEntries != null){
- client.runQuery(UserProtos.QueryType.LOGICAL, (Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8).replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries)), resultListener);
- }
- else{
- client.runQuery(UserProtos.QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8), resultListener);
+ String planText = Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8);
+ // substitute in the string for the read entries, allows reuse of the plan file for several tests
+ if (readEntries != null) {
+ planText = planText.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries);
}
+ if (runAsLogicalPlan)
+ client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener);
+ else
+ client.runQuery(UserProtos.QueryType.PHYSICAL, planText, resultListener);
resultListener.getResults();
for (String s : resultListener.valuesChecked.keySet()) {
assertEquals("Record count incorrect for column: " + s,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_nullable_varlen.json b/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
new file mode 100644
index 0000000..c21beec
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
@@ -0,0 +1,41 @@
+{
+ head:{
+ type:"APACHE_DRILL_LOGICAL",
+ version:"1",
+ generator:{
+ type:"manual",
+ info:"na"
+ }
+ },
+ storage:{
+ "parquet" :
+ {
+ "type":"parquet",
+ "dfsName" : "file:///"
+ }
+ },
+ query:[
+ {
+ @id:"1",
+ op:"scan",
+ memo:"initial_scan",
+ storageengine:"parquet",
+ selection: [
+ {
+ path: "/tmp/nullable_varlen.parquet"
+ }
+ ]
+ },
+ {
+ @id:"2",
+ input: 1,
+ op: "store",
+ memo: "output sink",
+ target: {
+ file: "console:///stdout"
+ }
+
+ }
+
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2dfadc9d/exec/java-exec/src/test/resources/parquet_selective_column_read.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_selective_column_read.json b/exec/java-exec/src/test/resources/parquet_selective_column_read.json
new file mode 100644
index 0000000..61e582d
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet_selective_column_read.json
@@ -0,0 +1,28 @@
+{
+ head : {
+ version : 1,
+ generator : {
+ type : "manual",
+ info : "na"
+ },
+ type : "APACHE_DRILL_PHYSICAL"
+ },
+ graph : [ {
+ pop : "parquet-scan",
+ @id : 1,
+ entries : [ {
+ path : "/tmp/test.parquet"
+ } ],
+ storageengine : {
+ type : "parquet",
+ dfsName : "file:///"
+ },
+ columns: [ "integer", "bigInt", "bin", "bin2"],
+ fragmentPointer : 0
+ }, {
+ pop : "screen",
+ @id : 2,
+ child : 1
+ } ]
+}
+
[24/38] Integrate new SQL changes with Hive storage engine,
moving to automatic file detection. Rename storage engine to storage
plugin. Separate storage plugins from format plugins,
updating Parquet and JSON to format engines. Refactor distribution log
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java
new file mode 100644
index 0000000..97124bc
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import org.apache.drill.exec.store.dfs.easy.FileWork;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class ReadEntryFromHDFS extends ReadEntryWithPath implements FileWork{
+
+ private long start;
+ private long length;
+
+ @JsonCreator
+ public ReadEntryFromHDFS(@JsonProperty("path") String path,@JsonProperty("start") long start, @JsonProperty("length") long length) {
+ this.path = path;
+ this.start = start;
+ this.length = length;
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public long getLength() {
+ return length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java
new file mode 100644
index 0000000..bf1d762
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+
+public class ReadEntryWithPath {
+
+ protected String path;
+
+
+ public ReadEntryWithPath(String path) {
+ super();
+ this.path = path;
+ }
+
+ public ReadEntryWithPath(){}
+
+ public String getPath(){
+ return path;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
new file mode 100644
index 0000000..c69edb7
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.planner.logical.DynamicDrillTable;
+import org.apache.drill.exec.planner.sql.ExpandingConcurrentMap;
+import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.SchemaHolder;
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.beust.jcommander.internal.Lists;
+
+public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFactory<String, DrillTable> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WorkspaceSchemaFactory.class);
+
+ private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<String, DrillTable>(this);
+ private final List<FormatMatcher> fileMatchers;
+ private final List<FormatMatcher> dirMatchers;
+
+ private final Path root;
+ private final DrillFileSystem fs;
+ private final String storageEngineName;
+ private final String schemaName;
+
+ public WorkspaceSchemaFactory(String schemaName, String storageEngineName, DrillFileSystem fileSystem, String path,
+ List<FormatMatcher> formatMatchers) throws ExecutionSetupException {
+ this.fs = fileSystem;
+ this.root = new Path(path);
+ this.fileMatchers = Lists.newArrayList();
+ this.dirMatchers = Lists.newArrayList();
+ for (FormatMatcher m : formatMatchers) {
+ if (m.supportDirectoryReads()) {
+ dirMatchers.add(m);
+ }
+ fileMatchers.add(m);
+ }
+ this.storageEngineName = storageEngineName;
+ this.schemaName = schemaName;
+ }
+
+ public WorkspaceSchema create(SchemaHolder holder) {
+ return new WorkspaceSchema(holder, schemaName);
+ }
+
+ @Override
+ public DrillTable create(String key) {
+ try {
+
+ FileSelection fileSelection = FileSelection.create(fs, root, key);
+ if(fileSelection == null) return null;
+
+ if (fileSelection.containsDirectories(fs)) {
+ for (FormatMatcher m : dirMatchers) {
+ try {
+ Object selection = m.isReadable(fileSelection);
+ if (selection != null)
+ return new DynamicDrillTable(storageEngineName, selection, m.getFormatPlugin().getStorageConfig());
+ } catch (IOException e) {
+ logger.debug("File read failed.", e);
+ }
+ }
+ fileSelection = fileSelection.minusDirectorries(fs);
+ }
+
+ for (FormatMatcher m : fileMatchers) {
+ Object selection = m.isReadable(fileSelection);
+ if (selection != null)
+ return new DynamicDrillTable(storageEngineName, selection, m.getFormatPlugin().getStorageConfig());
+ }
+ return null;
+
+ } catch (IOException e) {
+ logger.debug("Failed to create DrillTable with root {} and name {}", root, key, e);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void destroy(DrillTable value) {
+ }
+
+ public class WorkspaceSchema extends AbstractSchema {
+
+ public WorkspaceSchema(SchemaHolder parentSchema, String name) {
+ super(parentSchema, name);
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return tables.keySet();
+ }
+
+ @Override
+ public DrillTable getTable(String name) {
+ return tables.get(name);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyBatchCreator.java
new file mode 100644
index 0000000..d79e542
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyBatchCreator.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.easy;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.record.RecordBatch;
+
+public class EasyBatchCreator implements BatchCreator<EasySubScan>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasyBatchCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, EasySubScan config, List<RecordBatch> children)
+ throws ExecutionSetupException {
+ assert children == null || children.isEmpty();
+ return config.getFormatPlugin().getBatch(context, config);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
new file mode 100644
index 0000000..8a41575
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.easy;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
+import org.apache.drill.exec.physical.impl.ScanBatch;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.QueryOptimizerRule;
+import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.store.dfs.BasicFormatMatcher;
+import org.apache.drill.exec.store.dfs.FileSelection;
+import org.apache.drill.exec.store.dfs.FormatMatcher;
+import org.apache.drill.exec.store.dfs.FormatPlugin;
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+
+import com.beust.jcommander.internal.Lists;
+
+public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements FormatPlugin {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasyFormatPlugin.class);
+
+ private final BasicFormatMatcher matcher;
+ private final DrillbitContext context;
+ private final boolean readable;
+ private final boolean writable;
+ private final boolean blockSplittable;
+ private final DrillFileSystem fs;
+ private final StoragePluginConfig storageConfig;
+ private final FormatPluginConfig formatConfig;
+ private final String name;
+
+ protected EasyFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, T formatConfig, boolean readable, boolean writable, boolean blockSplittable, String extension, String defaultName){
+ this.matcher = new BasicFormatMatcher(this, fs, extension);
+ this.readable = readable;
+ this.writable = writable;
+ this.context = context;
+ this.blockSplittable = blockSplittable;
+ this.fs = fs;
+ this.storageConfig = storageConfig;
+ this.formatConfig = formatConfig;
+ this.name = name == null ? defaultName : name;
+ }
+
+ @Override
+ public DrillFileSystem getFileSystem() {
+ return fs;
+ }
+
+ @Override
+ public DrillbitContext getContext() {
+ return context;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Whether or not you can split the format based on blocks within file boundaries. If not, the simple format engine will
+ * only split on file boundaries.
+ *
+ * @return True if splittable.
+ */
+ public boolean isBlockSplittable(){
+ return blockSplittable;
+ };
+
+ public abstract RecordReader getRecordReader(FragmentContext context, FileWork fileWork, FieldReference ref, List<SchemaPath> columns) throws ExecutionSetupException;
+
+
+ RecordBatch getBatch(FragmentContext context, EasySubScan scan) throws ExecutionSetupException {
+ List<RecordReader> readers = Lists.newArrayList();
+ for(FileWork work : scan.getWorkUnits()){
+ readers.add(getRecordReader(context, work, scan.getRef(), scan.getColumns()));
+ }
+
+ return new ScanBatch(context, readers.iterator());
+ }
+
+ @Override
+ public AbstractGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException {
+ return new EasyGroupScan(selection, this, outputRef, null);
+ }
+
+ @Override
+ public FormatPluginConfig getConfig() {
+ return formatConfig;
+ }
+
+ @Override
+ public StoragePluginConfig getStorageConfig() {
+ return storageConfig;
+ }
+
+ @Override
+ public boolean supportsRead() {
+ return readable;
+ }
+
+ @Override
+ public boolean supportsWrite() {
+ return writable;
+ }
+
+ @Override
+ public FormatMatcher getMatcher() {
+ return matcher;
+ }
+
+ @Override
+ public List<QueryOptimizerRule> getOptimizerRules() {
+ return Collections.emptyList();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
new file mode 100644
index 0000000..5fcf7a5
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.easy;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.physical.EndpointAffinity;
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.dfs.FileSelection;
+import org.apache.drill.exec.store.schedule.AffinityCreator;
+import org.apache.drill.exec.store.schedule.AssignmentCreator;
+import org.apache.drill.exec.store.schedule.BlockMapBuilder;
+import org.apache.drill.exec.store.schedule.CompleteFileWork;
+import org.apache.drill.exec.store.schedule.CompleteFileWork.FileWorkImpl;
+
+import com.beust.jcommander.internal.Lists;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ListMultimap;
+
+@JsonTypeName("fs-scan")
+public class EasyGroupScan extends AbstractGroupScan{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasyGroupScan.class);
+
+ private final FileSelection selection;
+ private final EasyFormatPlugin<?> formatPlugin;
+ private final FieldReference ref;
+ private final int maxWidth;
+ private final List<SchemaPath> columns;
+
+ private ListMultimap<Integer, CompleteFileWork> mappings;
+ private List<CompleteFileWork> chunks;
+ private List<EndpointAffinity> endpointAffinities;
+
+ @JsonCreator
+ public EasyGroupScan(
+ @JsonProperty("files") List<String> files, //
+ @JsonProperty("storage") StoragePluginConfig storageConfig, //
+ @JsonProperty("format") FormatPluginConfig formatConfig, //
+ @JacksonInject StoragePluginRegistry engineRegistry, //
+ @JsonProperty("ref") FieldReference ref, //
+ @JsonProperty("columns") List<SchemaPath> columns
+ ) throws IOException, ExecutionSetupException {
+
+ this.formatPlugin = (EasyFormatPlugin<?>) engineRegistry.getFormatPlugin(storageConfig, formatConfig);
+ this.selection = new FileSelection(files, true);
+ this.maxWidth = selection.getFileStatusList(formatPlugin.getFileSystem()).size();
+ this.ref = ref;
+ this.columns = columns;
+ }
+
+ public EasyGroupScan(
+ FileSelection selection, //
+ EasyFormatPlugin<?> formatPlugin, //
+ FieldReference ref, //
+ List<SchemaPath> columns
+ ) throws IOException{
+ this.selection = selection;
+ this.maxWidth = selection.getFileStatusList(formatPlugin.getFileSystem()).size();
+ this.formatPlugin = formatPlugin;
+ this.ref = ref;
+ this.columns = columns;
+ }
+
+ @Override
+ public int getMaxParallelizationWidth() {
+ return maxWidth;
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ return new OperatorCost(1,1,1,1);
+ }
+
+ @Override
+ public Size getSize() {
+ return new Size(1024,1024);
+ }
+
+ @JsonProperty("files")
+ public List<String> getFiles() {
+ return selection.getAsFiles();
+ }
+
+ @JsonIgnore
+ public FileSelection getFileSelection(){
+ return selection;
+ }
+
+ @Override
+ public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
+ assert children == null || children.isEmpty();
+ return this;
+ }
+
+ @JsonProperty("columns")
+ public List<SchemaPath> getColumns(){
+ return columns;
+ }
+
+ @Override
+ public List<EndpointAffinity> getOperatorAffinity() {
+ if (this.endpointAffinities == null) {
+ try{
+ BlockMapBuilder b = new BlockMapBuilder(formatPlugin.getFileSystem().getUnderlying(), formatPlugin.getContext().getBits());
+ this.chunks = b.generateFileWork(selection.getFileStatusList(formatPlugin.getFileSystem()), formatPlugin.isBlockSplittable());
+ this.endpointAffinities = AffinityCreator.getAffinityMap(chunks);
+ }catch(IOException e){
+ logger.warn("Failure determining endpoint affinity.", e);
+ this.endpointAffinities = Collections.emptyList();
+ }
+ }
+ return this.endpointAffinities;
+ }
+
+ @Override
+ public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) {
+ this.mappings = AssignmentCreator.getMappings(incomingEndpoints, chunks);
+ }
+
+ @Override
+ public EasySubScan getSpecificScan(int minorFragmentId) {
+ assert minorFragmentId < mappings.size() : String.format(
+ "Mappings length [%d] should be longer than minor fragment id [%d] but it isn't.", mappings.size(),
+ minorFragmentId);
+
+ List<CompleteFileWork> filesForMinor = mappings.get(minorFragmentId);
+
+ Preconditions.checkArgument(!filesForMinor.isEmpty(),
+ String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId));
+
+ return new EasySubScan(convert(filesForMinor), formatPlugin, ref, columns);
+ }
+
+ private List<FileWorkImpl> convert(List<CompleteFileWork> list){
+ List<FileWorkImpl> newList = Lists.newArrayList();
+ for(CompleteFileWork f : list){
+ newList.add(f.getAsFileWork());
+ }
+ return newList;
+ }
+
+ @JsonProperty("storage")
+ public StoragePluginConfig getStorageConfig(){
+ return formatPlugin.getStorageConfig();
+ }
+
+ @JsonProperty("format")
+ public FormatPluginConfig getFormatConfig(){
+ return formatPlugin.getConfig();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java
new file mode 100644
index 0000000..72d1fe6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.easy;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.physical.base.AbstractSubScan;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.schedule.CompleteFileWork.FileWorkImpl;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.Preconditions;
+
+@JsonTypeName("fs-sub-scan")
+public class EasySubScan extends AbstractSubScan{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasySubScan.class);
+
+ private final List<FileWorkImpl> files;
+ private final EasyFormatPlugin<?> formatPlugin;
+ private final FieldReference ref;
+ private final List<SchemaPath> columns;
+
+ @JsonCreator
+ public EasySubScan(
+ @JsonProperty("files") List<FileWorkImpl> files, //
+ @JsonProperty("storage") StoragePluginConfig storageConfig, //
+ @JsonProperty("format") FormatPluginConfig formatConfig, //
+ @JacksonInject StoragePluginRegistry engineRegistry, //
+ @JsonProperty("ref") FieldReference ref, //
+ @JsonProperty("columns") List<SchemaPath> columns //
+ ) throws IOException, ExecutionSetupException {
+
+ this.formatPlugin = (EasyFormatPlugin<?>) engineRegistry.getFormatPlugin(storageConfig, formatConfig);
+ Preconditions.checkNotNull(this.formatPlugin);
+ this.files = files;
+ this.ref = ref;
+ this.columns = columns;
+ }
+
+ public EasySubScan(List<FileWorkImpl> files, EasyFormatPlugin<?> plugin, FieldReference ref, List<SchemaPath> columns){
+ this.formatPlugin = plugin;
+ this.files = files;
+ this.ref = ref;
+ this.columns = columns;
+ }
+
+ @JsonIgnore
+ public EasyFormatPlugin<?> getFormatPlugin(){
+ return formatPlugin;
+ }
+
+ @JsonProperty("files")
+ public List<FileWorkImpl> getWorkUnits() {
+ return files;
+ }
+
+ @JsonProperty("storage")
+ public StoragePluginConfig getStorageConfig(){
+ return formatPlugin.getStorageConfig();
+ }
+
+ @JsonProperty("format")
+ public FormatPluginConfig getFormatConfig(){
+ return formatPlugin.getConfig();
+ }
+
+ @JsonProperty("ref")
+ public FieldReference getRef() {
+ return ref;
+ }
+
+ @JsonProperty("columns")
+ public List<SchemaPath> getColumns(){
+ return columns;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java
new file mode 100644
index 0000000..170d339
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.easy;
+
+
+public interface FileWork {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileWork.class);
+
+ public String getPath();
+ public long getStart();
+ public long getLength();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java
new file mode 100644
index 0000000..8af6aaf
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.shim;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Wraps the underlying filesystem to provide advanced file system features. Delegates to underlying file system if
+ * those features are exposed.
+ */
+public abstract class DrillFileSystem implements AutoCloseable{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFileSystem.class);
+
+ public abstract FileSystem getUnderlying();
+
+ public abstract BlockLocation[] getBlockLocations(FileStatus status, long start, long length) throws IOException;
+ public abstract List<FileStatus> list(boolean recursive, Path... paths) throws IOException;
+ public abstract FileStatus getFileStatus(Path p) throws IOException;
+ public abstract DrillOutputStream create(Path p) throws IOException;
+ public abstract DrillInputStream open(Path p) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java
new file mode 100644
index 0000000..82764a3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.shim;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+
+public abstract class DrillInputStream implements AutoCloseable{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillInputStream.class);
+
+// public abstract AccountingByteBuf readNow(long start, long length) throws IOException;
+// public abstract void readNow(AccountingByteBuf b, long start, long length) throws IOException;
+// public abstract AccountingByteBuf readNow() throws IOException;
+
+ public abstract FSDataInputStream getInputStream();
+// public abstract CheckedFuture<Long, IOException> readFuture(AccountingByteBuf b, long start, long length) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java
new file mode 100644
index 0000000..c2446e9
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.shim;
+
+import java.io.OutputStream;
+
+
+public abstract class DrillOutputStream implements AutoCloseable{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillOutputStream.class);
+
+ public abstract OutputStream getOuputStream();
+// public abstract CheckedFuture<Long, IOException> writeFuture(AccountingByteBuf b);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java
new file mode 100644
index 0000000..eaab5ef
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.shim;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.store.dfs.shim.fallback.FallbackFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+public class FileSystemCreator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemCreator.class);
+
+ public static DrillFileSystem getFileSystem(DrillConfig config, Configuration fsConf) throws IOException{
+ FileSystem fs = FileSystem.get(fsConf);
+ return new FallbackFileSystem(config, fs);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java
new file mode 100644
index 0000000..5743ca1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs.shim.fallback;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+import org.apache.drill.exec.store.dfs.shim.DrillInputStream;
+import org.apache.drill.exec.store.dfs.shim.DrillOutputStream;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.collect.Lists;
+
+public class FallbackFileSystem extends DrillFileSystem {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FallbackFileSystem.class);
+
+ final FileSystem fs;
+
+ public FallbackFileSystem(DrillConfig config, FileSystem fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ public FileSystem getUnderlying() {
+ return fs;
+ }
+
+ @Override
+ public List<FileStatus> list(boolean recursive, Path... paths) throws IOException {
+ if (recursive) {
+ List<FileStatus> statuses = Lists.newArrayList();
+ for (Path p : paths) {
+ addRecursiveStatus(fs.getFileStatus(p), statuses);
+ }
+ return statuses;
+
+ } else {
+ return Lists.newArrayList(fs.listStatus(paths));
+ }
+ }
+
+
+ private void addRecursiveStatus(FileStatus parent, List<FileStatus> listToFill) throws IOException {
+ if (parent.isDir()) {
+ Path pattern = new Path(parent.getPath(), "/*");
+ FileStatus[] sub = fs.globStatus(pattern);
+ for(FileStatus s : sub){
+ listToFill.add(s);
+ }
+ } else {
+ listToFill.add(parent);
+ }
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path p) throws IOException {
+ return fs.getFileStatus(p);
+ }
+
+ @Override
+ public DrillOutputStream create(Path p) throws IOException {
+ return new Out(fs.create(p));
+ }
+
+ @Override
+ public DrillInputStream open(Path p) throws IOException {
+ return new In(fs.open(p));
+ }
+
+ @Override
+ public void close() throws Exception {
+ fs.close();
+ }
+
+ @Override
+ public BlockLocation[] getBlockLocations(FileStatus status, long start, long len) throws IOException {
+ return fs.getFileBlockLocations(status, start, len);
+ }
+
+ private class Out extends DrillOutputStream {
+
+ private final FSDataOutputStream out;
+
+ public Out(FSDataOutputStream out) {
+ super();
+ this.out = out;
+ }
+
+ @Override
+ public void close() throws Exception {
+ out.close();
+ }
+
+ @Override
+ public FSDataOutputStream getOuputStream() {
+ return out;
+ }
+
+ }
+
+ private class In extends DrillInputStream {
+
+ private final FSDataInputStream in;
+
+ public In(FSDataInputStream in) {
+ super();
+ this.in = in;
+ }
+
+ @Override
+ public FSDataInputStream getInputStream() {
+ return in;
+ }
+
+ @Override
+ public void close() throws Exception {
+ in.close();
+ }
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
new file mode 100644
index 0000000..82bf3bf
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.easy.json;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.store.dfs.easy.EasyFormatPlugin;
+import org.apache.drill.exec.store.dfs.easy.FileWork;
+import org.apache.drill.exec.store.dfs.shim.DrillFileSystem;
+import org.apache.drill.exec.store.easy.json.JSONFormatPlugin.JSONFormatConfig;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
+
+ public JSONFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig) {
+ this(name, context, fs, storageConfig, new JSONFormatConfig());
+ }
+
+ public JSONFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig config, JSONFormatConfig formatPluginConfig) {
+ super(name, context, fs, config, formatPluginConfig, true, false, false, "json", "json");
+ }
+
+ @Override
+ public RecordReader getRecordReader(FragmentContext context, FileWork fileWork, FieldReference ref,
+ List<SchemaPath> columns) throws ExecutionSetupException {
+ return new JSONRecordReader(context, fileWork.getPath(), this.getFileSystem().getUnderlying(), ref, columns);
+ }
+
+ @JsonTypeName("json")
+ public static class JSONFormatConfig implements FormatPluginConfig {
+
+ @Override
+ public int hashCode() {
+ return 31;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() == obj.getClass())
+ return true;
+ return false;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
new file mode 100644
index 0000000..67e8b3f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
@@ -0,0 +1,541 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.easy.json;
+
+import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
+import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
+import static com.fasterxml.jackson.core.JsonToken.FIELD_NAME;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.schema.DiffSchema;
+import org.apache.drill.exec.schema.Field;
+import org.apache.drill.exec.schema.NamedField;
+import org.apache.drill.exec.schema.ObjectSchema;
+import org.apache.drill.exec.schema.RecordSchema;
+import org.apache.drill.exec.schema.SchemaIdGenerator;
+import org.apache.drill.exec.schema.json.jackson.JacksonHelper;
+import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.store.VectorHolder;
+import org.apache.drill.exec.vector.AllocationHelper;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.NullableFloat4Vector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.RepeatedBitVector;
+import org.apache.drill.exec.vector.RepeatedFloat4Vector;
+import org.apache.drill.exec.vector.RepeatedIntVector;
+import org.apache.drill.exec.vector.RepeatedVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class JSONRecordReader implements RecordReader {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader.class);
+ private static final int DEFAULT_LENGTH = 256 * 1024; // 256kb
+ public static final Charset UTF_8 = Charset.forName("UTF-8");
+
+ private final Map<String, VectorHolder> valueVectorMap;
+ private final FileSystem fileSystem;
+ private final Path hadoopPath;
+
+ private JsonParser parser;
+ private SchemaIdGenerator generator;
+ private DiffSchema diffSchema;
+ private RecordSchema currentSchema;
+ private List<Field> removedFields;
+ private OutputMutator outputMutator;
+ private BufferAllocator allocator;
+ private int batchSize;
+ private final FieldReference ref;
+ private final List<SchemaPath> columns;
+
+ public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize,
+ FieldReference ref, List<SchemaPath> columns) {
+ this.hadoopPath = new Path(inputPath);
+ this.fileSystem = fileSystem;
+ this.allocator = fragmentContext.getAllocator();
+ this.batchSize = batchSize;
+ valueVectorMap = Maps.newHashMap();
+ this.ref = ref;
+ this.columns = columns;
+ }
+
+ public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, FieldReference ref,
+ List<SchemaPath> columns) {
+ this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, ref, columns);
+ }
+
+ private JsonParser getParser() {
+ return parser;
+ }
+
+ @Override
+ public void setup(OutputMutator output) throws ExecutionSetupException {
+ outputMutator = output;
+ currentSchema = new ObjectSchema();
+ diffSchema = new DiffSchema();
+ removedFields = Lists.newArrayList();
+
+ try {
+ JsonFactory factory = new JsonFactory();
+ parser = factory.createJsonParser(fileSystem.open(hadoopPath));
+ parser.nextToken(); // Read to the first START_OBJECT token
+ generator = new SchemaIdGenerator();
+ } catch (IOException e) {
+ throw new ExecutionSetupException(e);
+ }
+ }
+
+ @Override
+ public int next() {
+ if (parser.isClosed() || !parser.hasCurrentToken()) {
+ return 0;
+ }
+
+ resetBatch();
+
+ int nextRowIndex = 0;
+
+ try {
+ while (ReadType.OBJECT.readRecord(this, null, nextRowIndex++, 0)) {
+ parser.nextToken(); // Read to START_OBJECT token
+
+ if (!parser.hasCurrentToken()) {
+ parser.close();
+ break;
+ }
+ }
+
+ parser.nextToken();
+
+ if (!parser.hasCurrentToken()) {
+ parser.close();
+ }
+
+ // Garbage collect fields never referenced in this batch
+ for (Field field : Iterables.concat(currentSchema.removeUnreadFields(), removedFields)) {
+ diffSchema.addRemovedField(field);
+ outputMutator.removeField(field.getAsMaterializedField(ref));
+ }
+
+ if (diffSchema.isChanged()) {
+ outputMutator.setNewSchema();
+ }
+
+
+ } catch (IOException | SchemaChangeException e) {
+ logger.error("Error reading next in Json reader", e);
+ }
+
+ for (VectorHolder holder : valueVectorMap.values()) {
+ holder.populateVectorLength();
+ }
+
+ return nextRowIndex;
+ }
+
+ private void resetBatch() {
+ for (VectorHolder value : valueVectorMap.values()) {
+ value.reset();
+ }
+
+ currentSchema.resetMarkedFields();
+ diffSchema.reset();
+ removedFields.clear();
+ }
+
+ @Override
+ public void cleanup() {
+ try {
+ parser.close();
+ } catch (IOException e) {
+ logger.warn("Error closing Json parser", e);
+ }
+ }
+
+
+ private RecordSchema getCurrentSchema() {
+ return currentSchema;
+ }
+
+ private void setCurrentSchema(RecordSchema schema) {
+ currentSchema = schema;
+ }
+
+ private List<Field> getRemovedFields() {
+ return removedFields;
+ }
+
+ public BufferAllocator getAllocator() {
+ return allocator;
+ }
+
+ private boolean fieldSelected(String field){
+ SchemaPath sp = new SchemaPath(field, ExpressionPosition.UNKNOWN);
+ if (this.columns != null && this.columns.size() > 0){
+ for (SchemaPath expr : this.columns){
+ if ( sp.equals(expr)){
+ return true;
+ }
+ }
+ return false;
+ }
+ return true;
+ }
+
+ public static enum ReadType {
+ ARRAY(END_ARRAY) {
+ @Override
+ public Field createField(RecordSchema parentSchema, String prefixFieldName, String fieldName, MajorType fieldType, int index) {
+ return new NamedField(parentSchema, prefixFieldName, fieldName, fieldType);
+ }
+
+ @Override
+ public RecordSchema createSchema() throws IOException {
+ return new ObjectSchema();
+ }
+ },
+ OBJECT(END_OBJECT) {
+ @Override
+ public Field createField(RecordSchema parentSchema,
+ String prefixFieldName,
+ String fieldName,
+ MajorType fieldType,
+ int index) {
+ return new NamedField(parentSchema, prefixFieldName, fieldName, fieldType);
+ }
+
+ @Override
+ public RecordSchema createSchema() throws IOException {
+ return new ObjectSchema();
+ }
+ };
+
+ private final JsonToken endObject;
+
+ ReadType(JsonToken endObject) {
+ this.endObject = endObject;
+ }
+
+ public JsonToken getEndObject() {
+ return endObject;
+ }
+
+ @SuppressWarnings("ConstantConditions")
+ public boolean readRecord(JSONRecordReader reader,
+ String prefixFieldName,
+ int rowIndex,
+ int groupCount) throws IOException, SchemaChangeException {
+ JsonParser parser = reader.getParser();
+ JsonToken token = parser.nextToken();
+ JsonToken endObject = getEndObject();
+ int colIndex = 0;
+ boolean isFull = false;
+ while (token != endObject) {
+ if (token == FIELD_NAME) {
+ token = parser.nextToken();
+ continue;
+ }
+
+ String fieldName = parser.getCurrentName();
+ if ( fieldName != null && ! reader.fieldSelected(fieldName)){
+ // this field was not requested in the query
+ token = parser.nextToken();
+ colIndex += 1;
+ continue;
+ }
+ MajorType fieldType = JacksonHelper.getFieldType(token, this == ReadType.ARRAY);
+ ReadType readType = null;
+ switch (token) {
+ case START_ARRAY:
+ readType = ReadType.ARRAY;
+ groupCount++;
+ break;
+ case START_OBJECT:
+ readType = ReadType.OBJECT;
+ groupCount = 0;
+ break;
+ }
+
+ if (fieldType != null) { // Including nulls
+ boolean currentFieldFull = !recordData(
+ readType,
+ reader,
+ fieldType,
+ prefixFieldName,
+ fieldName,
+ rowIndex,
+ colIndex,
+ groupCount);
+
+ isFull = isFull || currentFieldFull;
+ }
+ token = parser.nextToken();
+ colIndex += 1;
+ }
+ return !isFull;
+ }
+
+ private void removeChildFields(List<Field> removedFields, Field field) {
+ RecordSchema schema = field.getAssignedSchema();
+ if (schema == null) {
+ return;
+ }
+ for (Field childField : schema.getFields()) {
+ removedFields.add(childField);
+ if (childField.hasSchema()) {
+ removeChildFields(removedFields, childField);
+ }
+ }
+ }
+
+ private boolean recordData(JSONRecordReader.ReadType readType,
+ JSONRecordReader reader,
+ MajorType fieldType,
+ String prefixFieldName,
+ String fieldName,
+ int rowIndex,
+ int colIndex,
+ int groupCount) throws IOException, SchemaChangeException {
+ RecordSchema currentSchema = reader.getCurrentSchema();
+ Field field = currentSchema.getField(fieldName == null ? prefixFieldName : fieldName, colIndex);
+ boolean isFieldFound = field != null;
+ List<Field> removedFields = reader.getRemovedFields();
+ boolean newFieldLateBound = fieldType.getMinorType().equals(MinorType.LATE);
+
+ if (isFieldFound && !field.getFieldType().equals(fieldType)) {
+ boolean existingFieldLateBound = field.getFieldType().getMinorType().equals(MinorType.LATE);
+
+ if (newFieldLateBound && !existingFieldLateBound) {
+ fieldType = Types.overrideMinorType(fieldType, field.getFieldType().getMinorType());
+ } else if (!newFieldLateBound && existingFieldLateBound) {
+ field.setFieldType(Types.overrideMinorType(field.getFieldType(), fieldType.getMinorType()));
+ } else if (!newFieldLateBound && !existingFieldLateBound) {
+ if (field.hasSchema()) {
+ removeChildFields(removedFields, field);
+ }
+ removedFields.add(field);
+ currentSchema.removeField(field, colIndex);
+
+ isFieldFound = false;
+ }
+ }
+
+ if (!isFieldFound) {
+ field = createField(
+ currentSchema,
+ prefixFieldName,
+ fieldName,
+ fieldType,
+ colIndex
+ );
+
+ reader.recordNewField(field);
+ currentSchema.addField(field);
+ }
+
+ field.setRead(true);
+
+ VectorHolder holder = getOrCreateVectorHolder(reader, field);
+ if (readType != null) {
+ RecordSchema fieldSchema = field.getAssignedSchema();
+ RecordSchema newSchema = readType.createSchema();
+
+ if (readType != ReadType.ARRAY) {
+ reader.setCurrentSchema(fieldSchema);
+ if (fieldSchema == null) reader.setCurrentSchema(newSchema);
+ readType.readRecord(reader, field.getFullFieldName(), rowIndex, groupCount);
+ } else {
+ readType.readRecord(reader, field.getFullFieldName(), rowIndex, groupCount);
+ }
+
+ reader.setCurrentSchema(currentSchema);
+
+ } else if (holder != null && !newFieldLateBound && fieldType.getMinorType() != MinorType.LATE) {
+ return addValueToVector(
+ rowIndex,
+ holder,
+ JacksonHelper.getValueFromFieldType(
+ reader.getParser(),
+ fieldType.getMinorType()
+ ),
+ fieldType.getMinorType(),
+ groupCount
+ );
+ }
+
+ return true;
+ }
+
+ private static <T> boolean addValueToVector(int index, VectorHolder holder, T val, MinorType minorType, int groupCount) {
+ switch (minorType) {
+ case INT: {
+ holder.incAndCheckLength(NullableIntHolder.WIDTH * 8 + 1);
+ if (groupCount == 0) {
+ if (val != null) {
+ NullableIntVector int4 = (NullableIntVector) holder.getValueVector();
+ NullableIntVector.Mutator m = int4.getMutator();
+ m.set(index, (Integer) val);
+ }
+ } else {
+ if (val == null) {
+ throw new UnsupportedOperationException("Nullable repeated int is not supported.");
+ }
+
+ RepeatedIntVector repeatedInt4 = (RepeatedIntVector) holder.getValueVector();
+ RepeatedIntVector.Mutator m = repeatedInt4.getMutator();
+ holder.setGroupCount(index);
+ m.add(index, (Integer) val);
+ }
+
+ return holder.hasEnoughSpace(NullableIntHolder.WIDTH * 8 + 1);
+ }
+ case FLOAT4: {
+ holder.incAndCheckLength(NullableFloat4Holder.WIDTH * 8 + 1);
+ if (groupCount == 0) {
+ if (val != null) {
+ NullableFloat4Vector float4 = (NullableFloat4Vector) holder.getValueVector();
+ NullableFloat4Vector.Mutator m = float4.getMutator();
+ m.set(index, (Float) val);
+ }
+ } else {
+ if (val == null) {
+ throw new UnsupportedOperationException("Nullable repeated float is not supported.");
+ }
+
+ RepeatedFloat4Vector repeatedFloat4 = (RepeatedFloat4Vector) holder.getValueVector();
+ RepeatedFloat4Vector.Mutator m = repeatedFloat4.getMutator();
+ holder.setGroupCount(index);
+ m.add(index, (Float) val);
+ }
+ return holder.hasEnoughSpace(NullableFloat4Holder.WIDTH * 8 + 1);
+ }
+ case VARCHAR: {
+ if (val == null) {
+ return (index + 1) * 4 <= holder.getLength();
+ } else {
+ byte[] bytes = ((String) val).getBytes(UTF_8);
+ int length = bytes.length;
+ holder.incAndCheckLength(length);
+ if (groupCount == 0) {
+ NullableVarCharVector varLen4 = (NullableVarCharVector) holder.getValueVector();
+ NullableVarCharVector.Mutator m = varLen4.getMutator();
+ m.set(index, bytes);
+ } else {
+ RepeatedVarCharVector repeatedVarLen4 = (RepeatedVarCharVector) holder.getValueVector();
+ RepeatedVarCharVector.Mutator m = repeatedVarLen4.getMutator();
+ holder.setGroupCount(index);
+ m.add(index, bytes);
+ }
+ return holder.hasEnoughSpace(length + 4 + 1);
+ }
+ }
+ case BIT: {
+ holder.incAndCheckLength(NullableBitHolder.WIDTH + 1);
+ if (groupCount == 0) {
+ if (val != null) {
+ NullableBitVector bit = (NullableBitVector) holder.getValueVector();
+ NullableBitVector.Mutator m = bit.getMutator();
+ m.set(index, (Boolean) val ? 1 : 0);
+ }
+ } else {
+ if (val == null) {
+ throw new UnsupportedOperationException("Nullable repeated boolean is not supported.");
+ }
+
+ RepeatedBitVector repeatedBit = (RepeatedBitVector) holder.getValueVector();
+ RepeatedBitVector.Mutator m = repeatedBit.getMutator();
+ holder.setGroupCount(index);
+ m.add(index, (Boolean) val ? 1 : 0);
+ }
+ return holder.hasEnoughSpace(NullableBitHolder.WIDTH + 1);
+ }
+ default:
+ throw new DrillRuntimeException("Type not supported to add value. Type: " + minorType);
+ }
+ }
+
+ private VectorHolder getOrCreateVectorHolder(JSONRecordReader reader, Field field) throws SchemaChangeException {
+ return reader.getOrCreateVectorHolder(field);
+ }
+
+ public abstract RecordSchema createSchema() throws IOException;
+
+ public abstract Field createField(RecordSchema parentSchema,
+ String prefixFieldName,
+ String fieldName,
+ MajorType fieldType,
+ int index);
+ }
+
+ private void recordNewField(Field field) {
+ diffSchema.recordNewField(field);
+ }
+
+ private VectorHolder getOrCreateVectorHolder(Field field) throws SchemaChangeException {
+ String fullFieldName = ref != null ? ref.getPath() + "." + field.getFullFieldName() : field.getFullFieldName();
+ VectorHolder holder = valueVectorMap.get(fullFieldName);
+
+ if (holder == null) {
+ MajorType type = field.getFieldType();
+ MinorType minorType = type.getMinorType();
+
+ if (minorType.equals(MinorType.MAP) || minorType.equals(MinorType.LATE)) {
+ return null;
+ }
+
+ MaterializedField f = MaterializedField.create(new SchemaPath(fullFieldName, ExpressionPosition.UNKNOWN), type);
+
+ ValueVector v = TypeHelper.getNewVector(f, allocator);
+ AllocationHelper.allocate(v, batchSize, 50);
+ holder = new VectorHolder(v);
+ valueVectorMap.put(fullFieldName, holder);
+ outputMutator.addField(v);
+ return holder;
+ }
+ return holder;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
index 6211e21..f330a1e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
@@ -17,19 +17,19 @@
*/
package org.apache.drill.exec.store.hive;
-import com.beust.jcommander.internal.Lists;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+
import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.ReadEntry;
import org.apache.drill.exec.physical.base.Size;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
-public class HiveReadEntry implements ReadEntry {
+public class HiveReadEntry {
@JsonProperty("table")
public HiveTable table;
@@ -60,18 +60,5 @@ public class HiveReadEntry implements ReadEntry {
return partitionsUnwrapped;
}
- @Override
- public OperatorCost getCost() {
- // TODO: need to come up with way to calculate the cost for Hive tables
- return new OperatorCost(1, 1, 2, 2);
- }
-
- @Override
- public Size getSize() {
- // TODO: contact the metastore and find the size of the data in table
- Size size = new Size(1, 1);
-
- return size;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java
deleted file mode 100644
index ef7266c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.hive;
-
-import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.base.Size;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-public class HiveReadEntryOld implements ReadEntry {
- private final HiveConf conf;
- private final String table;
- private Size size;
-
- public HiveReadEntryOld(HiveConf conf, String table) {
- this.conf = conf;
- this.table = table;
- }
-
- @Override
- public OperatorCost getCost() {
- // TODO: need to come up with way to calculate the cost for Hive tables
- return new OperatorCost(1, 1, 2, 2);
- }
-
- @Override
- public Size getSize() {
- if (size != null) {
- // TODO: contact the metastore and find the size of the data in table
- size = new Size(1, 1);
- }
-
- return size;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
index bc2a16b..a1e8f1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java
@@ -34,7 +34,7 @@ import org.apache.drill.exec.physical.base.Size;
import org.apache.drill.exec.physical.base.SubScan;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -58,9 +58,9 @@ public class HiveScan extends AbstractGroupScan {
@JsonIgnore
private List<InputSplit> inputSplits = Lists.newArrayList();
@JsonIgnore
- public HiveStorageEngine storageEngine;
- @JsonProperty("storageengine")
- public HiveStorageEngineConfig engineConfig;
+ public HiveStoragePlugin storagePlugin;
+ @JsonProperty("storage-plugin")
+ public String storagePluginName;
@JsonIgnore
public List<Partition> partitions;
@@ -77,27 +77,27 @@ public class HiveScan extends AbstractGroupScan {
Map<InputSplit, Partition> partitionMap = new HashMap();
@JsonCreator
- public HiveScan(@JsonProperty("hive-table") HiveReadEntry hiveReadEntry, @JsonProperty("storageengine") HiveStorageEngineConfig config,
+ public HiveScan(@JsonProperty("hive-table") HiveReadEntry hiveReadEntry, @JsonProperty("storage-plugin") String storagePluginName,
@JsonProperty("columns") List<FieldReference> columns,
- @JacksonInject StorageEngineRegistry engineRegistry) throws ExecutionSetupException {
+ @JacksonInject StoragePluginRegistry engineRegistry) throws ExecutionSetupException {
this.hiveReadEntry = hiveReadEntry;
this.table = hiveReadEntry.getTable();
- this.engineConfig = config;
- this.storageEngine = (HiveStorageEngine) engineRegistry.getEngine(config);
+ this.storagePluginName = storagePluginName;
+ this.storagePlugin = (HiveStoragePlugin) engineRegistry.getEngine(storagePluginName);
this.columns = columns;
this.partitions = hiveReadEntry.getPartitions();
getSplits();
- endpoints = storageEngine.getContext().getBits();
+ endpoints = storagePlugin.getContext().getBits();
}
- public HiveScan(HiveReadEntry hiveReadEntry, HiveStorageEngine storageEngine, List<FieldReference> columns) throws ExecutionSetupException {
+ public HiveScan(HiveReadEntry hiveReadEntry, HiveStoragePlugin storageEngine, List<FieldReference> columns) throws ExecutionSetupException {
this.table = hiveReadEntry.getTable();
this.hiveReadEntry = hiveReadEntry;
this.columns = columns;
this.partitions = hiveReadEntry.getPartitions();
getSplits();
endpoints = storageEngine.getContext().getBits();
- this.engineConfig = storageEngine.getConfig();
+ this.storagePluginName = storageEngine.getName();
}
public List<FieldReference> getColumns() {
@@ -112,7 +112,7 @@ public class HiveScan extends AbstractGroupScan {
for (Object obj : properties.keySet()) {
job.set((String) obj, (String) properties.get(obj));
}
- InputFormat format = (InputFormat) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance();
+ InputFormat<?, ?> format = (InputFormat<?, ?>) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance();
job.setInputFormat(format.getClass());
Path path = new Path(table.getSd().getLocation());
FileInputFormat.addInputPath(job, path);
@@ -130,7 +130,7 @@ public class HiveScan extends AbstractGroupScan {
for (Object obj : properties.keySet()) {
job.set((String) obj, (String) properties.get(obj));
}
- InputFormat format = (InputFormat) Class.forName(partition.getSd().getInputFormat()).getConstructor().newInstance();
+ InputFormat<?, ?> format = (InputFormat<?, ?>) Class.forName(partition.getSd().getInputFormat()).getConstructor().newInstance();
job.setInputFormat(format.getClass());
FileInputFormat.addInputPath(job, new Path(partition.getSd().getLocation()));
format = job.getInputFormat();
@@ -192,12 +192,12 @@ public class HiveScan extends AbstractGroupScan {
@Override
public List<EndpointAffinity> getOperatorAffinity() {
- Map<String, DrillbitEndpoint> endpointMap = new HashMap();
+ Map<String, DrillbitEndpoint> endpointMap = new HashMap<>();
for (DrillbitEndpoint endpoint : endpoints) {
endpointMap.put(endpoint.getAddress(), endpoint);
logger.debug("endpoing address: {}", endpoint.getAddress());
}
- Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap();
+ Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<>();
try {
long totalSize = 0;
for (InputSplit split : inputSplits) {
@@ -242,6 +242,6 @@ public class HiveScan extends AbstractGroupScan {
@Override
public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
- return new HiveScan(hiveReadEntry, storageEngine, columns);
+ return new HiveScan(hiveReadEntry, storagePlugin, columns);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java
deleted file mode 100644
index 0f6f3bc..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.hive;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.base.Size;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.AbstractStorageEngine;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.thrift.TException;
-
-import java.io.IOException;
-import java.util.List;
-
-public class HiveStorageEngine extends AbstractStorageEngine {
-
- private HiveStorageEngineConfig config;
- private HiveConf hiveConf;
- private HiveSchemaProvider schemaProvider;
- static private DrillbitContext context;
-
- public HiveStorageEngine(HiveStorageEngineConfig config, DrillbitContext context) throws ExecutionSetupException {
- this.config = config;
- this.context = context;
- this.hiveConf = config.getHiveConf();
- }
-
- public HiveStorageEngineConfig getConfig() {
- return config;
- }
-
- public DrillbitContext getContext() {
- return context;
- }
-
- @Override
- public HiveScan getPhysicalScan(Scan scan) throws IOException {
- HiveReadEntry hiveReadEntry = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){});
- try {
- List<Partition> partitions = getSchemaProvider().getPartitions(hiveReadEntry.getTable().getDbName(), hiveReadEntry.getTable().getTableName());
- return new HiveScan(hiveReadEntry, this, null);
- } catch (ExecutionSetupException | TException e) {
- throw new DrillRuntimeException(e);
- }
- }
-
- @Override
- public HiveSchemaProvider getSchemaProvider() {
- try {
- if (schemaProvider == null) {
- schemaProvider = new HiveSchemaProvider(config, context.getConfig());
- }
- return schemaProvider;
- } catch (ExecutionSetupException e) {
- throw new DrillRuntimeException(e);
- }
- }
-
- List<String> getPartitions(String dbName, String tableName) throws TException {
- List<Partition> partitions = getSchemaProvider().getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE);
- List<String> partitionLocations = Lists.newArrayList();
- if (partitions == null) return null;
- for (Partition part : partitions) {
- partitionLocations.add(part.getSd().getLocation());
- }
- return partitionLocations;
- }
-
- public static class HiveEntry implements ReadEntry {
-
- private Table table;
-
- public HiveEntry(Table table) {
- this.table = table;
- }
-
- public Table getTable() {
- return table;
- }
-
- @Override
- public OperatorCost getCost() {
- throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " +
- "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost.");
- }
-
- @Override
- public Size getSize() {
- throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " +
- "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost.");
- }
- }
-
- public static class HiveSchemaProvider implements SchemaProvider {
-
- private HiveConf hiveConf;
- private HiveMetaStoreClient metaClient;
-
- public HiveSchemaProvider(HiveStorageEngineConfig config, DrillConfig dConfig) throws ExecutionSetupException {
- hiveConf = config.getHiveConf();
- }
-
- public HiveMetaStoreClient getMetaClient() throws MetaException {
- if (metaClient == null) {
- metaClient = new HiveMetaStoreClient(hiveConf);
- }
- return metaClient;
- }
-
- public Table getTable(String dbName, String tableName) throws TException {
- HiveMetaStoreClient mClient = getMetaClient();
- try {
- return mClient.getTable(dbName, tableName);
- }catch (NoSuchObjectException e) {
- logger.error("Database: {} table: {} not found", dbName, tableName);
- throw new RuntimeException(e);
- } catch (TException e) {
- mClient.reconnect();
- return mClient.getTable(dbName, tableName);
- }
- }
-
- List<Partition> getPartitions(String dbName, String tableName) throws TException {
- HiveMetaStoreClient mClient = getMetaClient();
- List<Partition> partitions;
- try {
- partitions = getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE);
- } catch (TException e) {
- mClient.reconnect();
- partitions = getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE);
- }
- return partitions;
- }
-
- @Override
- public HiveReadEntry getSelectionBaseOnName(String name) {
- String[] dbNameTableName = name.split("\\.");
- String dbName;
- String t;
- if (dbNameTableName.length > 1) {
- dbName = dbNameTableName[0];
- t = dbNameTableName[1];
- } else {
- dbName = "default";
- t = name;
- }
-
- try {
- Table table = getTable(dbName, t);
- List<Partition> partitions = getPartitions(dbName, t);
- List<HiveTable.HivePartition> hivePartitions = Lists.newArrayList();
- for(Partition part : partitions) {
- hivePartitions.add(new HiveTable.HivePartition(part));
- }
- if (hivePartitions.size() == 0) hivePartitions = null;
- return new HiveReadEntry(new HiveTable(table), hivePartitions);
- } catch (NoSuchObjectException e) {
- throw new DrillRuntimeException(e);
- } catch (TException e) {
- throw new DrillRuntimeException(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
deleted file mode 100644
index 91fec3b..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.hive;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import java.util.Map;
-
-@JsonTypeName("hive")
-public class HiveStorageEngineConfig extends StorageEngineConfigBase {
- @JsonProperty
- public Map<String, String> configProps;
- @JsonIgnore
- private HiveConf hiveConf;
-
- @JsonIgnore
- public HiveConf getHiveConf() {
- if (hiveConf == null) {
- hiveConf = new HiveConf();
- if (configProps != null) {
- for (Map.Entry<String, String> entry : configProps.entrySet()) {
- hiveConf.set(entry.getKey(), entry.getValue());
- }
- }
- }
-
- return hiveConf;
- }
-
- @JsonCreator
- public HiveStorageEngineConfig(@JsonProperty("config") Map<String, String> props) {
- this.configProps = props;
- }
-
- @Override
- public int hashCode() {
- return configProps != null ? configProps.hashCode() : 0;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- HiveStorageEngineConfig that = (HiveStorageEngineConfig) o;
-
- if (configProps != null ? !configProps.equals(that.configProps) : that.configProps != null) return false;
-
- return true;
- }
-}
[31/38] git commit: DRILL-376: Create separate DataTunnel for
receiving fragment
Posted by ja...@apache.org.
DRILL-376: Create separate DataTunnel for receiving fragment
Signed-off-by: Jacques Nadeau <ja...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ad3ac806
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ad3ac806
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ad3ac806
Branch: refs/heads/master
Commit: ad3ac8063a8c857db02dfdb5ca68672752f4a7c8
Parents: 08bc16d
Author: Steven Phillips <sp...@maprtech.com>
Authored: Wed Feb 12 16:42:13 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../java/org/apache/drill/exec/ops/FragmentContext.java | 10 +++++-----
.../exec/physical/impl/TestHashToRandomExchange.java | 2 +-
exec/java-exec/src/test/resources/drill-module.conf | 4 ++--
.../src/test/resources/sender/hash_exchange.json | 10 ++++++++++
4 files changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ad3ac806/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 80a7819..630355e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -52,7 +52,7 @@ public class FragmentContext implements Closeable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentContext.class);
- private Map<DrillbitEndpoint, DataTunnel> tunnels = Maps.newHashMap();
+ private Map<FragmentHandle, DataTunnel> tunnels = Maps.newHashMap();
private final DrillbitContext context;
private final UserClientConnection connection;
@@ -154,10 +154,10 @@ public class FragmentContext implements Closeable {
}
public DataTunnel getDataTunnel(DrillbitEndpoint endpoint, FragmentHandle remoteHandle) {
- DataTunnel tunnel = tunnels.get(endpoint);
+ DataTunnel tunnel = tunnels.get(remoteHandle);
if (tunnel == null) {
tunnel = context.getDataConnectionsPool().getTunnel(endpoint, remoteHandle);
- tunnels.put(endpoint, tunnel);
+ tunnels.put(remoteHandle, tunnel);
}
return tunnel;
}
@@ -165,8 +165,8 @@ public class FragmentContext implements Closeable {
/**
* Add a new thread to this fragment's context. This thread will likely run for the life of the fragment but should be
* terminated when the fragment completes. When the fragment completes, the threads will be interrupted.
- *
- * @param runnable
+ *
+ * @param thread
*/
public void addDaemonThread(Thread thread) {
daemonThreads.add(thread);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ad3ac806/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java
index bbe1c18..d20928d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java
@@ -57,7 +57,7 @@ public class TestHashToRandomExchange extends PopUnitTestBase {
if (b.getHeader().getRowCount() != 0)
count += b.getHeader().getRowCount();
}
- assertEquals(100, count);
+ assertEquals(200, count);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ad3ac806/exec/java-exec/src/test/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/drill-module.conf b/exec/java-exec/src/test/resources/drill-module.conf
index 5803610..56d1c16 100644
--- a/exec/java-exec/src/test/resources/drill-module.conf
+++ b/exec/java-exec/src/test/resources/drill-module.conf
@@ -56,8 +56,8 @@ drill.exec: {
start: 35000
},
work: {
- max.width.per.endpoint: 1,
+ max.width.per.endpoint: 2,
global.max.width: 100,
- executor.threads: 1
+ executor.threads: 4
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ad3ac806/exec/java-exec/src/test/resources/sender/hash_exchange.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/sender/hash_exchange.json b/exec/java-exec/src/test/resources/sender/hash_exchange.json
index d8f9579..fc45ec1 100644
--- a/exec/java-exec/src/test/resources/sender/hash_exchange.json
+++ b/exec/java-exec/src/test/resources/sender/hash_exchange.json
@@ -21,6 +21,16 @@
{name: "blue", type: "INT", mode: "REQUIRED"},
{name: "red", type: "BIGINT", mode: "REQUIRED"},
{name: "green", type: "VARBINARY", mode: "REQUIRED"}
+ ]},
+ {records: 100, types: [
+ {name: "blue", type: "INT", mode: "REQUIRED"},
+ {name: "red", type: "BIGINT", mode: "REQUIRED"},
+ {name: "green", type: "VARBINARY", mode: "REQUIRED"}
+ ]},
+ {records: 100, types: [
+ {name: "blue", type: "INT", mode: "REQUIRED"},
+ {name: "red", type: "BIGINT", mode: "REQUIRED"},
+ {name: "green", type: "VARBINARY", mode: "REQUIRED"}
]}
]
},
[03/38] git commit: DRILL 354: Hive storage engine - Phase 1
Posted by ja...@apache.org.
DRILL 354: Hive storage engine - Phase 1
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a9a7ea84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a9a7ea84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a9a7ea84
Branch: refs/heads/master
Commit: a9a7ea84c99d8a9efcccc7d9a870121a26212b49
Parents: b5ab447
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Jan 31 14:13:14 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:21:50 2014 -0800
----------------------------------------------------------------------
contrib/pom.xml | 1 +
contrib/storage-hive/pom.xml | 120 +++++++++
distribution/src/assemble/bin.xml | 4 +
exec/java-exec/pom.xml | 11 +
.../drill/exec/store/hive/HiveReadEntry.java | 50 ++++
.../exec/store/hive/HiveSchemaProvider.java | 39 +++
.../store/hive/HiveStorageEngineConfig.java | 67 +++++
pom.xml | 6 +-
sqlparser/pom.xml | 31 +++
.../org/apache/drill/jdbc/DrillHandler.java | 30 ++-
.../sql/client/full/HiveDatabaseSchema.java | 265 +++++++++++++++++++
.../drill/sql/client/full/HiveSchema.java | 161 +++++++++++
.../apache/drill/jdbc/test/FullEngineTest.java | 47 +++-
.../src/test/resources/storage-engines.json | 42 +++
14 files changed, 864 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/contrib/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/pom.xml b/contrib/pom.xml
index dc07979..578513d 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -33,6 +33,7 @@
<modules>
<module>storage-hbase</module>
+ <module>storage-hive</module>
<module>sqlline</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/contrib/storage-hive/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/pom.xml b/contrib/storage-hive/pom.xml
new file mode 100644
index 0000000..d329950
--- /dev/null
+++ b/contrib/storage-hive/pom.xml
@@ -0,0 +1,120 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+ xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>drill-root</artifactId>
+ <groupId>org.apache.drill</groupId>
+ <version>1.0.0-m2-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>storage-hive</artifactId>
+ <packaging>jar</packaging>
+ <name>contrib/storage-hive</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>0.12.0</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>0.12.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.hive:hive-exec</include>
+ </includes>
+ </artifactSet>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <relocations>
+ <relocation>
+ <pattern>com.google.</pattern>
+ <shadedPattern>com.google.hive12.</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index fdd6c70..5538d40 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -86,6 +86,10 @@
<outputDirectory>jars</outputDirectory>
</file>
<file>
+ <source>../contrib/storage-hive/target/storage-hive-${project.version}.jar</source>
+ <outputDirectory>jars</outputDirectory>
+ </file>
+ <file>
<source>src/resources/runbit</source>
<outputDirectory>bin</outputDirectory>
</file>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 1c4dc32..2cc3fb4 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -123,6 +123,17 @@
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
+ <artifactId>storage-hive</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
<artifactId>drill-common</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
new file mode 100644
index 0000000..41a4d3d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.ReadEntry;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+public class HiveReadEntry implements ReadEntry {
+ private final HiveConf conf;
+ private final String table;
+ private Size size;
+
+ public HiveReadEntry(HiveConf conf, String table) {
+ this.conf = conf;
+ this.table = table;
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ // TODO: need to come up with way to calculate the cost for Hive tables
+ return new OperatorCost(1, 1, 2, 2);
+ }
+
+ @Override
+ public Size getSize() {
+ if (size != null) {
+ // TODO: contact the metastore and find the size of the data in table
+ size = new Size(1, 1);
+ }
+
+ return size;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSchemaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSchemaProvider.java
new file mode 100644
index 0000000..38ec007
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveSchemaProvider.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.store.SchemaProvider;
+
+import com.beust.jcommander.internal.Lists;
+
+public class HiveSchemaProvider implements SchemaProvider{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveSchemaProvider.class);
+
+ final HiveStorageEngineConfig configuration;
+
+ public HiveSchemaProvider(HiveStorageEngineConfig configuration, DrillConfig config){
+ this.configuration = configuration;
+ }
+
+ @Override
+ public Object getSelectionBaseOnName(String tableName) {
+ HiveReadEntry re = new HiveReadEntry(configuration.getHiveConf(), tableName);
+ return Lists.newArrayList(re);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
new file mode 100644
index 0000000..0a2c5de
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.drill.common.logical.StorageEngineConfigBase;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import java.util.Map;
+
+@JsonTypeName("hive")
+public class HiveStorageEngineConfig extends StorageEngineConfigBase {
+ private Map<String, String> configProps;
+ private HiveConf hiveConf;
+
+ public HiveConf getHiveConf() {
+ if (hiveConf == null) {
+ hiveConf = new HiveConf();
+ if (configProps != null) {
+ for (Map.Entry<String, String> entry : configProps.entrySet()) {
+ hiveConf.set(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ return hiveConf;
+ }
+
+ @JsonCreator
+ public HiveStorageEngineConfig(@JsonProperty("config") Map<String, String> props) {
+ this.configProps = props;
+ }
+
+ @Override
+ public int hashCode() {
+ return configProps != null ? configProps.hashCode() : 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ HiveStorageEngineConfig that = (HiveStorageEngineConfig) o;
+
+ if (configProps != null ? !configProps.equals(that.configProps) : that.configProps != null) return false;
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 69d280c..2121adc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -114,7 +114,8 @@
<exclude>**/*.proto</exclude>
<exclude>**/*.fmpp</exclude>
<exclude>**/*.tdd</exclude>
-
+ <exclude>**/drill_hive_db/**</exclude>
+ <exclude>**/drill_hive_wh/**</exclude>
</excludes>
</configuration>
</plugin>
@@ -209,6 +210,9 @@
<version>2.15</version>
<configuration>
<argLine>-XX:MaxDirectMemorySize=4096M </argLine>
+ <additionalClasspathElements>
+ <additionalClasspathElement>./sqlparser/src/test/resources/storage-engines.json</additionalClasspathElement>
+ </additionalClasspathElements>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/sqlparser/pom.xml
----------------------------------------------------------------------
diff --git a/sqlparser/pom.xml b/sqlparser/pom.xml
index 34622f7..c7ceb96 100644
--- a/sqlparser/pom.xml
+++ b/sqlparser/pom.xml
@@ -30,6 +30,17 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>storage-hive</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-ref</artifactId>
<version>${project.version}</version>
@@ -103,6 +114,26 @@
</excludes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>test</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <tasks>
+ <copy todir="/tmp/drill_hive_wh">
+ <fileset dir="${basedir}/../sample-data/drill_hive_wh"/>
+ </copy>
+ </tasks>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<profiles>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
index 51600ec..fefe7bc 100644
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
@@ -41,11 +41,15 @@ import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.store.SchemaProvider;
import org.apache.drill.exec.store.SchemaProviderRegistry;
+import org.apache.drill.exec.store.hive.HiveSchemaProvider;
+import org.apache.drill.exec.store.json.JsonSchemaProvider;
+import org.apache.drill.exec.store.parquet.ParquetSchemaProvider;
import org.apache.drill.sql.client.full.FileSystemSchema;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.io.Resources;
+import org.apache.drill.sql.client.full.HiveSchema;
public class DrillHandler extends HandlerImpl {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHandler.class);
@@ -114,9 +118,7 @@ public class DrillHandler extends HandlerImpl {
for (Map.Entry<String, StorageEngineConfig> entry : engines) {
SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
- FileSystemSchema schema = new FileSystemSchema(client, entry.getValue(), provider,
- rootSchema.getTypeFactory(), rootSchema, entry.getKey(), rootSchema.getExpression(),
- rootSchema.getQueryProvider());
+ Schema schema = getSchema(provider, client, entry.getKey(), entry.getValue(), rootSchema);
rootSchema.addSchema(entry.getKey(), schema);
}
@@ -139,6 +141,25 @@ public class DrillHandler extends HandlerImpl {
connection.setSchema(schemaName);
}
+ final String catalogName = connection.getProperties().getProperty("catalog");
+ if (catalogName != null) {
+ connection.setCatalog(catalogName);
+ }
+ }
+
+ private Schema getSchema(SchemaProvider provider, DrillClient client, String name, StorageEngineConfig config, Schema rootSchema)
+ throws SQLException {
+ if (provider instanceof ParquetSchemaProvider || provider instanceof JsonSchemaProvider) {
+ return new FileSystemSchema(client, config, provider,
+ rootSchema.getTypeFactory(), rootSchema, name, rootSchema.getExpression(),
+ rootSchema.getQueryProvider());
+ } else if (provider instanceof HiveSchemaProvider) {
+ return new HiveSchema(client, config, provider,
+ rootSchema.getTypeFactory(), rootSchema, name, rootSchema.getExpression(),
+ rootSchema.getQueryProvider());
+ }
+
+ throw new SQLException("Unknown schema provider");
}
public class FakeSchema extends MapSchema {
@@ -171,6 +192,9 @@ public class DrillHandler extends HandlerImpl {
} catch (IOException e) {
throw new RuntimeException("Failure closing coordinator.", e);
}
+ bit = null;
+ client = null;
+ coordinator = null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
new file mode 100644
index 0000000..c12057a
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
@@ -0,0 +1,265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.sql.client.full;
+
+import java.nio.charset.Charset;
+import java.util.*;
+
+import net.hydromatic.linq4j.QueryProvider;
+import net.hydromatic.linq4j.expressions.Expression;
+import net.hydromatic.linq4j.expressions.Expressions;
+import net.hydromatic.linq4j.expressions.MethodCallExpression;
+import net.hydromatic.optiq.BuiltinMethod;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.Table;
+import net.hydromatic.optiq.impl.java.JavaTypeFactory;
+
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.exec.store.hive.HiveStorageEngineConfig;
+import org.apache.drill.jdbc.DrillTable;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.sql.SqlCollation;
+import org.eigenbase.sql.type.SqlTypeName;
+
+public class HiveDatabaseSchema implements Schema{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveDatabaseSchema.class);
+
+ private final JavaTypeFactory typeFactory;
+ private final HiveSchema parentSchema;
+ private final String name;
+ private final Expression expression;
+ private final QueryProvider queryProvider;
+ private final SchemaProvider schemaProvider;
+ private final DrillClient client;
+ private final HiveStorageEngineConfig config;
+
+ public HiveDatabaseSchema(DrillClient client, HiveStorageEngineConfig config, SchemaProvider schemaProvider,
+ JavaTypeFactory typeFactory, HiveSchema parentSchema, String name,
+ Expression expression, QueryProvider queryProvider) {
+ super();
+ this.client = client;
+ this.typeFactory = typeFactory;
+ this.parentSchema = parentSchema;
+ this.name = name;
+ this.expression = expression;
+ this.queryProvider = queryProvider;
+ this.schemaProvider = schemaProvider;
+ this.config = config;
+ }
+
+ @Override
+ public Schema getSubSchema(String name) {
+ return null;
+ }
+
+ @Override
+ public JavaTypeFactory getTypeFactory() {
+ return typeFactory;
+ }
+
+ @Override
+ public Schema getParentSchema() {
+ return parentSchema;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Expression getExpression() {
+ return expression;
+ }
+
+ @Override
+ public QueryProvider getQueryProvider() {
+ return queryProvider;
+ }
+
+ // TODO: Need to integrates UDFs?
+ @Override
+ public Collection<TableFunctionInSchema> getTableFunctions(String name) {
+ return Collections.EMPTY_LIST;
+ }
+
+ // TODO: Need to integrates UDFs?
+ @Override
+ public Multimap<String, TableFunctionInSchema> getTableFunctions() {
+ return ArrayListMultimap.create();
+ }
+
+ @Override
+ /**
+ * No more sub schemas within a database schema
+ */
+ public Collection<String> getSubSchemaNames() {
+ return Collections.EMPTY_LIST;
+ }
+
+ static Map<PrimitiveObjectInspector.PrimitiveCategory, SqlTypeName> mapPrimHive2Sql = new HashMap<>();
+
+ private RelDataType getRelDataTypeFromHivePrimitiveType(PrimitiveObjectInspector poi) {
+ switch(poi.getPrimitiveCategory()) {
+ case BOOLEAN:
+ return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+ case BYTE:
+ return typeFactory.createSqlType(SqlTypeName.TINYINT);
+
+ case SHORT:
+ return typeFactory.createSqlType(SqlTypeName.SMALLINT);
+
+ case INT:
+ return typeFactory.createSqlType(SqlTypeName.INTEGER);
+
+ case LONG:
+ return typeFactory.createSqlType(SqlTypeName.BIGINT);
+
+ case FLOAT:
+ return typeFactory.createSqlType(SqlTypeName.FLOAT);
+
+ case DOUBLE:
+ return typeFactory.createSqlType(SqlTypeName.DOUBLE);
+
+ case DATE:
+ return typeFactory.createSqlType(SqlTypeName.DATE);
+
+ case TIMESTAMP:
+ return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
+
+ case BINARY:
+ return typeFactory.createSqlType(SqlTypeName.BINARY);
+
+ case DECIMAL:
+ return typeFactory.createSqlType(SqlTypeName.DECIMAL);
+
+ case STRING:
+ case VARCHAR: {
+ return typeFactory.createTypeWithCharsetAndCollation(
+ typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
+ Charset.forName("UTF-16"), /*unicode char set*/
+ SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
+ );
+ }
+
+ case UNKNOWN:
+ case VOID:
+ default:
+ throw new RuntimeException("Unknown or unsupported hive type");
+ }
+ }
+
+ private RelDataType getRelDataTypeFromHiveType(ObjectInspector oi) {
+ switch(oi.getCategory()) {
+ case PRIMITIVE:
+ return getRelDataTypeFromHivePrimitiveType(((PrimitiveObjectInspector) oi));
+ case LIST:
+ case MAP:
+ case STRUCT:
+ case UNION:
+ default:
+ throw new RuntimeException("Unknown or unsupported hive type");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <E> Table<E> getTable(String name, Class<E> elementType) {
+ try {
+ org.apache.hadoop.hive.ql.metadata.Table hiveTable =
+ parentSchema.getHiveDb().getTable(getName(), name, false /*throwException*/);
+
+ if (hiveTable == null) {
+ logger.debug("Table name {} is invalid", name);
+ return null;
+ }
+
+ Object selection = schemaProvider.getSelectionBaseOnName(name);
+ if(selection == null) return null;
+
+ final MethodCallExpression call = Expressions.call(getExpression(), //
+ BuiltinMethod.DATA_CONTEXT_GET_TABLE.method, //
+ Expressions.constant(name), //
+ Expressions.constant(Object.class));
+
+ ArrayList<RelDataType> typeList = new ArrayList<>();
+ ArrayList<String> fieldNameList = new ArrayList<>();
+
+ ArrayList<StructField> hiveFields = hiveTable.getFields();
+ for(StructField hiveField : hiveFields) {
+ fieldNameList.add(hiveField.getFieldName());
+ typeList.add(getRelDataTypeFromHiveType(hiveField.getFieldObjectInspector()));
+ }
+
+ final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList);
+ return (Table<E>) new DrillTable(
+ client,
+ this,
+ Object.class,
+ call,
+ rowType,
+ name,
+ null /*storageEngineName*/,
+ selection,
+ config /*storageEngineConfig*/);
+ } catch (HiveException ex) {
+ logger.error("getTable failed", ex);
+ return null;
+ }
+ }
+
+ @Override
+ public Map<String, TableInSchema> getTables() {
+ Map<String, TableInSchema> tables = Maps.newHashMap();
+
+ try {
+ List<String> dbTables = parentSchema.getHiveDb().getAllTables(name);
+ for(String table : dbTables) {
+ TableInfo tableInfo = new TableInfo(this, table);
+ tables.put(tableInfo.name, tableInfo);
+ }
+ } catch (HiveException ex) {
+ throw new RuntimeException("Failed to get tables from HiveMetaStore", ex);
+ }
+
+ return tables;
+ }
+
+ private class TableInfo extends TableInSchema{
+
+ public TableInfo(HiveDatabaseSchema schema, String tableName) {
+ super(schema, schema.getName() + "." + tableName, TableType.TABLE);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <E> Table<E> getTable(Class<E> elementType) {
+ if( !elementType.isAssignableFrom(DrillTable.class)) throw new UnsupportedOperationException();
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
new file mode 100644
index 0000000..ffd6a6e
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.sql.client.full;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentMap;
+
+import net.hydromatic.linq4j.QueryProvider;
+import net.hydromatic.linq4j.expressions.Expression;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.Table;
+import net.hydromatic.optiq.impl.java.JavaTypeFactory;
+
+import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.exec.store.hive.HiveStorageEngineConfig;
+
+import org.apache.drill.jdbc.DrillTable;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
+public class HiveSchema implements Schema{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveSchema.class);
+
+ private final JavaTypeFactory typeFactory;
+ private final Schema parentSchema;
+ private final String name;
+ private final Expression expression;
+ private final QueryProvider queryProvider;
+ private final SchemaProvider schemaProvider;
+ private final DrillClient client;
+ private final HiveStorageEngineConfig config;
+ private Hive hiveDb;
+
+ public HiveSchema(DrillClient client, StorageEngineConfig config, SchemaProvider schemaProvider,
+ JavaTypeFactory typeFactory, Schema parentSchema, String name,
+ Expression expression, QueryProvider queryProvider) {
+ super();
+ this.client = client;
+ this.typeFactory = typeFactory;
+ this.parentSchema = parentSchema;
+ this.name = name;
+ this.expression = expression;
+ this.queryProvider = queryProvider;
+ this.schemaProvider = schemaProvider;
+ this.config = (HiveStorageEngineConfig) config;
+ }
+
+ public Hive getHiveDb() {
+ try {
+ if (hiveDb == null) {
+ this.hiveDb = Hive.get(this.config.getHiveConf());
+ }
+ } catch(HiveException ex) {
+ throw new RuntimeException("Failed to create Hive MetaStore Client", ex);
+ }
+
+ return hiveDb;
+ }
+
+ /**
+ * fetch the database details from
+ */
+ @Override
+ public Schema getSubSchema(String name) {
+ return new HiveDatabaseSchema(client, config, schemaProvider, typeFactory, this,
+ name, expression, queryProvider);
+ }
+
+ @Override
+ public JavaTypeFactory getTypeFactory() {
+ return typeFactory;
+ }
+
+ @Override
+ public Schema getParentSchema() {
+ return parentSchema;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Expression getExpression() {
+ return expression;
+ }
+
+ @Override
+ public QueryProvider getQueryProvider() {
+ return queryProvider;
+ }
+
+ @Override
+ public Collection<TableFunctionInSchema> getTableFunctions(String name) {
+ return Collections.EMPTY_LIST;
+ }
+
+ @Override
+ public Multimap<String, TableFunctionInSchema> getTableFunctions() {
+ return ArrayListMultimap.create();
+ }
+
+ /**
+ * return all databases in metastore
+ */
+ @Override
+ public Collection<String> getSubSchemaNames() {
+ try {
+ return getHiveDb().getAllDatabases();
+ } catch(HiveException ex) {
+ throw new RuntimeException("Failed to get databases from Hive MetaStore", ex);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <E> Table<E> getTable(String name, Class<E> elementType) {
+ return null;
+ }
+
+ /**
+ * Fetch the databases and tables in each database
+ * @return
+ */
+ @Override
+ public Map<String, TableInSchema> getTables() {
+ Map<String, TableInSchema> allTables = Maps.newHashMap();
+
+ Collection<String> dbs = getSubSchemaNames();
+ for(String db : dbs) {
+ // create a sub schema for each database and query the tables within it
+ HiveDatabaseSchema dbSchema = new HiveDatabaseSchema(client, config, schemaProvider,
+ typeFactory, this, db, expression, queryProvider);
+ allTables.putAll(dbSchema.getTables());
+ }
+
+ return allTables;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
index f271bad..4f9d210 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
@@ -17,13 +17,13 @@
*/
package org.apache.drill.jdbc.test;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
+import org.junit.*;
+ import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
+import java.io.File;
+
public class FullEngineTest {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FullEngineTest.class);
@@ -32,8 +32,8 @@ public class FullEngineTest {
static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
// Set a timeout unless we're debugging.
- @Rule public TestRule globalTimeout = IS_DEBUG ? new TestName() : new Timeout(10000);
-
+ @Rule public TestRule globalTimeout = IS_DEBUG ? new TestName() : new Timeout(100000);
+
@Test
@Ignore // since this is a specifically located file.
public void fullSelectStarEngine() throws Exception {
@@ -48,4 +48,39 @@ public class FullEngineTest {
// .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
.sql("select * from \"department.json\" ").displayResults(50);
}
+
+ /**
+ * List tables using the system table (metadata.TABLES) which is same as "!tables" from SQLLine
+ * @throws Exception
+ */
+ @Test(timeout=100000) // derby initialization is slow
+ public void listHiveTables() throws Exception {
+ JdbcAssert.withFull("hive-derby")
+ .sql("select * from \"metadata\".\"TABLES\"")
+ .returns(
+ "tableCat=null; tableSchem=hive-derby; tableName=testdb1.kv_seq; tableType=TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n" +
+ "tableCat=null; tableSchem=hive-derby; tableName=default.all_types; tableType=TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n" +
+ "tableCat=null; tableSchem=hive-derby; tableName=default.kv_text; tableType=TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n" +
+ "tableCat=null; tableSchem=metadata; tableName=COLUMNS; tableType=SYSTEM_TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n" +
+ "tableCat=null; tableSchem=metadata; tableName=TABLES; tableType=SYSTEM_TABLE; remarks=null; typeCat=null; typeSchem=null; typeName=null; selfReferencingColName=null; refGeneration=null\n"
+ );
+ }
+
+ @Test(timeout=1000000) // derby initialization is slow
+ @Ignore // ignore this until Hive SerDe/InputFormat work is intergrated
+ public void selectFromHiveTable() throws Exception {
+ JdbcAssert.withFull("hive-derby")
+ .sql("select * from \"testdb1\".\"kv\"").displayResults(10);
+ }
+
+ @AfterClass
+ public static void cleanup() {
+ // derby creates a derby.log in cwd. Currently there seems to be no way to pass the config
+ // property from hive component to derby to disable or write logging in a different location
+ // need to delete this file for RAT pass
+ File derbyLog = new File("./derby.log");
+ if (derbyLog.exists()) {
+ derbyLog.delete();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a9a7ea84/sqlparser/src/test/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/resources/storage-engines.json b/sqlparser/src/test/resources/storage-engines.json
new file mode 100644
index 0000000..da2f6c1
--- /dev/null
+++ b/sqlparser/src/test/resources/storage-engines.json
@@ -0,0 +1,42 @@
+{
+ "storage":{
+ "parquet-local" :
+ {
+ "type":"parquet",
+ "dfsName" : "file:///"
+ },
+ "parquet-cp" :
+ {
+ "type":"parquet",
+ "dfsName" : "classpath:///"
+ },
+ "jsonl" :
+ {
+ "type":"json",
+ "dfsName" : "file:///"
+ },
+ "json-cp" :
+ {
+ "type":"json",
+ "dfsName" : "classpath:///"
+ },
+ "parquet" :
+ {
+ "type":"parquet",
+ "dfsName" : "file:///"
+ },
+ "hive-derby" :
+ {
+ "type":"hive",
+ "config" :
+ {
+ "hive.metastore.uris" : "",
+ "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../sample-data/drill_hive_db;create=true",
+ "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
+ "fs.default.name" : "file:///",
+ "hive.metastore.sasl.enabled" : "false"
+ }
+ }
+ }
+}
+
[14/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
new file mode 100644
index 0000000..b4c1bf9
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.BitSet;
+import java.util.List;
+
+import net.hydromatic.linq4j.Ord;
+import net.hydromatic.optiq.util.BitSets;
+
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.ValueExpressions;
+import org.apache.drill.common.logical.data.GroupingAggregate;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.exec.planner.torel.ConversionContext;
+import org.eigenbase.rel.AggregateCall;
+import org.eigenbase.rel.AggregateRelBase;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelTraitSet;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Aggregation implemented in Drill.
+ */
+public class DrillAggregateRel extends AggregateRelBase implements DrillRel {
+ /** Creates a DrillAggregateRel. */
+ public DrillAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, BitSet groupSet,
+ List<AggregateCall> aggCalls) throws InvalidRelException {
+ super(cluster, traits, child, groupSet, aggCalls);
+ for (AggregateCall aggCall : aggCalls) {
+ if (aggCall.isDistinct()) {
+ throw new InvalidRelException("DrillAggregateRel does not support DISTINCT aggregates");
+ }
+ }
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ try {
+ return new DrillAggregateRel(getCluster(), traitSet, sole(inputs), getGroupSet(), aggCalls);
+ } catch (InvalidRelException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+
+ GroupingAggregate.Builder builder = GroupingAggregate.builder();
+ builder.setInput(implementor.visitChild(this, 0, getChild()));
+ final List<String> childFields = getChild().getRowType().getFieldNames();
+ final List<String> fields = getRowType().getFieldNames();
+
+ for (int group : BitSets.toIter(groupSet)) {
+ FieldReference fr = new FieldReference(childFields.get(group), ExpressionPosition.UNKNOWN);
+ builder.addKey(fr, fr);
+ }
+
+ for (Ord<AggregateCall> aggCall : Ord.zip(aggCalls)) {
+ FieldReference ref = new FieldReference(fields.get(groupSet.cardinality() + aggCall.i));
+ LogicalExpression expr = toDrill(aggCall.e, childFields, implementor);
+ builder.addExpr(ref, expr);
+ }
+
+ return builder.build();
+ }
+
+
+
+
+ private LogicalExpression toDrill(AggregateCall call, List<String> fn, DrillImplementor implementor) {
+ List<LogicalExpression> args = Lists.newArrayList();
+ for(Integer i : call.getArgList()){
+ args.add(new FieldReference(fn.get(i)));
+ }
+
+ // for count(1).
+ if(args.isEmpty()) args.add(new ValueExpressions.LongExpression(1l));
+ LogicalExpression expr = implementor.getContext().getRegistry().createExpression(call.getAggregation().getName().toLowerCase(), ExpressionPosition.UNKNOWN, args);
+ return expr;
+ }
+
+ public static DrillAggregateRel convert(GroupingAggregate groupBy, ConversionContext value)
+ throws InvalidRelException {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
new file mode 100644
index 0000000..77f1ba6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.AggregateRel;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.*;
+import org.eigenbase.trace.EigenbaseTrace;
+
+import java.util.logging.Logger;
+
+/**
+ * Rule that converts an {@link AggregateRel} to a {@link DrillAggregateRel}, implemented by a Drill "segment" operation
+ * followed by a "collapseaggregate" operation.
+ */
+public class DrillAggregateRule extends RelOptRule {
+ public static final RelOptRule INSTANCE = new DrillAggregateRule();
+ protected static final Logger tracer = EigenbaseTrace.getPlannerTracer();
+
+ private DrillAggregateRule() {
+ super(RelOptHelper.some(AggregateRel.class, Convention.NONE, RelOptHelper.any(RelNode.class)), "DrillAggregateRule");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final AggregateRel aggregate = (AggregateRel) call.rel(0);
+ final RelNode input = call.rel(1);
+ final RelTraitSet traits = aggregate.getTraitSet().plus(DrillRel.CONVENTION);
+ final RelNode convertedInput = convert(input, traits);
+ try {
+ call.transformTo(new DrillAggregateRel(aggregate.getCluster(), traits, convertedInput, aggregate.getGroupSet(),
+ aggregate.getAggCallList()));
+ } catch (InvalidRelException e) {
+ tracer.warning(e.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRel.java
new file mode 100644
index 0000000..cc147e3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRel.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.List;
+
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.exec.planner.torel.ConversionContext;
+import org.eigenbase.rel.FilterRelBase;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptCost;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.rex.RexNode;
+
+/**
+ * Filter implemented in Drill.
+ */
+public class DrillFilterRel extends FilterRelBase implements DrillRel {
+ protected DrillFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) {
+ super(cluster, traits, child, condition);
+ assert getConvention() == CONVENTION;
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new DrillFilterRel(getCluster(), traitSet, sole(inputs), getCondition());
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ return super.computeSelfCost(planner).multiplyBy(0.1);
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ final LogicalOperator input = implementor.visitChild(this, 0, getChild());
+ Filter f = new Filter(DrillOptiq.toDrill(implementor.getContext(), getChild(), getCondition()));
+ f.setInput(input);
+ return f;
+ }
+
+ public static DrillFilterRel convert(Filter filter, ConversionContext context) throws InvalidRelException{
+ RelNode input = context.toRel(filter.getInput());
+ return new DrillFilterRel(context.getCluster(), context.getLogicalTraits(), input, context.toRex(filter.getExpr()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java
new file mode 100644
index 0000000..d4fb239
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.FilterRel;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.*;
+
+/**
+ * Rule that converts a {@link org.eigenbase.rel.FilterRel} to a Drill "filter" operation.
+ */
+public class DrillFilterRule extends RelOptRule {
+ public static final RelOptRule INSTANCE = new DrillFilterRule();
+
+ private DrillFilterRule() {
+ super(RelOptHelper.some(FilterRel.class, Convention.NONE, RelOptHelper.any(RelNode.class)), "DrillFilterRule");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final FilterRel filter = (FilterRel) call.rel(0);
+ final RelNode input = call.rel(1);
+ final RelTraitSet traits = filter.getTraitSet().plus(DrillRel.CONVENTION);
+ final RelNode convertedInput = convert(input, traits);
+ call.transformTo(new DrillFilterRel(filter.getCluster(), traits, convertedInput, filter.getCondition()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java
new file mode 100644
index 0000000..acd218c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.Set;
+
+import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.logical.LogicalPlanBuilder;
+import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
+import org.apache.drill.common.logical.PlanProperties.PlanType;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
+import org.eigenbase.rel.RelNode;
+
+import com.google.common.collect.Sets;
+
+/**
+ * Context for converting a tree of {@link DrillRel} nodes into a Drill logical plan.
+ */
+public class DrillImplementor {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillImplementor.class);
+
+ private Set<DrillTable> tables = Sets.newHashSet();
+ private LogicalPlanBuilder planBuilder = new LogicalPlanBuilder();
+ private LogicalPlan plan;
+ private final DrillParseContext context;
+
+
+ public DrillImplementor(DrillParseContext context, ResultMode mode) {
+ planBuilder.planProperties(PlanType.APACHE_DRILL_LOGICAL, 1, DrillImplementor.class.getName(), "", mode);
+ this.context = context;
+ }
+
+ public DrillParseContext getContext(){
+ return context;
+ }
+
+ public void registerSource(DrillTable table){
+ if(tables.add(table)){
+ planBuilder.addStorageEngine(table.getStorageEngineName(), table.getStorageEngineConfig());
+ }
+ }
+
+ public void go(DrillRel root) {
+ LogicalOperator rootLOP = root.implement(this);
+ rootLOP.accept(new AddOpsVisitor(), null);
+ }
+
+ public LogicalPlan getPlan(){
+ if(plan == null){
+ plan = planBuilder.build();
+ planBuilder = null;
+ }
+ return plan;
+ }
+
+ public LogicalOperator visitChild(DrillRel parent, int ordinal, RelNode child) {
+ return ((DrillRel) child).implement(this);
+ }
+
+ private class AddOpsVisitor extends AbstractLogicalVisitor<Void, Void, RuntimeException> {
+ @Override
+ public Void visitOp(LogicalOperator op, Void value) throws RuntimeException {
+ planBuilder.addLogicalOperator(op);
+ for(LogicalOperator o : op){
+ o.accept(this, null);
+ }
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java
new file mode 100644
index 0000000..c08712e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.JoinCondition;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.exec.planner.torel.ConversionContext;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.JoinRelBase;
+import org.eigenbase.rel.JoinRelType;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptUtil;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeField;
+import org.eigenbase.rex.RexNode;
+import org.eigenbase.rex.RexUtil;
+import org.eigenbase.sql.fun.SqlStdOperatorTable;
+import org.eigenbase.util.Pair;
+
+/**
+ * Join implemented in Drill.
+ */
+public class DrillJoinRel extends JoinRelBase implements DrillRel {
+ private final List<Integer> leftKeys = new ArrayList<>();
+ private final List<Integer> rightKeys = new ArrayList<>();
+
+ /** Creates a DrillJoinRel. */
+ public DrillJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition,
+ JoinRelType joinType) throws InvalidRelException {
+ super(cluster, traits, left, right, condition, joinType, Collections.<String> emptySet());
+
+ RexNode remaining = RelOptUtil.splitJoinCondition(left, right, condition, leftKeys, rightKeys);
+ if (!remaining.isAlwaysTrue()) {
+ throw new InvalidRelException("DrillJoinRel only supports equi-join");
+ }
+ }
+
+ @Override
+ public DrillJoinRel copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType) {
+ try {
+ return new DrillJoinRel(getCluster(), traitSet, left, right, condition, joinType);
+ } catch (InvalidRelException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ final List<String> fields = getRowType().getFieldNames();
+ assert isUnique(fields);
+ final int leftCount = left.getRowType().getFieldCount();
+ final List<String> leftFields = fields.subList(0, leftCount);
+ final List<String> rightFields = fields.subList(leftCount, fields.size());
+
+ final LogicalOperator leftOp = implementInput(implementor, 0, 0, left);
+ final LogicalOperator rightOp = implementInput(implementor, 1, leftCount, right);
+
+ Join.Builder builder = Join.builder();
+ builder.type(joinType);
+ builder.left(leftOp);
+ builder.right(rightOp);
+
+ for (Pair<Integer, Integer> pair : Pair.zip(leftKeys, rightKeys)) {
+ builder.addCondition("==", new FieldReference(leftFields.get(pair.left)), new FieldReference(rightFields.get(pair.right)));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Check to make sure that the fields of the inputs are the same as the output field names. If not, insert a project renaming them.
+ * @param implementor
+ * @param i
+ * @param offset
+ * @param input
+ * @return
+ */
+ private LogicalOperator implementInput(DrillImplementor implementor, int i, int offset, RelNode input) {
+ final LogicalOperator inputOp = implementor.visitChild(this, i, input);
+ assert uniqueFieldNames(input.getRowType());
+ final List<String> fields = getRowType().getFieldNames();
+ final List<String> inputFields = input.getRowType().getFieldNames();
+ final List<String> outputFields = fields.subList(offset, offset + inputFields.size());
+ if (!outputFields.equals(inputFields)) {
+ // Ensure that input field names are the same as output field names.
+ // If there are duplicate field names on left and right, fields will get
+ // lost.
+ return rename(implementor, inputOp, inputFields, outputFields);
+ } else {
+ return inputOp;
+ }
+ }
+
+ private LogicalOperator rename(DrillImplementor implementor, LogicalOperator inputOp, List<String> inputFields, List<String> outputFields) {
+ Project.Builder builder = Project.builder();
+ builder.setInput(inputOp);
+ for (Pair<String, String> pair : Pair.zip(inputFields, outputFields)) {
+ builder.addExpr(new FieldReference(pair.right), new FieldReference(pair.left));
+ }
+ return builder.build();
+ }
+
+ public static DrillJoinRel convert(Join join, ConversionContext context) throws InvalidRelException{
+ RelNode left = context.toRel(join.getLeft());
+ RelNode right = context.toRel(join.getRight());
+
+ List<RexNode> joinConditions = new ArrayList<RexNode>();
+ // right fields appear after the LHS fields.
+ final int rightInputOffset = left.getRowType().getFieldCount();
+ for (JoinCondition condition : join.getConditions()) {
+ RelDataTypeField leftField = left.getRowType().getField(ExprHelper.getFieldName(condition.getLeft()), true);
+ RelDataTypeField rightField = right.getRowType().getField(ExprHelper.getFieldName(condition.getRight()), true);
+ joinConditions.add(
+ context.getRexBuilder().makeCall(
+ SqlStdOperatorTable.EQUALS,
+ context.getRexBuilder().makeInputRef(leftField.getType(), leftField.getIndex()),
+ context.getRexBuilder().makeInputRef(rightField.getType(), rightInputOffset + rightField.getIndex())
+ )
+ );
+ }
+ RexNode rexCondition = RexUtil.composeConjunction(context.getRexBuilder(), joinConditions, false);
+ return new DrillJoinRel(context.getCluster(), context.getLogicalTraits(), left, right, rexCondition, join.getJoinType());
+ }
+
+
+ /**
+ * Returns whether there are any elements in common between left and right.
+ */
+ private static <T> boolean intersects(List<T> left, List<T> right) {
+ return new HashSet<>(left).removeAll(right);
+ }
+
+ private boolean uniqueFieldNames(RelDataType rowType) {
+ return isUnique(rowType.getFieldNames());
+ }
+
+ private static <T> boolean isUnique(List<T> list) {
+ return new HashSet<>(list).size() == list.size();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
new file mode 100644
index 0000000..f32fa59
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.JoinRel;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.*;
+import org.eigenbase.trace.EigenbaseTrace;
+
+import java.util.logging.Logger;
+
+/**
+ * Rule that converts a {@link JoinRel} to a {@link DrillJoinRel}, which is implemented by Drill "join" operation.
+ */
+public class DrillJoinRule extends RelOptRule {
+ public static final RelOptRule INSTANCE = new DrillJoinRule();
+ protected static final Logger tracer = EigenbaseTrace.getPlannerTracer();
+
+ private DrillJoinRule() {
+ super(
+ RelOptHelper.some(JoinRel.class, Convention.NONE, RelOptHelper.any(RelNode.class), RelOptHelper.any(RelNode.class)),
+ "DrillJoinRule");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final JoinRel join = (JoinRel) call.rel(0);
+ final RelNode left = call.rel(1);
+ final RelNode right = call.rel(2);
+ final RelTraitSet traits = join.getTraitSet().plus(DrillRel.CONVENTION);
+
+ final RelNode convertedLeft = convert(left, traits);
+ final RelNode convertedRight = convert(right, traits);
+ try {
+ call.transformTo(new DrillJoinRel(join.getCluster(), traits, convertedLeft, convertedRight, join.getCondition(),
+ join.getJoinType()));
+ } catch (InvalidRelException e) {
+ tracer.warning(e.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRel.java
new file mode 100644
index 0000000..54be052
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRel.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.exec.planner.torel.ConversionContext;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.SingleRel;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.rex.RexLiteral;
+import org.eigenbase.rex.RexNode;
+import org.eigenbase.sql.type.SqlTypeName;
+
+public class DrillLimitRel extends SingleRel implements DrillRel {
+ private RexNode offset;
+ private RexNode fetch;
+
+ public DrillLimitRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, RexNode offset, RexNode fetch) {
+ super(cluster, traitSet, child);
+ this.offset = offset;
+ this.fetch = fetch;
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new DrillLimitRel(getCluster(), traitSet, sole(inputs), offset, fetch);
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ LogicalOperator inputOp = implementor.visitChild(this, 0, getChild());
+
+ // First offset to include into results (inclusive). Null implies it is starting from offset 0
+ int first = offset != null ? Math.max(0, RexLiteral.intValue(offset)) : 0;
+
+ // Last offset to stop including into results (exclusive), translating fetch row counts into an offset.
+ // Null value implies including entire remaining result set from first offset
+ Integer last = fetch != null ? Math.max(0, RexLiteral.intValue(fetch)) + first : null;
+ Limit limit = new Limit(first, last);
+ limit.setInput(inputOp);
+ return limit;
+ }
+
+ public static DrillLimitRel convert(Limit limit, ConversionContext context) throws InvalidRelException{
+ RelNode input = context.toRel(limit.getInput());
+ RexNode first = context.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(limit.getFirst()), context.getTypeFactory().createSqlType(SqlTypeName.INTEGER));
+ RexNode last = context.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(limit.getLast()), context.getTypeFactory().createSqlType(SqlTypeName.INTEGER));
+ return new DrillLimitRel(context.getCluster(), context.getLogicalTraits(), input, first, last);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java
new file mode 100644
index 0000000..85c594e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.SortRel;
+import org.eigenbase.relopt.Convention;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
+
+/**
+ * This rule converts a SortRel that has either a offset and fetch into a Drill Sort and Limit Rel
+ */
+public class DrillLimitRule extends RelOptRule {
+ public static DrillLimitRule INSTANCE = new DrillLimitRule();
+
+ private DrillLimitRule() {
+ super(RelOptHelper.some(SortRel.class, Convention.NONE, RelOptHelper.any(RelNode.class)), "DrillLimitRule");
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final SortRel sort = call.rel(0);
+ return sort.offset != null || sort.fetch != null;
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final SortRel incomingSort = call.rel(0);
+ final RelTraitSet incomingTraits = incomingSort.getTraitSet();
+ RelNode input = incomingSort.getChild();
+
+ // if the Optiq sort rel includes a collation and a limit, we need to create a copy the sort rel that excludes the
+ // limit information.
+ if (!incomingSort.getCollation().getFieldCollations().isEmpty()) {
+ input = incomingSort.copy(incomingTraits, input, incomingSort.getCollation(), null, null);
+ }
+
+ RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.CONVENTION));
+ call.transformTo(new DrillLimitRel(incomingSort.getCluster(), incomingTraits.plus(DrillRel.CONVENTION), convertedInput, incomingSort.offset, incomingSort.fetch));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
new file mode 100644
index 0000000..2e29bd4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.math.BigDecimal;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.ValueExpressions;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.exec.record.NullExpression;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.reltype.RelDataTypeField;
+import org.eigenbase.rex.RexCall;
+import org.eigenbase.rex.RexCorrelVariable;
+import org.eigenbase.rex.RexDynamicParam;
+import org.eigenbase.rex.RexFieldAccess;
+import org.eigenbase.rex.RexInputRef;
+import org.eigenbase.rex.RexLiteral;
+import org.eigenbase.rex.RexLocalRef;
+import org.eigenbase.rex.RexNode;
+import org.eigenbase.rex.RexOver;
+import org.eigenbase.rex.RexRangeRef;
+import org.eigenbase.rex.RexVisitorImpl;
+import org.eigenbase.sql.SqlSyntax;
+import org.eigenbase.sql.fun.SqlStdOperatorTable;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Utilities for Drill's planner.
+ */
+public class DrillOptiq {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillOptiq.class);
+
+ /**
+ * Converts a tree of {@link RexNode} operators into a scalar expression in Drill syntax.
+ */
+ static LogicalExpression toDrill(DrillParseContext context, RelNode input, RexNode expr) {
+ final RexToDrill visitor = new RexToDrill(context, input);
+ return expr.accept(visitor);
+ }
+
+ private static class RexToDrill extends RexVisitorImpl<LogicalExpression> {
+ private final RelNode input;
+ private final DrillParseContext context;
+
+ RexToDrill(DrillParseContext context, RelNode input) {
+ super(true);
+ this.context = context;
+ this.input = input;
+ }
+
+ @Override
+ public LogicalExpression visitInputRef(RexInputRef inputRef) {
+ final int index = inputRef.getIndex();
+ final RelDataTypeField field = input.getRowType().getFieldList().get(index);
+ return new FieldReference(field.getName());
+ }
+
+ @Override
+ public LogicalExpression visitCall(RexCall call) {
+ logger.debug("RexCall {}, {}", call);
+ final SqlSyntax syntax = call.getOperator().getSyntax();
+ switch (syntax) {
+ case BINARY:
+ logger.debug("Binary");
+ LogicalExpression op1 = call.getOperands().get(0).accept(this);
+ LogicalExpression op2 = call.getOperands().get(1).accept(this);
+ return context.getRegistry().createExpression(call.getOperator().getName(), Lists.newArrayList(op1, op2));
+ case FUNCTION:
+ logger.debug("Function");
+ List<LogicalExpression> exprs = Lists.newArrayList();
+ for(RexNode n : call.getOperands()){
+ exprs.add(n.accept(this));
+ }
+ return context.getRegistry().createExpression(call.getOperator().getName().toLowerCase(), Lists.newArrayList(exprs));
+ case SPECIAL:
+ logger.debug("Special");
+ switch(call.getKind()){
+
+ case CAST:
+ return getDrillCastFunctionFromOptiq(call);
+ }
+
+ if (call.getOperator() == SqlStdOperatorTable.ITEM) {
+ SchemaPath left = (SchemaPath) call.getOperands().get(0).accept(this);
+ final RexLiteral literal = (RexLiteral) call.getOperands().get(1);
+ return left.getChild((String) literal.getValue2());
+ }
+
+ // fall through
+ default:
+ throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
+ }
+ }
+
+ private LogicalExpression doUnknown(Object o){
+ logger.warn("Doesn't currently support consumption of {}.", o);
+ return NullExpression.INSTANCE;
+ }
+ @Override
+ public LogicalExpression visitLocalRef(RexLocalRef localRef) {
+ return doUnknown(localRef);
+ }
+
+ @Override
+ public LogicalExpression visitOver(RexOver over) {
+ return doUnknown(over);
+ }
+
+ @Override
+ public LogicalExpression visitCorrelVariable(RexCorrelVariable correlVariable) {
+ return doUnknown(correlVariable);
+ }
+
+ @Override
+ public LogicalExpression visitDynamicParam(RexDynamicParam dynamicParam) {
+ return doUnknown(dynamicParam);
+ }
+
+ @Override
+ public LogicalExpression visitRangeRef(RexRangeRef rangeRef) {
+ return doUnknown(rangeRef);
+ }
+
+ @Override
+ public LogicalExpression visitFieldAccess(RexFieldAccess fieldAccess) {
+ return super.visitFieldAccess(fieldAccess);
+ }
+
+
+ private LogicalExpression getDrillCastFunctionFromOptiq(RexCall call){
+ LogicalExpression arg = call.getOperands().get(0).accept(this);
+ List<LogicalExpression> args = Collections.singletonList(arg);
+ String fname = null;
+ switch(call.getType().getSqlTypeName().getName()){
+ case "VARCHAR": {
+ args = Lists.newArrayList(arg, new LongExpression(call.getType().getPrecision()));
+ return context.getRegistry().createExpression("castVARCHAR", args);
+ }
+ case "INTEGER": fname = "castINT"; break;
+ case "FLOAT": fname = "castFLOAT4"; break;
+ case "DOUBLE": fname = "castFLOAT8"; break;
+ case "DECIMAL": throw new UnsupportedOperationException("Need to add decimal.");
+ default: fname = "cast" + call.getType().getSqlTypeName().getName();
+ }
+ return context.getRegistry().createExpression(fname, args);
+
+ }
+
+
+
+ @Override
+ public LogicalExpression visitLiteral(RexLiteral literal) {
+ switch(literal.getTypeName()){
+ case BIGINT:
+ long l = ((BigDecimal) literal.getValue()).longValue();
+ return ValueExpressions.getBigInt(l);
+ case BOOLEAN:
+ return ValueExpressions.getBit(((Boolean) literal.getValue()));
+ case CHAR:
+ return ValueExpressions.getChar(((String) literal.getValue()));
+ case DOUBLE:
+ double d = ((BigDecimal) literal.getValue()).doubleValue();
+ return ValueExpressions.getFloat8(d);
+ case FLOAT:
+ float f = ((BigDecimal) literal.getValue()).floatValue();
+ return ValueExpressions.getFloat4(f);
+ case INTEGER:
+ case DECIMAL:
+ int i = ((BigDecimal) literal.getValue()).intValue();
+ return ValueExpressions.getInt(i);
+ case VARCHAR:
+ return ValueExpressions.getChar(((String) literal.getValue()));
+ default:
+ throw new UnsupportedOperationException(String.format("Unable to convert the value of %s and type %s to a Drill constant expression.", literal, literal.getTypeName()));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
new file mode 100644
index 0000000..b82fef5
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.apache.drill.common.expression.FunctionRegistry;
+
+public class DrillParseContext {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillParseContext.class);
+
+ private final FunctionRegistry registry;
+
+ public DrillParseContext(FunctionRegistry registry) {
+ super();
+ this.registry = registry;
+ }
+
+ public FunctionRegistry getRegistry(){
+ return registry;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRel.java
new file mode 100644
index 0000000..ee3a0f4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRel.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.exec.planner.torel.ConversionContext;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.ProjectRelBase;
+import org.eigenbase.rel.RelCollation;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptCost;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeField;
+import org.eigenbase.reltype.RelDataTypeFieldImpl;
+import org.eigenbase.reltype.RelRecordType;
+import org.eigenbase.rex.RexNode;
+import org.eigenbase.sql.type.SqlTypeName;
+import org.eigenbase.util.Pair;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
+
+/**
+ * Project implemented in Drill.
+ */
+public class DrillProjectRel extends ProjectRelBase implements DrillRel {
+ protected DrillProjectRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, List<RexNode> exps,
+ RelDataType rowType) {
+ super(cluster, traits, child, exps, rowType, Flags.BOXED);
+ assert getConvention() == CONVENTION;
+ }
+
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new DrillProjectRel(getCluster(), traitSet, sole(inputs), new ArrayList<RexNode>(exps), rowType);
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ return super.computeSelfCost(planner).multiplyBy(0.1);
+ }
+
+ private List<Pair<RexNode, String>> projects() {
+ return Pair.zip(exps, getRowType().getFieldNames());
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ LogicalOperator inputOp = implementor.visitChild(this, 0, getChild());
+ Project.Builder builder = Project.builder();
+ builder.setInput(inputOp);
+ for (Pair<RexNode, String> pair : projects()) {
+ LogicalExpression expr = DrillOptiq.toDrill(implementor.getContext(), getChild(), pair.left);
+ builder.addExpr(new FieldReference("output." + pair.right), expr);
+ }
+ return builder.build();
+ }
+
+ public static DrillProjectRel convert(Project project, ConversionContext context) throws InvalidRelException{
+ RelNode input = context.toRel(project.getInput());
+ List<RelDataTypeField> fields = Lists.newArrayList();
+ List<RexNode> exps = Lists.newArrayList();
+ for(NamedExpression expr : project.getSelections()){
+ fields.add(new RelDataTypeFieldImpl(expr.getRef().getPath().toString(), fields.size(), context.getTypeFactory().createSqlType(SqlTypeName.ANY) ));
+ exps.add(context.toRex(expr.getExpr()));
+ }
+ return new DrillProjectRel(context.getCluster(), context.getLogicalTraits(), input, exps, new RelRecordType(fields));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java
new file mode 100644
index 0000000..bf5bcff
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.ProjectRel;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.Convention;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
+
+/**
+ * Rule that converts a {@link org.eigenbase.rel.ProjectRel} to a Drill "project" operation.
+ */
+public class DrillProjectRule extends RelOptRule {
+ public static final RelOptRule INSTANCE = new DrillProjectRule();
+
+ private DrillProjectRule() {
+ super(RelOptHelper.some(ProjectRel.class, Convention.NONE, RelOptHelper.any(RelNode.class)), "DrillProjectRule");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final ProjectRel project = (ProjectRel) call.rel(0);
+ final RelNode input = call.rel(1);
+ final RelTraitSet traits = project.getTraitSet().plus(DrillRel.CONVENTION);
+ final RelNode convertedInput = convert(input, traits);
+ call.transformTo(new DrillProjectRel(project.getCluster(), traits, convertedInput, project.getProjects(), project
+ .getRowType()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRel.java
new file mode 100644
index 0000000..63a7207
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRel.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.Convention;
+
+/**
+ * Relational expression that is implemented in Drill.
+ */
+public interface DrillRel extends RelNode {
+ /** Calling convention for relational expressions that are "implemented" by
+ * generating Drill logical plans. */
+ Convention CONVENTION = new Convention.Impl("DRILL", DrillRel.class);
+
+ LogicalOperator implement(DrillImplementor implementor);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
new file mode 100644
index 0000000..53da67f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.Iterator;
+
+import net.hydromatic.optiq.tools.RuleSet;
+
+import org.eigenbase.rel.rules.MergeProjectRule;
+import org.eigenbase.rel.rules.PushFilterPastJoinRule;
+import org.eigenbase.rel.rules.PushFilterPastProjectRule;
+import org.eigenbase.rel.rules.PushJoinThroughJoinRule;
+import org.eigenbase.rel.rules.PushSortPastProjectRule;
+import org.eigenbase.rel.rules.ReduceAggregatesRule;
+import org.eigenbase.rel.rules.RemoveDistinctAggregateRule;
+import org.eigenbase.rel.rules.RemoveDistinctRule;
+import org.eigenbase.rel.rules.RemoveSortRule;
+import org.eigenbase.rel.rules.RemoveTrivialCalcRule;
+import org.eigenbase.rel.rules.RemoveTrivialProjectRule;
+import org.eigenbase.rel.rules.SwapJoinRule;
+import org.eigenbase.rel.rules.TableAccessRule;
+import org.eigenbase.rel.rules.UnionToDistinctRule;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.volcano.AbstractConverter.ExpandConversionRule;
+
+import com.google.common.collect.ImmutableSet;
+
+public class DrillRuleSets {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRuleSets.class);
+
+ public static final RuleSet DRILL_BASIC_RULES = new DrillRuleSet(ImmutableSet.of( //
+
+// ExpandConversionRule.instance,
+// SwapJoinRule.instance,
+// RemoveDistinctRule.instance,
+// UnionToDistinctRule.instance,
+// RemoveTrivialProjectRule.instance,
+// RemoveTrivialCalcRule.instance,
+// RemoveSortRule.INSTANCE,
+//
+// TableAccessRule.instance, //
+// MergeProjectRule.instance, //
+// PushFilterPastProjectRule.instance, //
+// PushFilterPastJoinRule.FILTER_ON_JOIN, //
+// RemoveDistinctAggregateRule.instance, //
+// ReduceAggregatesRule.instance, //
+// SwapJoinRule.instance, //
+// PushJoinThroughJoinRule.RIGHT, //
+// PushJoinThroughJoinRule.LEFT, //
+// PushSortPastProjectRule.INSTANCE, //
+
+ DrillScanRule.INSTANCE,
+ DrillFilterRule.INSTANCE,
+ DrillProjectRule.INSTANCE,
+ DrillAggregateRule.INSTANCE,
+
+ DrillLimitRule.INSTANCE,
+ DrillSortRule.INSTANCE,
+ DrillJoinRule.INSTANCE,
+ DrillUnionRule.INSTANCE
+ ));
+
+
+ private static class DrillRuleSet implements RuleSet{
+ final ImmutableSet<RelOptRule> rules;
+
+ public DrillRuleSet(ImmutableSet<RelOptRule> rules) {
+ super();
+ this.rules = rules;
+ }
+
+ @Override
+ public Iterator<RelOptRule> iterator() {
+ return rules.iterator();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
new file mode 100644
index 0000000..afc2d1b
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.apache.drill.common.JSONOptions;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.exec.planner.torel.ConversionContext;
+import org.eigenbase.rel.TableAccessRelBase;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptTable;
+import org.eigenbase.relopt.RelTraitSet;
+
+/**
+ * GroupScan of a Drill table.
+ */
+public class DrillScanRel extends TableAccessRelBase implements DrillRel {
+ private final DrillTable drillTable;
+
+ /** Creates a DrillScan. */
+ public DrillScanRel(RelOptCluster cluster, RelTraitSet traits, RelOptTable table) {
+ super(cluster, traits, table);
+ assert getConvention() == CONVENTION;
+ this.drillTable = table.unwrap(DrillTable.class);
+ assert drillTable != null;
+ }
+
+// @Override
+// public void register(RelOptPlanner planner) {
+// super.register(planner);
+// DrillOptiq.registerStandardPlannerRules(planner);
+// }
+
+ public LogicalOperator implement(DrillImplementor implementor) {
+ Scan.Builder builder = Scan.builder();
+ builder.storageEngine(drillTable.getStorageEngineName());
+ builder.selection(new JSONOptions(drillTable.getSelection()));
+ //builder.outputReference(new FieldReference("_MAP"));
+ implementor.registerSource(drillTable);
+ return builder.build();
+ }
+
+ public static DrillScanRel convert(Scan scan, ConversionContext context){
+ return new DrillScanRel(context.getCluster(), context.getLogicalTraits(), context.getTable(scan));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java
new file mode 100644
index 0000000..58e648a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import net.hydromatic.optiq.rules.java.JavaRules.EnumerableTableAccessRel;
+
+import org.eigenbase.relopt.Convention;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
+
+public class DrillScanRule extends RelOptRule {
+ public static final RelOptRule INSTANCE = new DrillScanRule();
+
+ private DrillScanRule() {
+ super(RelOptHelper.any(EnumerableTableAccessRel.class), "DrillTableRule");
+ }
+
+
+
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final EnumerableTableAccessRel access = (EnumerableTableAccessRel) call.rel(0);
+ final RelTraitSet traits = access.getTraitSet().plus(DrillRel.CONVENTION);
+ call.transformTo(new DrillScanRel(access.getCluster(), traits, access.getTable()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScreenRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScreenRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScreenRel.java
new file mode 100644
index 0000000..829947a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScreenRel.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.List;
+
+import net.hydromatic.optiq.impl.java.JavaTypeFactory;
+import net.hydromatic.optiq.rules.java.EnumerableConvention;
+import net.hydromatic.optiq.rules.java.EnumerableRel;
+import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
+import net.hydromatic.optiq.rules.java.JavaRowFormat;
+import net.hydromatic.optiq.rules.java.PhysType;
+import net.hydromatic.optiq.rules.java.PhysTypeImpl;
+
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Store;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.SingleRel;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptCost;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.relopt.RelTraitSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Relational expression that converts from Drill to Enumerable. At runtime it executes a Drill query and returns the
+ * results as an {@link net.hydromatic.linq4j.Enumerable}.
+ */
+public class DrillScreenRel extends SingleRel implements DrillRel {
+ private static final Logger logger = LoggerFactory.getLogger(DrillScreenRel.class);
+
+ private PhysType physType;
+
+ public DrillScreenRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode input) {
+ super(cluster, traitSet, input);
+ assert input.getConvention() == DrillRel.CONVENTION;
+ physType = PhysTypeImpl.of((JavaTypeFactory) cluster.getTypeFactory(), input.getRowType(), JavaRowFormat.ARRAY);
+ }
+
+ public PhysType getPhysType() {
+ return physType;
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ return super.computeSelfCost(planner).multiplyBy(.1);
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new DrillScreenRel(getCluster(), traitSet, sole(inputs));
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ LogicalOperator childOp = implementor.visitChild(this, 0, getChild());
+ return Store.builder().setInput(childOp).storageEngine("--SCREEN--").build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRel.java
new file mode 100644
index 0000000..04af9d5
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRel.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Order.Ordering;
+import org.apache.drill.exec.planner.torel.ConversionContext;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelCollation;
+import org.eigenbase.rel.RelCollationImpl;
+import org.eigenbase.rel.RelFieldCollation;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.SortRel;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.rex.RexNode;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * Sort implemented in Drill.
+ */
+public class DrillSortRel extends SortRel implements DrillRel {
+
+ /** Creates a DrillSortRel. */
+ public DrillSortRel(RelOptCluster cluster, RelTraitSet traits, RelNode input, RelCollation collation) {
+ super(cluster, traits, input, collation);
+ }
+
+ /** Creates a DrillSortRel with offset and fetch. */
+ public DrillSortRel(RelOptCluster cluster, RelTraitSet traits, RelNode input, RelCollation collation, RexNode offset, RexNode fetch) {
+ super(cluster, traits, input, collation, offset, fetch);
+ }
+
+ @Override
+ public DrillSortRel copy(RelTraitSet traitSet, RelNode input, RelCollation collation, RexNode offset, RexNode fetch) {
+ return new DrillSortRel(getCluster(), traitSet, input, collation, offset, fetch);
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ final Order.Builder builder = Order.builder();
+ builder.setInput(implementor.visitChild(this, 0, getChild()));
+
+ final List<String> childFields = getChild().getRowType().getFieldNames();
+ for(RelFieldCollation fieldCollation : this.collation.getFieldCollations()){
+ builder.addOrdering(fieldCollation.getDirection(),
+ new FieldReference(childFields.get(fieldCollation.getFieldIndex())),
+ fieldCollation.nullDirection);
+ }
+ return builder.build();
+ }
+
+
+ public static RelNode convert(Order order, ConversionContext context) throws InvalidRelException{
+
+ // if there are compound expressions in the order by, we need to convert into projects on either side.
+ RelNode input = context.toRel(order.getInput());
+ List<String> fields = input.getRowType().getFieldNames();
+
+ // build a map of field names to indices.
+ Map<String, Integer> fieldMap = Maps.newHashMap();
+ int i =0;
+ for(String field : fields){
+ fieldMap.put(field, i);
+ i++;
+ }
+
+ List<RelFieldCollation> collations = Lists.newArrayList();
+
+ for(Ordering o : order.getOrderings()){
+ String fieldName = ExprHelper.getFieldName(o.getExpr());
+ int fieldId = fieldMap.get(fieldName);
+ RelFieldCollation c = new RelFieldCollation(fieldId, o.getDirection(), o.getNullDirection());
+ }
+ return new DrillSortRel(context.getCluster(), context.getLogicalTraits(), input, RelCollationImpl.of(collations));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java
new file mode 100644
index 0000000..c968e85
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.SortRel;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.*;
+
+/**
+ * Rule that converts an {@link SortRel} to a {@link DrillSortRel}, implemented by a Drill "order" operation.
+ */
+public class DrillSortRule extends RelOptRule {
+ public static final RelOptRule INSTANCE = new DrillSortRule();
+
+ private DrillSortRule() {
+ super(RelOptHelper.some(SortRel.class, Convention.NONE, RelOptHelper.any(RelNode.class)), "DrillSortRule");
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final SortRel sort = call.rel(0);
+ return sort.offset == null && sort.fetch == null;
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+
+ final SortRel sort = call.rel(0);
+
+ final RelNode input = call.rel(1);
+ final RelTraitSet traits = sort.getTraitSet().plus(DrillRel.CONVENTION);
+
+ final RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.CONVENTION));
+ call.transformTo(new DrillSortRel(sort.getCluster(), traits, convertedInput, sort.getCollation()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillStoreRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillStoreRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillStoreRel.java
new file mode 100644
index 0000000..30c7810
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillStoreRel.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.List;
+
+import net.hydromatic.optiq.prepare.Prepare.CatalogReader;
+
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.TableModificationRelBase;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptTable;
+import org.eigenbase.relopt.RelTraitSet;
+
+public class DrillStoreRel extends TableModificationRelBase implements DrillRel{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillStoreRel.class);
+
+ protected DrillStoreRel(RelOptCluster cluster, RelTraitSet traits, RelOptTable table, CatalogReader catalogReader,
+ RelNode child, Operation operation, List<String> updateColumnList, boolean flattened) {
+ super(cluster, traits, table, catalogReader, child, operation, updateColumnList, flattened);
+
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
new file mode 100644
index 0000000..30dd48d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.Collections;
+
+import net.hydromatic.optiq.Schema.TableType;
+import net.hydromatic.optiq.Statistic;
+import net.hydromatic.optiq.Statistics;
+import net.hydromatic.optiq.Table;
+
+import org.apache.drill.common.logical.StorageEngineConfig;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptTable;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.sql.type.SqlTypeName;
+
+/** Optiq Table used by Drill. */
+public class DrillTable implements Table{
+
+ private final String name;
+ private final String storageEngineName;
+ public final StorageEngineConfig storageEngineConfig;
+ private Object selection;
+
+
+ /** Creates a DrillTable. */
+ public DrillTable(String name, String storageEngineName, Object selection, StorageEngineConfig storageEngineConfig) {
+ this.name = name;
+ this.selection = selection;
+ this.storageEngineConfig = storageEngineConfig;
+ this.storageEngineName = storageEngineName;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public StorageEngineConfig getStorageEngineConfig(){
+ return storageEngineConfig;
+ }
+
+ public Object getSelection() {
+ return selection;
+ }
+
+ public String getStorageEngineName() {
+ return storageEngineName;
+ }
+
+ @Override
+ public Statistic getStatistic() {
+ return Statistics.UNKNOWN;
+ }
+
+ public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable table) {
+ return new DrillScanRel(context.getCluster(),
+ context.getCluster().traitSetOf(DrillRel.CONVENTION),
+ table);
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ return new RelDataTypeDrillImpl(typeFactory);
+ }
+
+ @Override
+ public TableType getJdbcTableType() {
+ return null;
+ }
+
+
+
+// /** Factory for custom tables in Optiq schema. */
+// @SuppressWarnings("UnusedDeclaration")
+// public static class Factory implements TableFactory<DrillTable> {
+//
+// @Override
+// public DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) {
+//
+// final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
+// final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
+// inputConfig.path = "/" + name.toLowerCase() + ".json";
+// inputConfig.type = DataWriter.ConverterType.JSON;
+// return createTable(schema.getTypeFactory(), (MutableSchema) schema, name, "donuts-json", rseConfig, inputConfig);
+// }
+// }
+
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
new file mode 100644
index 0000000..1be9caf
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.List;
+
+import net.hydromatic.linq4j.Ord;
+
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.common.logical.data.Union;
+import org.apache.drill.exec.planner.torel.ConversionContext;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.UnionRelBase;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptCost;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.relopt.RelTraitSet;
+
+/**
+ * Union implemented in Drill.
+ */
+public class DrillUnionRel extends UnionRelBase implements DrillRel {
+ /** Creates a DrillUnionRel. */
+ public DrillUnionRel(RelOptCluster cluster, RelTraitSet traits,
+ List<RelNode> inputs, boolean all) {
+ super(cluster, traits, inputs, all);
+ }
+
+ @Override
+ public DrillUnionRel copy(RelTraitSet traitSet, List<RelNode> inputs,
+ boolean all) {
+ return new DrillUnionRel(getCluster(), traitSet, inputs, all);
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ // divide cost by two to ensure cheaper than EnumerableDrillRel
+ return super.computeSelfCost(planner).multiplyBy(.5);
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ Union.Builder builder = Union.builder();
+ for (Ord<RelNode> input : Ord.zip(inputs)) {
+ builder.addInput(implementor.visitChild(this, input.i, input.e));
+ }
+ builder.setDistinct(!all);
+ return builder.build();
+ }
+
+ public static DrillUnionRel convert(Union union, ConversionContext context) throws InvalidRelException{
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
new file mode 100644
index 0000000..bc1e6f4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.eigenbase.rel.UnionRel;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Rule that converts a {@link UnionRel} to a {@link DrillUnionRel}, implemented by a "union" operation.
+ */
+public class DrillUnionRule extends RelOptRule {
+ public static final RelOptRule INSTANCE = new DrillUnionRule();
+
+ private DrillUnionRule() {
+ super(RelOptHelper.any(UnionRel.class, Convention.NONE), "DrillUnionRule");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final UnionRel union = (UnionRel) call.rel(0);
+ final RelTraitSet traits = union.getTraitSet().plus(DrillRel.CONVENTION);
+ final List<RelNode> convertedInputs = new ArrayList<>();
+ for (RelNode input : union.getInputs()) {
+ final RelNode convertedInput = convert(input, traits);
+ convertedInputs.add(convertedInput);
+ }
+ call.transformTo(new DrillUnionRel(union.getCluster(), traits, convertedInputs, union.all));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
new file mode 100644
index 0000000..e770181
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.List;
+
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.ValuesRelBase;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptCost;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.rex.RexLiteral;
+
+/**
+ * Values implemented in Drill.
+ */
+public class DrillValuesRel extends ValuesRelBase implements DrillRel {
+ protected DrillValuesRel(RelOptCluster cluster, RelDataType rowType, List<List<RexLiteral>> tuples, RelTraitSet traits) {
+ super(cluster, rowType, tuples, traits);
+ assert getConvention() == CONVENTION;
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ assert inputs.isEmpty();
+ return new DrillValuesRel(getCluster(), rowType, tuples, traitSet);
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ return super.computeSelfCost(planner).multiplyBy(0.1);
+ }
+
+ @Override
+ public LogicalOperator implement(DrillImplementor implementor) {
+ // Update when https://issues.apache.org/jira/browse/DRILL-57 fixed
+ throw new UnsupportedOperationException();
+ }
+}
[23/38] Integrate new SQL changes with Hive storage engine,
moving to automatic file detection. Rename storage engine to storage
plugin. Separate storage plugins from format plugins,
updating Parquet and JSON to format engines. Refactor distribution log
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
new file mode 100644
index 0000000..a1d575c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import java.io.IOException;
+
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.AbstractStoragePlugin;
+import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class HiveStoragePlugin extends AbstractStoragePlugin {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveStoragePlugin.class);
+
+ private final HiveStoragePluginConfig config;
+ private final HiveConf hiveConf;
+ private final HiveSchemaFactory schemaFactory;
+ private final DrillbitContext context;
+ private final String name;
+
+ public HiveStoragePlugin(HiveStoragePluginConfig config, DrillbitContext context, String name) throws ExecutionSetupException {
+ this.config = config;
+ this.context = context;
+ this.schemaFactory = new HiveSchemaFactory(config, name, config.getHiveConf());
+ this.hiveConf = config.getHiveConf();
+ this.name = name;
+ }
+
+ public HiveStoragePluginConfig getConfig() {
+ return config;
+ }
+
+ public String getName(){
+ return name;
+ }
+
+ public DrillbitContext getContext() {
+ return context;
+ }
+
+ @Override
+ public HiveScan getPhysicalScan(Scan scan) throws IOException {
+ HiveReadEntry hiveReadEntry = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){});
+ try {
+ return new HiveScan(hiveReadEntry, this, null);
+ } catch (ExecutionSetupException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public Schema createAndAddSchema(SchemaPlus parent) {
+ return schemaFactory.add(parent);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePluginConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePluginConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePluginConfig.java
new file mode 100644
index 0000000..c9d76e5
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePluginConfig.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.drill.common.logical.StoragePluginConfigBase;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import java.util.Map;
+
+@JsonTypeName("hive")
+public class HiveStoragePluginConfig extends StoragePluginConfigBase {
+ @JsonProperty
+ public Map<String, String> configProps;
+ @JsonIgnore
+ private HiveConf hiveConf;
+
+ @JsonIgnore
+ public HiveConf getHiveConf() {
+ if (hiveConf == null) {
+ hiveConf = new HiveConf();
+ if (configProps != null) {
+ for (Map.Entry<String, String> entry : configProps.entrySet()) {
+ hiveConf.set(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ return hiveConf;
+ }
+
+ @JsonCreator
+ public HiveStoragePluginConfig(@JsonProperty("config") Map<String, String> props) {
+ this.configProps = props;
+ }
+
+ @Override
+ public int hashCode() {
+ return configProps != null ? configProps.hashCode() : 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ HiveStoragePluginConfig that = (HiveStoragePluginConfig) o;
+
+ if (configProps != null ? !configProps.equals(that.configProps) : that.configProps != null) return false;
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
new file mode 100644
index 0000000..abec2c5
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive.schema;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.store.hive.HiveReadEntry;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.sql.SqlCollation;
+import org.eigenbase.sql.type.SqlTypeName;
+
+public class DrillHiveTable extends DrillTable{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHiveTable.class);
+
+ private final Table hiveTable;
+
+ public DrillHiveTable(String storageEngineName, HiveReadEntry readEntry, StoragePluginConfig storageEngineConfig) {
+ super(storageEngineName, readEntry, storageEngineConfig);
+ this.hiveTable = new org.apache.hadoop.hive.ql.metadata.Table(readEntry.getTable());
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ ArrayList<RelDataType> typeList = new ArrayList<>();
+ ArrayList<String> fieldNameList = new ArrayList<>();
+
+ ArrayList<StructField> hiveFields = hiveTable.getFields();
+ for(StructField hiveField : hiveFields) {
+ fieldNameList.add(hiveField.getFieldName());
+ typeList.add(getRelDataTypeFromHiveType(typeFactory, hiveField.getFieldObjectInspector()));
+ }
+
+ for (FieldSchema field : hiveTable.getPartitionKeys()) {
+ fieldNameList.add(field.getName());
+ typeList.add(getRelDataTypeFromHiveTypeString(typeFactory, field.getType()));
+ }
+
+ final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList);
+ return rowType;
+ }
+
+ private RelDataType getRelDataTypeFromHiveTypeString(RelDataTypeFactory typeFactory, String type) {
+ switch(type) {
+ case "boolean":
+ return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+
+ case "tinyint":
+ return typeFactory.createSqlType(SqlTypeName.TINYINT);
+
+ case "smallint":
+ return typeFactory.createSqlType(SqlTypeName.SMALLINT);
+
+ case "int":
+ return typeFactory.createSqlType(SqlTypeName.INTEGER);
+
+ case "bigint":
+ return typeFactory.createSqlType(SqlTypeName.BIGINT);
+
+ case "float":
+ return typeFactory.createSqlType(SqlTypeName.FLOAT);
+
+ case "double":
+ return typeFactory.createSqlType(SqlTypeName.DOUBLE);
+
+ case "date":
+ return typeFactory.createSqlType(SqlTypeName.DATE);
+
+ case "timestamp":
+ return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
+
+ case "binary":
+ return typeFactory.createSqlType(SqlTypeName.BINARY);
+
+ case "decimal":
+ return typeFactory.createSqlType(SqlTypeName.DECIMAL);
+
+ case "string":
+ case "varchar": {
+ return typeFactory.createTypeWithCharsetAndCollation(
+ typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
+ Charset.forName("ISO-8859-1"), /*unicode char set*/
+ SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
+ );
+ }
+
+ default:
+ throw new RuntimeException("Unknown or unsupported hive type: " + type);
+ }
+ }
+
+ private RelDataType getRelDataTypeFromHivePrimitiveType(RelDataTypeFactory typeFactory, PrimitiveObjectInspector poi) {
+ switch(poi.getPrimitiveCategory()) {
+ case BOOLEAN:
+ return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+ case BYTE:
+ return typeFactory.createSqlType(SqlTypeName.TINYINT);
+
+ case SHORT:
+ return typeFactory.createSqlType(SqlTypeName.SMALLINT);
+
+ case INT:
+ return typeFactory.createSqlType(SqlTypeName.INTEGER);
+
+ case LONG:
+ return typeFactory.createSqlType(SqlTypeName.BIGINT);
+
+ case FLOAT:
+ return typeFactory.createSqlType(SqlTypeName.FLOAT);
+
+ case DOUBLE:
+ return typeFactory.createSqlType(SqlTypeName.DOUBLE);
+
+ case DATE:
+ return typeFactory.createSqlType(SqlTypeName.DATE);
+
+ case TIMESTAMP:
+ return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
+
+ case BINARY:
+ return typeFactory.createSqlType(SqlTypeName.BINARY);
+
+ case DECIMAL:
+ return typeFactory.createSqlType(SqlTypeName.DECIMAL);
+
+ case STRING:
+ case VARCHAR: {
+ return typeFactory.createTypeWithCharsetAndCollation(
+ typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
+ Charset.forName("ISO-8859-1"), /*unicode char set*/
+ SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
+ );
+ }
+
+ case UNKNOWN:
+ case VOID:
+ default:
+ throw new RuntimeException("Unknown or unsupported hive type");
+ }
+ }
+
+ private RelDataType getRelDataTypeFromHiveType(RelDataTypeFactory typeFactory, ObjectInspector oi) {
+ switch(oi.getCategory()) {
+ case PRIMITIVE:
+ return getRelDataTypeFromHivePrimitiveType(typeFactory, ((PrimitiveObjectInspector) oi));
+ case LIST:
+ case MAP:
+ case STRUCT:
+ case UNION:
+ default:
+ throw new RuntimeException("Unknown or unsupported hive type");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
new file mode 100644
index 0000000..e4ff372
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive.schema;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.SchemaHolder;
+import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory.HiveSchema;
+
+import com.google.common.collect.Sets;
+
+public class HiveDatabaseSchema extends AbstractSchema{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveDatabaseSchema.class);
+
+ private final HiveSchema hiveSchema;
+ private final Set<String> tables;
+
+ public HiveDatabaseSchema( //
+ List<String> tableList, //
+ HiveSchema hiveSchema, //
+ SchemaHolder parentSchema, //
+ String name) {
+ super(parentSchema, name);
+ this.hiveSchema = hiveSchema;
+ this.tables = Sets.newHashSet(tableList);
+ }
+
+ @Override
+ public DrillTable getTable(String tableName) {
+ return hiveSchema.getDrillTable(this.name, tableName);
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return tables;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
new file mode 100644
index 0000000..091381f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
@@ -0,0 +1,274 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive.schema;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.SchemaFactory;
+import org.apache.drill.exec.store.SchemaHolder;
+import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema;
+import org.apache.drill.exec.store.hive.HiveReadEntry;
+import org.apache.drill.exec.store.hive.HiveStoragePluginConfig;
+import org.apache.drill.exec.store.hive.HiveTable;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.thrift.TException;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+public class HiveSchemaFactory implements SchemaFactory {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveSchemaFactory.class);
+
+ private static final String DATABASES = "databases";
+
+ private final HiveMetaStoreClient mClient;
+ private LoadingCache<String, List<String>> databases;
+ private LoadingCache<String, List<String>> tableNameLoader;
+ private LoadingCache<String, LoadingCache<String, HiveReadEntry>> tableLoaders;
+ private HiveStoragePluginConfig pluginConfig;
+ private final String schemaName;
+
+ public HiveSchemaFactory(HiveStoragePluginConfig pluginConfig, String name, HiveConf hiveConf) throws ExecutionSetupException {
+ this.schemaName = name;
+ this.pluginConfig = pluginConfig;
+
+ try {
+ this.mClient = new HiveMetaStoreClient(hiveConf);
+ } catch (MetaException e) {
+ throw new ExecutionSetupException("Failure setting up Hive metastore client.", e);
+ }
+
+ databases = CacheBuilder //
+ .newBuilder() //
+ .expireAfterAccess(1, TimeUnit.MINUTES) //
+ .build(new DatabaseLoader());
+
+ tableNameLoader = CacheBuilder //
+ .newBuilder() //
+ .expireAfterAccess(1, TimeUnit.MINUTES) //
+ .build(new TableNameLoader());
+
+ tableLoaders = CacheBuilder //
+ .newBuilder() //
+ .expireAfterAccess(4, TimeUnit.HOURS) //
+ .maximumSize(20) //
+ .build(new TableLoaderLoader());
+ }
+
+ private class TableNameLoader extends CacheLoader<String, List<String>> {
+
+ @Override
+ public List<String> load(String dbName) throws Exception {
+ try {
+ return mClient.getAllTables(dbName);
+ } catch (TException e) {
+ logger.warn("Failure while attempting to get hive tables", e);
+ mClient.reconnect();
+ return mClient.getAllTables(dbName);
+ }
+ }
+
+ }
+
+ private class DatabaseLoader extends CacheLoader<String, List<String>> {
+
+ @Override
+ public List<String> load(String key) throws Exception {
+ if (!DATABASES.equals(key))
+ throw new UnsupportedOperationException();
+ try {
+ return mClient.getAllDatabases();
+ } catch (TException e) {
+ logger.warn("Failure while attempting to get hive tables", e);
+ mClient.reconnect();
+ return mClient.getAllDatabases();
+ }
+ }
+ }
+
+ private class TableLoaderLoader extends CacheLoader<String, LoadingCache<String, HiveReadEntry>> {
+
+ @Override
+ public LoadingCache<String, HiveReadEntry> load(String key) throws Exception {
+ return CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.MINUTES).build(new TableLoader(key));
+ }
+
+ }
+
+ private class TableLoader extends CacheLoader<String, HiveReadEntry> {
+
+ private final String dbName;
+
+ public TableLoader(String dbName) {
+ super();
+ this.dbName = dbName;
+ }
+
+ @Override
+ public HiveReadEntry load(String key) throws Exception {
+ Table t = null;
+ try {
+ t = mClient.getTable(dbName, key);
+ } catch (TException e) {
+ mClient.reconnect();
+ t = mClient.getTable(dbName, key);
+ }
+
+ if (t == null)
+ throw new UnknownTableException(String.format("Unable to find table '%s'.", key));
+
+ List<Partition> partitions = null;
+ try {
+ partitions = mClient.listPartitions(dbName, key, Short.MAX_VALUE);
+ } catch (TException e) {
+ mClient.reconnect();
+ partitions = mClient.listPartitions(dbName, key, Short.MAX_VALUE);
+ }
+
+ List<HiveTable.HivePartition> hivePartitions = Lists.newArrayList();
+ for (Partition part : partitions) {
+ hivePartitions.add(new HiveTable.HivePartition(part));
+ }
+
+ if (hivePartitions.size() == 0)
+ hivePartitions = null;
+ return new HiveReadEntry(new HiveTable(t), hivePartitions);
+
+ }
+
+ }
+
+ @Override
+ public Schema add(SchemaPlus parent) {
+ HiveSchema schema = new HiveSchema(new SchemaHolder(parent), schemaName);
+ SchemaPlus hPlus = parent.add(schema);
+ schema.setHolder(hPlus);
+ return schema;
+ }
+
+ class HiveSchema extends AbstractSchema {
+
+ private final SchemaHolder holder = new SchemaHolder();
+
+ private HiveDatabaseSchema defaultSchema;
+
+ public HiveSchema(SchemaHolder parentSchema, String name) {
+ super(parentSchema, name);
+ getSubSchema("default");
+ }
+
+ @Override
+ public Schema getSubSchema(String name) {
+ List<String> tables;
+ try {
+ tables = tableNameLoader.get(name);
+ HiveDatabaseSchema schema = new HiveDatabaseSchema(tables, this, holder, name);
+ if(name.equals("default")){
+ this.defaultSchema = schema;
+ }
+ return schema;
+ } catch (ExecutionException e) {
+ logger.warn("Failure while attempting to access HiveDatabase '{}'.", name, e.getCause());
+ return null;
+ }
+
+ }
+
+
+ void setHolder(SchemaPlus plusOfThis){
+ holder.setSchema(plusOfThis);
+ for(String s : getSubSchemaNames()){
+ plusOfThis.add(getSubSchema(s));
+ }
+ }
+
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ try{
+ List<String> dbs = databases.get(DATABASES);
+ return Sets.newHashSet(dbs);
+ }catch(ExecutionException e){
+ logger.warn("Failure while getting Hive database list.", e);
+ }
+ return super.getSubSchemaNames();
+ }
+
+ @Override
+ public DrillTable getTable(String name) {
+ if(defaultSchema == null){
+ return super.getTable(name);
+ }
+ return defaultSchema.getTable(name);
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ if(defaultSchema == null){
+ return super.getTableNames();
+ }
+ return defaultSchema.getTableNames();
+ }
+
+ List<String> getTableNames(String dbName){
+ try{
+ return tableNameLoader.get(dbName);
+ }catch(ExecutionException e){
+ logger.warn("Failure while loading table names for database '{}'.", dbName, e.getCause());
+ return Collections.emptyList();
+ }
+ }
+
+ DrillTable getDrillTable(String dbName, String t){
+ HiveReadEntry entry = getSelectionBaseOnName(dbName, t);
+ if(entry == null) return null;
+ return new DrillHiveTable(schemaName, entry, pluginConfig);
+ }
+
+ HiveReadEntry getSelectionBaseOnName(String dbName, String t) {
+ if(dbName == null) dbName = "default";
+ try{
+ return tableLoaders.get(dbName).get(t);
+ }catch(ExecutionException e){
+ logger.warn("Exception occurred while trying to read table. {}.{}", dbName, t, e.getCause());
+ return null;
+ }
+ }
+
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
deleted file mode 100644
index 4782d82..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.json;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.physical.EndpointAffinity;
-import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.base.AbstractGroupScan;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.Size;
-import org.apache.drill.exec.physical.base.SubScan;
-import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.store.StorageEngineRegistry;
-
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-@JsonTypeName("json-scan")
-public class JSONGroupScan extends AbstractGroupScan {
- private static int ESTIMATED_RECORD_SIZE = 1024; // 1kb
- private final JSONStorageEngine engine;
-
- private LinkedList<JSONGroupScan.ScanEntry>[] mappings;
- private final List<JSONGroupScan.ScanEntry> readEntries;
- private final OperatorCost cost;
- private final Size size;
- private final FieldReference ref;
- private final List<SchemaPath> columns;
-
- @JsonCreator
- public JSONGroupScan(@JsonProperty("entries") List<ScanEntry> entries,
- @JsonProperty("storageengine") JSONStorageEngineConfig storageEngineConfig,
- @JacksonInject StorageEngineRegistry engineRegistry, @JsonProperty("ref") FieldReference ref,
- @JsonProperty("columns") List<SchemaPath> columns) throws ExecutionSetupException {
- this(entries, (JSONStorageEngine) engineRegistry.getEngine(storageEngineConfig), ref, columns);
- }
-
- public JSONGroupScan(List<ScanEntry> entries, JSONStorageEngine engine, FieldReference ref,
- List<SchemaPath> columns) {
- this.engine = engine;
- this.readEntries = entries;
- OperatorCost cost = new OperatorCost(0, 0, 0, 0);
- Size size = new Size(0, 0);
- for (JSONGroupScan.ScanEntry r : readEntries) {
- cost = cost.add(r.getCost());
- size = size.add(r.getSize());
- }
- this.cost = cost;
- this.size = size;
- this.ref = ref;
- this.columns = columns;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void applyAssignments(List<CoordinationProtos.DrillbitEndpoint> endpoints) {
- checkArgument(endpoints.size() <= readEntries.size());
-
- mappings = new LinkedList[endpoints.size()];
-
- int i = 0;
- for (ScanEntry e : readEntries) {
- if (i == endpoints.size()) i = 0;
- LinkedList entries = mappings[i];
- if (entries == null) {
- entries = new LinkedList<>();
- mappings[i] = entries;
- }
- entries.add(e);
- i++;
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException{
- checkArgument(minorFragmentId < mappings.length, "Mappings length [%s] should be longer than minor fragment id [%s] but it isn't.", mappings.length, minorFragmentId);
- return new JSONSubScan(mappings[minorFragmentId], engine, ref, columns);
- }
-
- @Override
- public List<EndpointAffinity> getOperatorAffinity() {
- return Collections.emptyList();
- }
-
- @Override
- @JsonIgnore
- public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
- return new JSONGroupScan(readEntries, engine, ref, columns);
- }
-
- public static class ScanEntry implements ReadEntry {
- private final String path;
- private Size size;
-
- @JsonCreator
- public ScanEntry(@JsonProperty("path") String path) {
- this.path = path;
- size = new Size(ESTIMATED_RECORD_SIZE, ESTIMATED_RECORD_SIZE);
- }
-
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(1, 1, 2, 2);
- }
-
- @Override
- public Size getSize() {
- return size;
- }
-
- public String getPath() {
- return path;
- }
- }
-
- @Override
- public int getMaxParallelizationWidth() {
- return readEntries.size();
- }
-
- public List<SchemaPath> getColumns() {
- return columns;
- }
-
- @Override
- public OperatorCost getCost() {
- return cost;
- }
-
- @Override
- public Size getSize() {
- return size;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
deleted file mode 100644
index 11b972c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
+++ /dev/null
@@ -1,541 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.json;
-
-import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
-import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
-import static com.fasterxml.jackson.core.JsonToken.FIELD_NAME;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.holders.NullableBitHolder;
-import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
-import org.apache.drill.exec.expr.holders.NullableIntHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.schema.DiffSchema;
-import org.apache.drill.exec.schema.Field;
-import org.apache.drill.exec.schema.NamedField;
-import org.apache.drill.exec.schema.ObjectSchema;
-import org.apache.drill.exec.schema.RecordSchema;
-import org.apache.drill.exec.schema.SchemaIdGenerator;
-import org.apache.drill.exec.schema.json.jackson.JacksonHelper;
-import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.store.VectorHolder;
-import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.NullableBitVector;
-import org.apache.drill.exec.vector.NullableFloat4Vector;
-import org.apache.drill.exec.vector.NullableIntVector;
-import org.apache.drill.exec.vector.NullableVarCharVector;
-import org.apache.drill.exec.vector.RepeatedBitVector;
-import org.apache.drill.exec.vector.RepeatedFloat4Vector;
-import org.apache.drill.exec.vector.RepeatedIntVector;
-import org.apache.drill.exec.vector.RepeatedVarCharVector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class JSONRecordReader implements RecordReader {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader.class);
- private static final int DEFAULT_LENGTH = 256 * 1024; // 256kb
- public static final Charset UTF_8 = Charset.forName("UTF-8");
-
- private final Map<String, VectorHolder> valueVectorMap;
- private final FileSystem fileSystem;
- private final Path hadoopPath;
-
- private JsonParser parser;
- private SchemaIdGenerator generator;
- private DiffSchema diffSchema;
- private RecordSchema currentSchema;
- private List<Field> removedFields;
- private OutputMutator outputMutator;
- private BufferAllocator allocator;
- private int batchSize;
- private final FieldReference ref;
- private final List<SchemaPath> columns;
-
- public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize,
- FieldReference ref, List<SchemaPath> columns) {
- this.hadoopPath = new Path(inputPath);
- this.fileSystem = fileSystem;
- this.allocator = fragmentContext.getAllocator();
- this.batchSize = batchSize;
- valueVectorMap = Maps.newHashMap();
- this.ref = ref;
- this.columns = columns;
- }
-
- public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, FieldReference ref,
- List<SchemaPath> columns) {
- this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, ref, columns);
- }
-
- private JsonParser getParser() {
- return parser;
- }
-
- @Override
- public void setup(OutputMutator output) throws ExecutionSetupException {
- outputMutator = output;
- currentSchema = new ObjectSchema();
- diffSchema = new DiffSchema();
- removedFields = Lists.newArrayList();
-
- try {
- JsonFactory factory = new JsonFactory();
- parser = factory.createJsonParser(fileSystem.open(hadoopPath));
- parser.nextToken(); // Read to the first START_OBJECT token
- generator = new SchemaIdGenerator();
- } catch (IOException e) {
- throw new ExecutionSetupException(e);
- }
- }
-
- @Override
- public int next() {
- if (parser.isClosed() || !parser.hasCurrentToken()) {
- return 0;
- }
-
- resetBatch();
-
- int nextRowIndex = 0;
-
- try {
- while (ReadType.OBJECT.readRecord(this, null, nextRowIndex++, 0)) {
- parser.nextToken(); // Read to START_OBJECT token
-
- if (!parser.hasCurrentToken()) {
- parser.close();
- break;
- }
- }
-
- parser.nextToken();
-
- if (!parser.hasCurrentToken()) {
- parser.close();
- }
-
- // Garbage collect fields never referenced in this batch
- for (Field field : Iterables.concat(currentSchema.removeUnreadFields(), removedFields)) {
- diffSchema.addRemovedField(field);
- outputMutator.removeField(field.getAsMaterializedField(ref));
- }
-
- if (diffSchema.isChanged()) {
- outputMutator.setNewSchema();
- }
-
-
- } catch (IOException | SchemaChangeException e) {
- logger.error("Error reading next in Json reader", e);
- }
-
- for (VectorHolder holder : valueVectorMap.values()) {
- holder.populateVectorLength();
- }
-
- return nextRowIndex;
- }
-
- private void resetBatch() {
- for (VectorHolder value : valueVectorMap.values()) {
- value.reset();
- }
-
- currentSchema.resetMarkedFields();
- diffSchema.reset();
- removedFields.clear();
- }
-
- @Override
- public void cleanup() {
- try {
- parser.close();
- } catch (IOException e) {
- logger.warn("Error closing Json parser", e);
- }
- }
-
-
- private RecordSchema getCurrentSchema() {
- return currentSchema;
- }
-
- private void setCurrentSchema(RecordSchema schema) {
- currentSchema = schema;
- }
-
- private List<Field> getRemovedFields() {
- return removedFields;
- }
-
- public BufferAllocator getAllocator() {
- return allocator;
- }
-
- private boolean fieldSelected(String field){
- SchemaPath sp = new SchemaPath(field, ExpressionPosition.UNKNOWN);
- if (this.columns != null && this.columns.size() > 1){
- for (SchemaPath expr : this.columns){
- if ( sp.equals(expr)){
- return true;
- }
- }
- return false;
- }
- return true;
- }
-
- public static enum ReadType {
- ARRAY(END_ARRAY) {
- @Override
- public Field createField(RecordSchema parentSchema, String prefixFieldName, String fieldName, MajorType fieldType, int index) {
- return new NamedField(parentSchema, prefixFieldName, fieldName, fieldType);
- }
-
- @Override
- public RecordSchema createSchema() throws IOException {
- return new ObjectSchema();
- }
- },
- OBJECT(END_OBJECT) {
- @Override
- public Field createField(RecordSchema parentSchema,
- String prefixFieldName,
- String fieldName,
- MajorType fieldType,
- int index) {
- return new NamedField(parentSchema, prefixFieldName, fieldName, fieldType);
- }
-
- @Override
- public RecordSchema createSchema() throws IOException {
- return new ObjectSchema();
- }
- };
-
- private final JsonToken endObject;
-
- ReadType(JsonToken endObject) {
- this.endObject = endObject;
- }
-
- public JsonToken getEndObject() {
- return endObject;
- }
-
- @SuppressWarnings("ConstantConditions")
- public boolean readRecord(JSONRecordReader reader,
- String prefixFieldName,
- int rowIndex,
- int groupCount) throws IOException, SchemaChangeException {
- JsonParser parser = reader.getParser();
- JsonToken token = parser.nextToken();
- JsonToken endObject = getEndObject();
- int colIndex = 0;
- boolean isFull = false;
- while (token != endObject) {
- if (token == FIELD_NAME) {
- token = parser.nextToken();
- continue;
- }
-
- String fieldName = parser.getCurrentName();
- if ( fieldName != null && ! reader.fieldSelected(fieldName)){
- // this field was not requested in the query
- token = parser.nextToken();
- colIndex += 1;
- continue;
- }
- MajorType fieldType = JacksonHelper.getFieldType(token, this == ReadType.ARRAY);
- ReadType readType = null;
- switch (token) {
- case START_ARRAY:
- readType = ReadType.ARRAY;
- groupCount++;
- break;
- case START_OBJECT:
- readType = ReadType.OBJECT;
- groupCount = 0;
- break;
- }
-
- if (fieldType != null) { // Including nulls
- boolean currentFieldFull = !recordData(
- readType,
- reader,
- fieldType,
- prefixFieldName,
- fieldName,
- rowIndex,
- colIndex,
- groupCount);
-
- isFull = isFull || currentFieldFull;
- }
- token = parser.nextToken();
- colIndex += 1;
- }
- return !isFull;
- }
-
- private void removeChildFields(List<Field> removedFields, Field field) {
- RecordSchema schema = field.getAssignedSchema();
- if (schema == null) {
- return;
- }
- for (Field childField : schema.getFields()) {
- removedFields.add(childField);
- if (childField.hasSchema()) {
- removeChildFields(removedFields, childField);
- }
- }
- }
-
- private boolean recordData(JSONRecordReader.ReadType readType,
- JSONRecordReader reader,
- MajorType fieldType,
- String prefixFieldName,
- String fieldName,
- int rowIndex,
- int colIndex,
- int groupCount) throws IOException, SchemaChangeException {
- RecordSchema currentSchema = reader.getCurrentSchema();
- Field field = currentSchema.getField(fieldName == null ? prefixFieldName : fieldName, colIndex);
- boolean isFieldFound = field != null;
- List<Field> removedFields = reader.getRemovedFields();
- boolean newFieldLateBound = fieldType.getMinorType().equals(MinorType.LATE);
-
- if (isFieldFound && !field.getFieldType().equals(fieldType)) {
- boolean existingFieldLateBound = field.getFieldType().getMinorType().equals(MinorType.LATE);
-
- if (newFieldLateBound && !existingFieldLateBound) {
- fieldType = Types.overrideMinorType(fieldType, field.getFieldType().getMinorType());
- } else if (!newFieldLateBound && existingFieldLateBound) {
- field.setFieldType(Types.overrideMinorType(field.getFieldType(), fieldType.getMinorType()));
- } else if (!newFieldLateBound && !existingFieldLateBound) {
- if (field.hasSchema()) {
- removeChildFields(removedFields, field);
- }
- removedFields.add(field);
- currentSchema.removeField(field, colIndex);
-
- isFieldFound = false;
- }
- }
-
- if (!isFieldFound) {
- field = createField(
- currentSchema,
- prefixFieldName,
- fieldName,
- fieldType,
- colIndex
- );
-
- reader.recordNewField(field);
- currentSchema.addField(field);
- }
-
- field.setRead(true);
-
- VectorHolder holder = getOrCreateVectorHolder(reader, field);
- if (readType != null) {
- RecordSchema fieldSchema = field.getAssignedSchema();
- RecordSchema newSchema = readType.createSchema();
-
- if (readType != ReadType.ARRAY) {
- reader.setCurrentSchema(fieldSchema);
- if (fieldSchema == null) reader.setCurrentSchema(newSchema);
- readType.readRecord(reader, field.getFullFieldName(), rowIndex, groupCount);
- } else {
- readType.readRecord(reader, field.getFullFieldName(), rowIndex, groupCount);
- }
-
- reader.setCurrentSchema(currentSchema);
-
- } else if (holder != null && !newFieldLateBound && fieldType.getMinorType() != MinorType.LATE) {
- return addValueToVector(
- rowIndex,
- holder,
- JacksonHelper.getValueFromFieldType(
- reader.getParser(),
- fieldType.getMinorType()
- ),
- fieldType.getMinorType(),
- groupCount
- );
- }
-
- return true;
- }
-
- private static <T> boolean addValueToVector(int index, VectorHolder holder, T val, MinorType minorType, int groupCount) {
- switch (minorType) {
- case INT: {
- holder.incAndCheckLength(NullableIntHolder.WIDTH * 8 + 1);
- if (groupCount == 0) {
- if (val != null) {
- NullableIntVector int4 = (NullableIntVector) holder.getValueVector();
- NullableIntVector.Mutator m = int4.getMutator();
- m.set(index, (Integer) val);
- }
- } else {
- if (val == null) {
- throw new UnsupportedOperationException("Nullable repeated int is not supported.");
- }
-
- RepeatedIntVector repeatedInt4 = (RepeatedIntVector) holder.getValueVector();
- RepeatedIntVector.Mutator m = repeatedInt4.getMutator();
- holder.setGroupCount(index);
- m.add(index, (Integer) val);
- }
-
- return holder.hasEnoughSpace(NullableIntHolder.WIDTH * 8 + 1);
- }
- case FLOAT4: {
- holder.incAndCheckLength(NullableFloat4Holder.WIDTH * 8 + 1);
- if (groupCount == 0) {
- if (val != null) {
- NullableFloat4Vector float4 = (NullableFloat4Vector) holder.getValueVector();
- NullableFloat4Vector.Mutator m = float4.getMutator();
- m.set(index, (Float) val);
- }
- } else {
- if (val == null) {
- throw new UnsupportedOperationException("Nullable repeated float is not supported.");
- }
-
- RepeatedFloat4Vector repeatedFloat4 = (RepeatedFloat4Vector) holder.getValueVector();
- RepeatedFloat4Vector.Mutator m = repeatedFloat4.getMutator();
- holder.setGroupCount(index);
- m.add(index, (Float) val);
- }
- return holder.hasEnoughSpace(NullableFloat4Holder.WIDTH * 8 + 1);
- }
- case VARCHAR: {
- if (val == null) {
- return (index + 1) * 4 <= holder.getLength();
- } else {
- byte[] bytes = ((String) val).getBytes(UTF_8);
- int length = bytes.length;
- holder.incAndCheckLength(length);
- if (groupCount == 0) {
- NullableVarCharVector varLen4 = (NullableVarCharVector) holder.getValueVector();
- NullableVarCharVector.Mutator m = varLen4.getMutator();
- m.set(index, bytes);
- } else {
- RepeatedVarCharVector repeatedVarLen4 = (RepeatedVarCharVector) holder.getValueVector();
- RepeatedVarCharVector.Mutator m = repeatedVarLen4.getMutator();
- holder.setGroupCount(index);
- m.add(index, bytes);
- }
- return holder.hasEnoughSpace(length + 4 + 1);
- }
- }
- case BIT: {
- holder.incAndCheckLength(NullableBitHolder.WIDTH + 1);
- if (groupCount == 0) {
- if (val != null) {
- NullableBitVector bit = (NullableBitVector) holder.getValueVector();
- NullableBitVector.Mutator m = bit.getMutator();
- m.set(index, (Boolean) val ? 1 : 0);
- }
- } else {
- if (val == null) {
- throw new UnsupportedOperationException("Nullable repeated boolean is not supported.");
- }
-
- RepeatedBitVector repeatedBit = (RepeatedBitVector) holder.getValueVector();
- RepeatedBitVector.Mutator m = repeatedBit.getMutator();
- holder.setGroupCount(index);
- m.add(index, (Boolean) val ? 1 : 0);
- }
- return holder.hasEnoughSpace(NullableBitHolder.WIDTH + 1);
- }
- default:
- throw new DrillRuntimeException("Type not supported to add value. Type: " + minorType);
- }
- }
-
- private VectorHolder getOrCreateVectorHolder(JSONRecordReader reader, Field field) throws SchemaChangeException {
- return reader.getOrCreateVectorHolder(field);
- }
-
- public abstract RecordSchema createSchema() throws IOException;
-
- public abstract Field createField(RecordSchema parentSchema,
- String prefixFieldName,
- String fieldName,
- MajorType fieldType,
- int index);
- }
-
- private void recordNewField(Field field) {
- diffSchema.recordNewField(field);
- }
-
- private VectorHolder getOrCreateVectorHolder(Field field) throws SchemaChangeException {
- String fullFieldName = ref != null ? ref.getPath() + "." + field.getFullFieldName() : field.getFullFieldName();
- VectorHolder holder = valueVectorMap.get(fullFieldName);
-
- if (holder == null) {
- MajorType type = field.getFieldType();
- MinorType minorType = type.getMinorType();
-
- if (minorType.equals(MinorType.MAP) || minorType.equals(MinorType.LATE)) {
- return null;
- }
-
- MaterializedField f = MaterializedField.create(new SchemaPath(fullFieldName, ExpressionPosition.UNKNOWN), type);
-
- ValueVector v = TypeHelper.getNewVector(f, allocator);
- AllocationHelper.allocate(v, batchSize, 50);
- holder = new VectorHolder(v);
- valueVectorMap.put(fullFieldName, holder);
- outputMutator.addField(v);
- return holder;
- }
- return holder;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
deleted file mode 100644
index c40cb47..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.json;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.impl.BatchCreator;
-import org.apache.drill.exec.physical.impl.ScanBatch;
-import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.store.RecordReader;
-
-import java.util.List;
-
-public class JSONScanBatchCreator implements BatchCreator<JSONSubScan> {
-
- @Override
- public RecordBatch getBatch(FragmentContext context, JSONSubScan config, List<RecordBatch> children) throws ExecutionSetupException {
- Preconditions.checkArgument(children.isEmpty());
- List<JSONGroupScan.ScanEntry> entries = config.getReadEntries();
- List<RecordReader> readers = Lists.newArrayList();
- for (JSONGroupScan.ScanEntry e : entries) {
- readers.add(new JSONRecordReader(context, e.getPath(), config.getStorageEngine().getFileSystem(), config.getRef(),
- config.getColumns()));
- }
-
- return new ScanBatch(context, readers.iterator());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
deleted file mode 100644
index e4f2070..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.json;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.AbstractStorageEngine;
-import org.apache.drill.exec.store.ClassPathFileSystem;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.drill.exec.store.json.JSONGroupScan.ScanEntry;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-public class JSONStorageEngine extends AbstractStorageEngine {
- private final JSONStorageEngineConfig config;
- private final Configuration conf;
- private FileSystem fileSystem;
- public static final String HADOOP_DEFAULT_NAME = "fs.default.name";
- private final JsonSchemaProvider schemaProvider;
-
- public JSONStorageEngine(JSONStorageEngineConfig config, DrillbitContext context) {
- this.config = config;
- this.schemaProvider = new JsonSchemaProvider(config, context.getConfig());
-
- try {
- this.conf = new Configuration();
- this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
- this.conf.set(HADOOP_DEFAULT_NAME, config.getDfsName());
- this.fileSystem = FileSystem.get(conf);
-
- } catch (IOException ie) {
- throw new RuntimeException("Error setting up filesystem");
- }
- }
-
- public FileSystem getFileSystem() {
- return fileSystem;
- }
-
- public JSONStorageEngineConfig getConfig(){
- return config;
- }
-
- @Override
- public JSONGroupScan getPhysicalScan(Scan scan) throws IOException {
- ArrayList<ScanEntry> readEntries = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<ArrayList<ScanEntry>>() {});
- return new JSONGroupScan(readEntries, this, scan.getOutputReference(), null);
- }
-
- @Override
- public SchemaProvider getSchemaProvider() {
- return schemaProvider;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngineConfig.java
deleted file mode 100644
index 359e7ed..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngineConfig.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.json;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-
-@JsonTypeName("json")
-public class JSONStorageEngineConfig extends StorageEngineConfigBase {
- private String dfsName;
-
- public String getDfsName() {
- return dfsName;
- }
-
- @JsonCreator
- public JSONStorageEngineConfig(@JsonProperty("dfsName") String dfsName) {
- this.dfsName = dfsName;
- }
-
- @Override
- public int hashCode() {
- return dfsName != null ? dfsName.hashCode() : 0;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- JSONStorageEngineConfig that = (JSONStorageEngineConfig) o;
-
- if (dfsName != null ? !dfsName.equals(that.dfsName) : that.dfsName != null) return false;
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
deleted file mode 100644
index 92f6c0a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.json;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.AbstractBase;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
-import org.apache.drill.exec.physical.base.SubScan;
-import org.apache.drill.exec.store.StorageEngineRegistry;
-
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.collect.Iterators;
-
-@JsonTypeName("json-sub-scan")
-public class JSONSubScan extends AbstractBase implements SubScan {
-
- protected final List<JSONGroupScan.ScanEntry> readEntries;
- private final OperatorCost cost;
- private final Size size;
- private final JSONStorageEngine storageEngine;
- private final FieldReference ref;
- private final List<SchemaPath> columns;
-
- @JsonCreator
- public JSONSubScan(@JacksonInject StorageEngineRegistry registry,
- @JsonProperty("engineConfig") StorageEngineConfig engineConfig,
- @JsonProperty("readEntries") List<JSONGroupScan.ScanEntry> readEntries,
- @JsonProperty("ref") FieldReference ref,
- @JsonProperty("columns") List<SchemaPath> columns) throws ExecutionSetupException {
- this(readEntries, (JSONStorageEngine) registry.getEngine(engineConfig), ref, columns);
- }
-
- JSONSubScan(List<JSONGroupScan.ScanEntry> readEntries, JSONStorageEngine engine, FieldReference ref,
- List<SchemaPath> columns){
- this.readEntries = readEntries;
- this.storageEngine = engine;
- OperatorCost cost = new OperatorCost(0, 0, 0, 0);
- Size size = new Size(0, 0);
- for (JSONGroupScan.ScanEntry r : readEntries) {
- cost = cost.add(r.getCost());
- size = size.add(r.getSize());
- }
- this.cost = cost;
- this.size = size;
- this.ref = ref;
- this.columns = columns;
- }
-
- public FieldReference getRef() {
- return ref;
- }
-
- public List<JSONGroupScan.ScanEntry> getReadEntries() {
- return readEntries;
- }
-
- public StorageEngineConfig getEngineConfig() {
- return storageEngine.getConfig();
- }
-
- @JsonIgnore
- public JSONStorageEngine getStorageEngine() {
- return storageEngine;
- }
-
- @Override
- public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException{
- throw new UnsupportedOperationException();
- }
-
- @Override
- public OperatorCost getCost() {
- return cost;
- }
-
- @Override
- public Size getSize() {
- return size;
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
- return physicalVisitor.visitSubScan(this, value);
- }
-
- @Override
- public Iterator<PhysicalOperator> iterator() {
- return Iterators.emptyIterator();
- }
-
- public List<SchemaPath> getColumns() {
- return columns;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
deleted file mode 100644
index 66a4542..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.json;
-
-import java.io.IOException;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.physical.ReadEntryWithPath;
-import org.apache.drill.exec.store.ClassPathFileSystem;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-
-import com.beust.jcommander.internal.Lists;
-
-public class JsonSchemaProvider implements SchemaProvider{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonSchemaProvider.class);
-
- public static final String HADOOP_DEFAULT_NAME = "fs.default.name";
- final JSONStorageEngineConfig configuration;
- final FileSystem fs;
- final Configuration conf;
-
- public JsonSchemaProvider(JSONStorageEngineConfig configuration, DrillConfig config){
- this.configuration = configuration;
- try {
- this.conf = new Configuration();
- this.conf.set(HADOOP_DEFAULT_NAME, "file:///");
- this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
- this.fs = FileSystem.get(conf);
- } catch (IOException ie) {
- throw new RuntimeException("Error setting up filesystem");
- }
- }
-
- @Override
- public Object getSelectionBaseOnName(String tableName) {
- try{
-// if(!fs.exists(new Path(tableName))) return null;
- ReadEntryWithPath re = new ReadEntryWithPath(tableName);
- return Lists.newArrayList(re);
- }catch(Exception e){
- logger.warn(String.format("Failure while checking table name %s.", tableName), e);
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
index cb9cbf6..b71784b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
@@ -27,7 +27,6 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.ReadEntry;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.Size;
@@ -75,7 +74,7 @@ public class MockGroupScanPOP extends AbstractGroupScan {
return readEntries;
}
- public static class MockScanEntry implements ReadEntry {
+ public static class MockScanEntry{
private final int records;
private final MockColumn[] types;
@@ -93,7 +92,7 @@ public class MockGroupScanPOP extends AbstractGroupScan {
this.recordSize = size;
}
- @Override
+ @JsonIgnore
public OperatorCost getCost() {
return new OperatorCost(1, 2, 1, 1);
}
@@ -106,7 +105,7 @@ public class MockGroupScanPOP extends AbstractGroupScan {
return types;
}
- @Override
+ @JsonIgnore
public Size getSize() {
return new Size(records, recordSize);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java
index b58ab50..0578b06 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java
@@ -20,20 +20,22 @@ package org.apache.drill.exec.store.mock;
import java.io.IOException;
import java.util.ArrayList;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.SchemaPlus;
+
import org.apache.drill.common.logical.data.Scan;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.AbstractStorageEngine;
-import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.exec.store.AbstractStoragePlugin;
import org.apache.drill.exec.store.mock.MockGroupScanPOP.MockScanEntry;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
-public class MockStorageEngine extends AbstractStorageEngine {
+public class MockStorageEngine extends AbstractStoragePlugin {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class);
- public MockStorageEngine(MockStorageEngineConfig configuration, DrillbitContext context) {
+ public MockStorageEngine(MockStorageEngineConfig configuration, DrillbitContext context, String name) {
}
@@ -48,8 +50,9 @@ public class MockStorageEngine extends AbstractStorageEngine {
}
@Override
- public SchemaProvider getSchemaProvider() {
- throw new UnsupportedOperationException();
+ public Schema createAndAddSchema(SchemaPlus parent) {
+ return null;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java
index 786c0ec..8262544 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java
@@ -17,14 +17,14 @@
*/
package org.apache.drill.exec.store.mock;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
+import org.apache.drill.common.logical.StoragePluginConfigBase;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("mock")
-public class MockStorageEngineConfig extends StorageEngineConfigBase{
+public class MockStorageEngineConfig extends StoragePluginConfigBase{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngineConfig.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
index 800c7c5..0753be5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
@@ -41,21 +41,21 @@ public class MockSubScanPOP extends AbstractBase implements SubScan {
private final String url;
protected final List<MockGroupScanPOP.MockScanEntry> readEntries;
- private final OperatorCost cost;
- private final Size size;
+// private final OperatorCost cost;
+// private final Size size;
private LinkedList<MockGroupScanPOP.MockScanEntry>[] mappings;
@JsonCreator
public MockSubScanPOP(@JsonProperty("url") String url, @JsonProperty("entries") List<MockGroupScanPOP.MockScanEntry> readEntries) {
this.readEntries = readEntries;
- OperatorCost cost = new OperatorCost(0,0,0,0);
- Size size = new Size(0,0);
- for(MockGroupScanPOP.MockScanEntry r : readEntries){
- cost = cost.add(r.getCost());
- size = size.add(r.getSize());
- }
- this.cost = cost;
- this.size = size;
+// OperatorCost cost = new OperatorCost(0,0,0,0);
+// Size size = new Size(0,0);
+// for(MockGroupScanPOP.MockScanEntry r : readEntries){
+// cost = cost.add(r.getCost());
+// size = size.add(r.getSize());
+// }
+// this.cost = cost;
+// this.size = size;
this.url = url;
}
@@ -73,12 +73,12 @@ public class MockSubScanPOP extends AbstractBase implements SubScan {
return Iterators.emptyIterator();
}
- @Override
+ @Override @JsonIgnore
public OperatorCost getCost() {
throw new UnsupportedOperationException();
}
- @Override
+ @Override @JsonIgnore
public Size getSize() {
throw new UnsupportedOperationException();
}
@@ -86,7 +86,7 @@ public class MockSubScanPOP extends AbstractBase implements SubScan {
// will want to replace these two methods with an interface above for AbstractSubScan
@Override
public boolean isExecutable() {
- return true; //To change body of implemented methods use File | Settings | File Templates.
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
index d0049c7..c489d5b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java
@@ -23,7 +23,7 @@ import org.apache.drill.exec.vector.ValueVector;
import parquet.column.ColumnDescriptor;
import parquet.hadoop.metadata.ColumnChunkMetaData;
-public final class BitReader extends ColumnReader {
+final class BitReader extends ColumnReader {
private byte currentByte;
private byte nextByte;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
index ec44747..a890f1c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java
@@ -28,7 +28,7 @@ import parquet.bytes.BytesInput;
import parquet.format.PageHeader;
import parquet.format.Util;
-public class ColumnDataReader {
+class ColumnDataReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.class);
private final long endPosition;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
index 94ccbfc..2cc126c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
@@ -29,7 +29,7 @@ import parquet.schema.PrimitiveType;
import java.io.IOException;
-public abstract class ColumnReader {
+abstract class ColumnReader {
final ParquetRecordReader parentReader;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java
index 3aae189..4f14f60 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java
@@ -23,7 +23,7 @@ import org.apache.drill.exec.vector.ValueVector;
import parquet.column.ColumnDescriptor;
import parquet.hadoop.metadata.ColumnChunkMetaData;
-public class FixedByteAlignedReader extends ColumnReader {
+class FixedByteAlignedReader extends ColumnReader {
private byte[] bytes;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
index 1dde7c7..4c060f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
@@ -35,7 +35,7 @@ import java.io.IOException;
* because page/batch boundaries that do not land on byte boundaries require shifting of all of the values
* in the next batch.
*/
-public final class NullableBitReader extends ColumnReader {
+final class NullableBitReader extends ColumnReader {
NullableBitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
boolean fixedLength, ValueVector v) throws ExecutionSetupException {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
index 5ac9bb1..4c33aeb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
@@ -26,7 +26,7 @@ import parquet.hadoop.metadata.ColumnChunkMetaData;
import java.io.IOException;
-public abstract class NullableColumnReader extends ColumnReader{
+abstract class NullableColumnReader extends ColumnReader{
int nullsFound;
// used to skip nulls found
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
index bb81024..c2fc606 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
@@ -23,7 +23,7 @@ import org.apache.drill.exec.vector.ValueVector;
import parquet.column.ColumnDescriptor;
import parquet.hadoop.metadata.ColumnChunkMetaData;
-public class NullableFixedByteAlignedReader extends NullableColumnReader {
+class NullableFixedByteAlignedReader extends NullableColumnReader {
private byte[] bytes;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
index c08dcf3..1aef7b4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
@@ -31,7 +31,7 @@ import parquet.format.PageHeader;
import parquet.hadoop.metadata.ColumnChunkMetaData;
// class to keep track of the read position of variable length columns
-public final class PageReadStatus {
+final class PageReadStatus {
private final ColumnReader parentColumnReader;
private final ColumnDataReader dataReader;
@@ -90,9 +90,9 @@ public final class PageReadStatus {
bytesIn,
pageHeader.data_page_header.num_values,
pageHeader.uncompressed_page_size,
- ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding),
- ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding),
- ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.encoding)
+ ParquetFormatPlugin.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding),
+ ParquetFormatPlugin.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding),
+ ParquetFormatPlugin.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.encoding)
);
byteLength = pageHeader.uncompressed_page_size;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java
new file mode 100644
index 0000000..33eb68a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.common.logical.FormatPluginConfig;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("parquet")
+public class ParquetFormatConfig implements FormatPluginConfig{
+
+ @Override
+ public int hashCode() {
+ return 7;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof ParquetFormatConfig;
+ }
+
+}
[08/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/TestUtils.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/TestUtils.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/TestUtils.java
deleted file mode 100644
index 6ebbb4b..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/TestUtils.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collection;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.ref.RunOutcome.OutcomeType;
-import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
-import org.apache.drill.exec.ref.rse.JSONRecordReader;
-import org.apache.drill.exec.ref.rse.RSERegistry;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-import com.google.common.io.Files;
-
-public class TestUtils {
- public static RecordIterator jsonToRecordIterator(String schemaPath, String j) throws IOException {
- InputStream is = new ByteArrayInputStream(j.getBytes());
- JSONRecordReader reader = new JSONRecordReader(new SchemaPath(schemaPath, ExpressionPosition.UNKNOWN), DrillConfig.create(), is, null);
- return reader.getIterator();
- }
-
- public static int getIteratorCount(RecordIterator out) {
- RecordIterator.NextOutcome next;
- int counter = 0;
- while((next = out.next()) != RecordIterator.NextOutcome.NONE_LEFT) {
- counter++;
- //RecordPointer rp = out.getRecordPointer();
- //System.out.println(rp);
- }
- return counter;
- }
-
- /**
- *
- * @param resourcePath path for json plan file
- * @param recordCount expected record count
- * @throws Exception
- */
- public static void assertProduceCount(String resourcePath, int recordCount) throws Exception {
- DrillConfig config = getConfigWithQueue(0);
- Collection<RunOutcome> outcomes = getOutcome(config, resourcePath);
- assertEquals(recordCount, outcomes.iterator().next().records);
- }
-
- /**
- * Runs a logical query plan and returns output
- * @param config DrillConfig to be ustilized.
- * @param resourcePath Path for JSON logical plan
- * @return A collection of RunOutcomes
- * @throws IOException
- */
- public static Collection<RunOutcome> getOutcome(DrillConfig config, String resourcePath) throws IOException{
- LogicalPlan plan = LogicalPlan.parse(config, Files.toString(FileUtils.getResourceAsFile(resourcePath), Charsets.UTF_8));
- IteratorRegistry ir = new IteratorRegistry();
- ReferenceInterpreter i = new ReferenceInterpreter(plan, ir, new BasicEvaluatorFactory(ir), new RSERegistry(config));
- i.setup();
- return i.run();
- }
-
- private static DrillConfig getConfigWithQueue(int queueNumber){
- DrillConfig config = DrillConfig.create();
- final BlockingQueue<Object> queue = new ArrayBlockingQueue<Object>(100);
- config.setSinkQueues(queueNumber, queue);
- return config;
- }
-
-
-
- public static List<UnbackedRecord> getResultAsUnbackedRecords(String resourcePath) throws Exception{
- DrillConfig config = getConfigWithQueue(0);
- Collection<RunOutcome> outcomes = getOutcome(config, resourcePath);
- if(outcomes.size() != 1) throw new Exception("Only supports logical plans that provide a single outcome.");
- RunOutcome out = outcomes.iterator().next();
- switch(out.outcome){
- case CANCELED:
- case FAILED:
- if(out.exception != null) throw (Exception) out.exception;
- throw new Exception("Failure while running query.");
- }
- Object o;
- Queue<Object> q = config.getQueue(0);
- List<UnbackedRecord> records = Lists.newArrayList();
- while( (o = q.poll()) != null){
- if(o instanceof OutcomeType) continue;
- if( !(o instanceof UnbackedRecord) ) throw new Exception(String.format("This method only works when the QueueRSE is configured to use RECORD encoding. One of the queue objects was of type %s", o.getClass().getCanonicalName()));
- records.add( (UnbackedRecord) o);
- }
-
- return records;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/CollapsingAggregateTest.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/CollapsingAggregateTest.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/CollapsingAggregateTest.java
deleted file mode 100644
index d63c0cc..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/CollapsingAggregateTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.ref.TestUtils;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues.LongScalar;
-import org.junit.Test;
-
-public class CollapsingAggregateTest {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CollapsingAggregateTest.class);
-
-
- @Test
- public void checkNullsHandling() throws Exception{
- List<UnbackedRecord> records = TestUtils.getResultAsUnbackedRecords("/collapse/test1.json");
-
- DataValue[] depts = {DataValue.NULL_VALUE, new LongScalar(31), new LongScalar(33), new LongScalar(34)};
- DataValue[] cnts = {new LongScalar(1), new LongScalar(1), new LongScalar(2), new LongScalar(2)};
- SchemaPath typeCount = new SchemaPath("typeCount", ExpressionPosition.UNKNOWN);
- SchemaPath dept = new SchemaPath("deptId", ExpressionPosition.UNKNOWN);
- for(int i =0; i < depts.length; i++){
- UnbackedRecord r = records.get(i);
- assertEquals(String.format("Invalid dept value for record %d.", i), depts[i], r.getField(dept));
- assertEquals(String.format("Invalid type count value for record %d with deptId %s.", i, depts[i]), cnts[i], r.getField(typeCount));
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/ConstantROPTest.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/ConstantROPTest.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/ConstantROPTest.java
deleted file mode 100644
index be9426f..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/ConstantROPTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import java.util.Collection;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.logical.data.Constant;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.ReferenceInterpreter;
-import org.apache.drill.exec.ref.RunOutcome;
-import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
-import org.apache.drill.exec.ref.rse.RSERegistry;
-import org.apache.drill.exec.ref.values.ScalarValues;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-/**
- * Created with IntelliJ IDEA.
- * User: jaltekruse
- * Date: 6/4/13
- * Time: 4:15 PM
- * To change this template use File | Settings | File Templates.
- */
-public class ConstantROPTest {
-
- @Test
- public void testConstant(){
-
- ObjectMapper map = DrillConfig.create().getMapper();
- Constant con;
- try{
- con = map.readValue( FileUtils.getResourceAsString("/constant.json"), Constant.class);
- if (con == null){
- System.out.println("constant is null");
- }
- System.out.println(con);
-
- ConstantROP rop = new ConstantROP(con);
-
- rop.setupIterators(new IteratorRegistry());
- RecordIterator iter = rop.getIteratorInternal();
- RecordPointer ptr = iter.getRecordPointer();
-
- int i = 1;
- while (iter.next() != RecordIterator.NextOutcome.NONE_LEFT){
- System.out.println(ptr);
- org.junit.Assert.assertEquals("Integer value in record " + i + " is incorrect.",
- ptr.getField(new SchemaPath("c1", ExpressionPosition.UNKNOWN)), new ScalarValues.IntegerScalar(i));
- org.junit.Assert.assertEquals("String value in record " + i + " is incorrect.",
- ptr.getField(new SchemaPath("c2", ExpressionPosition.UNKNOWN)), new ScalarValues.StringScalar("string " + i));
- i++;
- }
- org.junit.Assert.assertEquals("Incorrect number of records returned by 'constant' record iterator.", 3, i - 1);
- } catch (Exception ex){ ex.printStackTrace(); }
- System.out.println("end test");
- }
-
- // not sure if we want to keep this as a test and check the results. Now that the internals of the ConstantROP work
- // it might now be worth running the reference intepreter with every build
- @Test
- @Ignore // this plan needs to be updated.
- public void testRefInterp() throws Exception{
- DrillConfig config = DrillConfig.create();
- final String jsonFile = "/constant2.json";
- LogicalPlan plan = LogicalPlan.parse(config, FileUtils.getResourceAsString(jsonFile));
- org.junit.Assert.assertEquals("Constant operator not read in properly or not recognized as a source operator.",
- plan.getGraph().getLeaves().toString(), "[Constant [memo=null]]");
-
- org.junit.Assert.assertEquals("Edge between constant operator and sink not recognized.",
- plan.getGraph().getRoots().toString(), "[Store [memo=output sink]]");
-
-
- IteratorRegistry ir = new IteratorRegistry();
- ReferenceInterpreter i = new ReferenceInterpreter(plan, ir, new BasicEvaluatorFactory(ir), new RSERegistry(config));
- i.setup();
- Collection<RunOutcome> outcomes = i.run();
-
- for(RunOutcome outcome : outcomes){
- System.out.println("============");
- System.out.println(outcome);
- if(outcome.outcome == RunOutcome.OutcomeType.FAILED && outcome.exception != null){
- outcome.exception.printStackTrace();
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/JoinROPTest.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/JoinROPTest.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/JoinROPTest.java
deleted file mode 100644
index 0fd1241..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/JoinROPTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import org.apache.drill.exec.ref.TestUtils;
-import org.junit.Test;
-
-public class JoinROPTest {
-
- @Test
- public void testInnerJoin() throws Exception {
- TestUtils.assertProduceCount("/join/simple_join.json", 5);
- }
-
- @Test
- public void testOuterJoin() throws Exception {
- TestUtils.assertProduceCount("/join/full_join.json", 7);
- }
-
- @Test
- public void testLeftJoin() throws Exception {
- TestUtils.assertProduceCount("/join/left_join.json", 6);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/LimitROPTest.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/LimitROPTest.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/LimitROPTest.java
deleted file mode 100644
index 73e9821..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/LimitROPTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import static org.junit.Assert.*;
-
-import org.apache.drill.exec.ref.*;
-import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import org.apache.drill.common.logical.data.Limit;
-
-
-public class LimitROPTest {
- final String input = "" +
- "{id: 1, name: \"jim\"}" +
- "{id: 2, name: \"bob\"}" +
- "{id: 3, name: \"larry\"}" +
- "{id: 4, name: \"fred\"}";
-
- private LimitROP buildROP(Limit l) {
- LimitROP limitROP = new LimitROP(l);
- limitROP.setupEvals(new BasicEvaluatorFactory(new IteratorRegistry()));
- return limitROP;
- }
-
- @Test
- public void limitStart() throws IOException {
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", input);
- LimitROP limitROP = buildROP(new Limit(0, 2));
- limitROP.setInput(incoming);
-
- int nRecords = TestUtils.getIteratorCount(limitROP.getOutput());
- assertEquals(2, nRecords);
- }
-
- @Test
- public void limitMiddle() throws IOException {
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", input);
- LimitROP limitROP = buildROP(new Limit(1, 2));
- limitROP.setInput(incoming);
-
- int nRecords = TestUtils.getIteratorCount(limitROP.getOutput());
- assertEquals(1, nRecords);
- }
-
- @Test
- public void limitEnd() throws IOException {
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", input);
- LimitROP limitROP = buildROP(new Limit(3, 10));
- limitROP.setInput(incoming);
-
- int nRecords = TestUtils.getIteratorCount(limitROP.getOutput());
- assertEquals(1, nRecords);
- }
-
- @Test
- public void limitZeroLength() throws IOException {
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", input);
- LimitROP limitROP = buildROP(new Limit(1, 1));
- limitROP.setInput(incoming);
-
- int nRecords = TestUtils.getIteratorCount(limitROP.getOutput());
- assertEquals(0, nRecords);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/OrderROPTest.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/OrderROPTest.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/OrderROPTest.java
deleted file mode 100644
index d4ea473..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/OrderROPTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.ref.TestUtils;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues.LongScalar;
-import org.junit.Test;
-
-public class OrderROPTest {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OrderROPTest.class);
-
-
- @Test
- public void checkNullsFirst() throws Exception{
- List<UnbackedRecord> records = TestUtils.getResultAsUnbackedRecords("/order/nulls-first.json");
-
- DataValue[] depts = {DataValue.NULL_VALUE, new LongScalar(31), new LongScalar(33), new LongScalar(34)};
- SchemaPath dept = new SchemaPath("deptId", ExpressionPosition.UNKNOWN);
- for(int i =0; i < depts.length; i++){
- UnbackedRecord r = records.get(i);
- assertEquals(String.format("Invalid dept value for record %d.", i), depts[i], r.getField(dept));
- }
-
- }
-
- @Test
- public void checkNullsLast() throws Exception{
- List<UnbackedRecord> records = TestUtils.getResultAsUnbackedRecords("/order/nulls-last.json");
-
- DataValue[] depts = {new LongScalar(31), new LongScalar(33), new LongScalar(34), DataValue.NULL_VALUE};
- SchemaPath dept = new SchemaPath("deptId", ExpressionPosition.UNKNOWN);
- for(int i =0; i < depts.length; i++){
- UnbackedRecord r = records.get(i);
- assertEquals(String.format("Invalid dept value for record %d.", i), depts[i], r.getField(dept));
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/UnionROPTest.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/UnionROPTest.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/UnionROPTest.java
deleted file mode 100644
index 3a61078..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/UnionROPTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.ref.TestUtils;
-import org.apache.drill.exec.ref.UnbackedRecord;
-import org.apache.drill.exec.ref.values.DataValue;
-import org.apache.drill.exec.ref.values.ScalarValues.LongScalar;
-import org.junit.Test;
-
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-
-public class UnionROPTest {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnionROPTest.class);
-
-
- @Test
- public void checkDistinct() throws Exception{
- TestUtils.assertProduceCount("/union/distinct.json", 5);
- }
-
- @Test
- public void checkNonDistinct() throws Exception{
- TestUtils.assertProduceCount("/union/nondistinct.json", 10);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/WindowFrameROPTest.java
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/WindowFrameROPTest.java b/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/WindowFrameROPTest.java
deleted file mode 100644
index 735c9cb..0000000
--- a/exec/ref/src/test/java/org/apache/drill/exec/ref/rops/WindowFrameROPTest.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.ref.rops;
-
-import com.google.common.collect.Lists;
-
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.WindowFrame;
-import org.apache.drill.exec.ref.RecordIterator;
-import org.apache.drill.exec.ref.RecordPointer;
-import org.apache.drill.exec.ref.TestUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-
-import static junit.framework.Assert.assertEquals;
-
-public class WindowFrameROPTest {
- final String input = "" +
- "{id: 0}" +
- "{id: 1}" +
- "{id: 2}" +
- "{id: 3}" +
- "{id: 4}";
-
- @Test
- public void windowShouldWorkWithBefore() throws IOException {
- WindowFrameROP rop = new WindowFrameROP(new WindowFrame(null, null, -2L, 0L));
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", input);
- rop.setInput(incoming);
- RecordIterator out = rop.getOutput();
-
- List<WindowObj> windows = Lists.newArrayList(
- new WindowObj(0, 0, 0),
- new WindowObj(1, 0, 0),
- new WindowObj(1, 1, 1),
- new WindowObj(2, 0, 0),
- new WindowObj(2, 1, 1),
- new WindowObj(2, 2, -1),
- new WindowObj(3, 1, 0),
- new WindowObj(3, 2, 1),
- new WindowObj(3, 3, -1),
- new WindowObj(4, 2, 0),
- new WindowObj(4, 3, 1),
- new WindowObj(4, 4, -1)
- );
-
- verifyWindowOrder(windows, out);
- }
-
- @Test
- public void windowShouldWorkWithAfter() throws IOException {
- WindowFrameROP rop = new WindowFrameROP(new WindowFrame(null, null, 0L, 2L));
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", input);
- rop.setInput(incoming);
- RecordIterator out = rop.getOutput();
-
- List<WindowObj> windows = Lists.newArrayList(
- new WindowObj(0, 0, 0),
- new WindowObj(0, 1, 1),
- new WindowObj(0, 2, -1),
- new WindowObj(1, 1, 0),
- new WindowObj(1, 2, 1),
- new WindowObj(1, 3, -1),
- new WindowObj(2, 2, 0),
- new WindowObj(2, 3, 1),
- new WindowObj(2, 4, -1),
- new WindowObj(3, 3, 0),
- new WindowObj(3, 4, 1),
- new WindowObj(4, 4, 0)
- );
-
- verifyWindowOrder(windows, out);
- }
-
- @Test
- public void windowShouldWorkWithBeforeAndAfter() throws IOException {
- WindowFrameROP rop = new WindowFrameROP(new WindowFrame(null, null, -2L, 2L));
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", input);
- rop.setInput(incoming);
- RecordIterator out = rop.getOutput();
-
- List<WindowObj> windows = Lists.newArrayList(
- new WindowObj(0, 0, 0),
- new WindowObj(0, 1, 1),
- new WindowObj(0, 2, -1),
- new WindowObj(1, 0, 0),
- new WindowObj(1, 1, 1),
- new WindowObj(1, 2, -1),
- new WindowObj(1, 3, 2),
- new WindowObj(2, 0, 0),
- new WindowObj(2, 1, 1),
- new WindowObj(2, 2, -1),
- new WindowObj(2, 3, 2),
- new WindowObj(2, 4, -2),
- new WindowObj(3, 1, 0),
- new WindowObj(3, 2, 1),
- new WindowObj(3, 3, -1),
- new WindowObj(3, 4, 2),
- new WindowObj(4, 2, 0),
- new WindowObj(4, 3, 1),
- new WindowObj(4, 4, -1)
- );
-
- verifyWindowOrder(windows, out);
- }
-
- @Test
- public void windowShouldNotCrossWithin() throws IOException {
- String withinInput = "" +
- "{id: 0, v: 0}" +
- "{id: 1, v: 1}" +
- "{id: 2, v: 2}" +
- "{id: 3, v: 3}" +
- "{id: 4, v: 4}";
- WindowFrameROP rop = new WindowFrameROP(new WindowFrame(new FieldReference("test.v", ExpressionPosition.UNKNOWN), null, -2L, 2L));
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", withinInput);
- rop.setInput(incoming);
- RecordIterator out = rop.getOutput();
-
- List<WindowObj> windows = Lists.newArrayList(
- new WindowObj(0, 0, 0),
- new WindowObj(1, 1, 0),
- new WindowObj(2, 2, 0),
- new WindowObj(3, 3, 0),
- new WindowObj(4, 4, 0)
- );
-
- verifyWindowOrder(windows, out);
- }
-
- @Test
- public void windowShouldNotCrossWithins() throws IOException {
- String withinInput = "" +
- "{id: 0, v: 0}" +
- "{id: 1, v: 0}" +
- "{id: 2, v: 1}" +
- "{id: 3, v: 1}" +
- "{id: 4, v: 2}";
- WindowFrameROP rop = new WindowFrameROP(new WindowFrame(new FieldReference("test.v", ExpressionPosition.UNKNOWN), null, -1L, 2L));
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", withinInput);
- rop.setInput(incoming);
- RecordIterator out = rop.getOutput();
-
- List<WindowObj> windows = Lists.newArrayList(
- new WindowObj(0, 0, 0),
- new WindowObj(0, 1, 1),
- new WindowObj(1, 0, 0),
- new WindowObj(1, 1, 1),
- new WindowObj(2, 2, 0),
- new WindowObj(2, 3, 1),
- new WindowObj(3, 2, 0),
- new WindowObj(3, 3, 1),
- new WindowObj(4, 4, 0)
- );
-
- verifyWindowOrder(windows, out);
- }
-
- @Test
- public void windowShouldNotCrossWithinAndRange() throws IOException {
- String withinInput = "" +
- "{id: 0, v: 0}" +
- "{id: 1, v: 0}" +
- "{id: 2, v: 0}" +
- "{id: 3, v: 0}" +
- "{id: 4, v: 1}" +
- "{id: 5, v: 1}" +
- "{id: 6, v: 2}";
- WindowFrameROP rop = new WindowFrameROP(new WindowFrame(new FieldReference("test.v", ExpressionPosition.UNKNOWN), null, -1L, 3L));
- RecordIterator incoming = TestUtils.jsonToRecordIterator("test", withinInput);
- rop.setInput(incoming);
- RecordIterator out = rop.getOutput();
-
- List<WindowObj> windows = Lists.newArrayList(
- new WindowObj(0, 0, 0),
- new WindowObj(0, 1, 1),
- new WindowObj(0, 2, -1),
- new WindowObj(0, 3, 2),
- new WindowObj(1, 0, 0),
- new WindowObj(1, 1, 1),
- new WindowObj(1, 2, -1),
- new WindowObj(1, 3, 2),
- new WindowObj(2, 1, 0),
- new WindowObj(2, 2, 1),
- new WindowObj(2, 3, -1),
- new WindowObj(3, 2, 0),
- new WindowObj(3, 3, 1),
- new WindowObj(4, 4, 0),
- new WindowObj(4, 5, 1),
- new WindowObj(5, 4, 0),
- new WindowObj(5, 5, 1),
- new WindowObj(6, 6, 0)
- );
-
- verifyWindowOrder(windows, out);
- }
-
- private void verifyWindowOrder(List<WindowObj> expectedIds, RecordIterator out) {
- verifyWindowOrder(expectedIds, out, new SchemaPath("ref.segment", ExpressionPosition.UNKNOWN), new SchemaPath("ref.position", ExpressionPosition.UNKNOWN));
- }
-
- private void verifyWindowOrder(List<WindowObj> expectedIds, RecordIterator out, SchemaPath segment, SchemaPath position) {
- RecordIterator.NextOutcome outcome = out.next();
- RecordPointer pointer = out.getRecordPointer();
- int count = 0;
- SchemaPath id = new SchemaPath("test.id", ExpressionPosition.UNKNOWN);
- int expectedSize = expectedIds.size();
- while (outcome != RecordIterator.NextOutcome.NONE_LEFT) {
- count += 1;
- WindowObj windowObj = expectedIds.get(count - 1);
- //System.out.println(windowObj);
- assertEquals("Id mismatch", windowObj.id, pointer.getField(id).getAsNumeric().getAsInt());
- assertEquals("Window id mismatch", windowObj.windowId, pointer.getField(segment).getAsNumeric().getAsInt());
- assertEquals("Position mismatch", windowObj.position, pointer.getField(position).getAsNumeric().getAsInt());
- outcome = out.next();
- }
- assertEquals(expectedSize, count);
- }
-
- private class WindowObj {
- int id;
- int position;
- int windowId;
-
- private WindowObj(int windowId, int id, int position) {
- this.id = id;
- this.position = position;
- this.windowId = windowId;
- }
-
- @Override
- public String toString() {
- return "WindowObj{" +
- "id=" + id +
- ", position=" + position +
- ", windowId=" + windowId +
- '}';
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/collapse/test1.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/collapse/test1.json b/exec/ref/src/test/resources/collapse/test1.json
deleted file mode 100644
index d4b1e24..0000000
--- a/exec/ref/src/test/resources/collapse/test1.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "head" : {
- "type" : "APACHE_DRILL_LOGICAL",
- "version" : 1,
- "generator" : {
- "type" : "manual",
- "info" : "na"
- }
- },
- "storage" : {
- "queue" : {
- "type" : "queue",
- "encoding" : "RECORD"
- },
- "donuts-json" : {
- "type": "classpath"
- }
- },
-
- "query" : [ {
- "op" : "scan",
- "@id" : 1,
- "memo" : "initial_scan",
- "storageengine" : "donuts-json",
- "selection" : {
- "path" : "/employees.json",
- "type" : "JSON"
- },
- "ref" : "_MAP"
- }, {
- "op" : "project",
- "input" : 1,
- "@id" : 2,
- "projections" : [ {
- "ref" : "output.deptId",
- "expr" : "_MAP.deptId"
- } ]
- }, {
- op: "segment",
- "input" : 2,
- "@id" : 3,
- ref: "segment",
- exprs: ["deptId"]
- }, {
- "input" : 3,
- "@id" : 4,
- op: "collapsingaggregate",
- within: "segment",
- carryovers: [ "deptId" ],
- aggregations: [
- { ref: "typeCount", expr: "count(1)" }
- ]
- },
- {
- op: "order",
- "input" : 4,
- "@id" : 5,
- orderings: [
- {order: "asc", expr: "deptId"}
- ]
- },
- {
- "op" : "store",
- "input" : 5,
- "@id" : 6,
- "memo" : "output sink",
- "target" : {
- "number" : 0
- },
- "partition" : null,
- "storageEngine" : "queue"
- } ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/constant.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/constant.json b/exec/ref/src/test/resources/constant.json
deleted file mode 100644
index 9fa5434..0000000
--- a/exec/ref/src/test/resources/constant.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "op" : "constant",
- "content" : [{ "c1" : 1, "c2" : "string 1" }, {"c1" : 2, "c2" : "string 2" }, { "c1" : 3, "c2" : "string 3" }]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/constant2.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/constant2.json b/exec/ref/src/test/resources/constant2.json
deleted file mode 100644
index bad1aa3..0000000
--- a/exec/ref/src/test/resources/constant2.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_LOGICAL",
- version:"1",
- generator:{
- type:"manual",
- info:"na"
- }
- },
- storage:[
- {
- type:"console",
- name:"console"
- },
- {
- type:"fs",
- name:"fs1",
- root:"file:///"
- },
- {
- type:"classpath",
- name:"cp"
- }
- ],
- query:[
- { op:"sequence",
- do:[
- {
- op:"constant",
- content : [{ "c1" : 1, "c2" : "string 1" }, {"c1" : 2, "c2" : "string 2" }, { "c1" : 3, "c2" : "string 3" }]
- },
- {
- op: "store",
- memo: "output sink",
- storageengine: "console",
- target: {pipe: "STD_OUT"}
- }
-
- ]}
-
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/departments.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/departments.json b/exec/ref/src/test/resources/departments.json
deleted file mode 100644
index 3cf0a85..0000000
--- a/exec/ref/src/test/resources/departments.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "deptId": 31,
- "name": "Sales"
-}
-{
- "deptId": 33,
- "name": "Engineering"
-}
-{
- "deptId": 34,
- "name": "Clerical"
-}
-{
- "deptId": 35,
- "name": "Marketing"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/donuts.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/donuts.json b/exec/ref/src/test/resources/donuts.json
deleted file mode 100644
index d16ef46..0000000
--- a/exec/ref/src/test/resources/donuts.json
+++ /dev/null
@@ -1,132 +0,0 @@
-{
- "id": "0001",
- "type": "donut",
- "name": "Cake",
- "ppu": 0.55,
- "sales": 35,
-
-
- "batters":
- {
- "batter":
- [
- { "id": "1001", "type": "Regular" },
- { "id": "1002", "type": "Chocolate" },
- { "id": "1003", "type": "Blueberry" },
- { "id": "1004", "type": "Devil's Food" }
- ]
- },
- "topping":
- [
- { "id": "5001", "type": "None" },
- { "id": "5002", "type": "Glazed" },
- { "id": "5005", "type": "Sugar" },
- { "id": "5007", "type": "Powdered Sugar" },
- { "id": "5006", "type": "Chocolate with Sprinkles" },
- { "id": "5003", "type": "Chocolate" },
- { "id": "5004", "type": "Maple" }
- ]
- }
- {
- "id": "0002",
- "type": "donut",
- "name": "Raised",
- "ppu": 0.69,
- "sales": 145,
- "batters":
- {
- "batter":
- [
- { "id": "1001", "type": "Regular" }
- ]
- },
- "topping":
- [
- { "id": "5001", "type": "None" },
- { "id": "5002", "type": "Glazed" },
- { "id": "5005", "type": "Sugar" },
- { "id": "5003", "type": "Chocolate" },
- { "id": "5004", "type": "Maple" }
- ]
- }
- {
- "id": "0003",
- "type": "donut",
- "name": "Old Fashioned",
- "ppu": 0.55,
- "sales": 300,
-
-
- "batters":
- {
- "batter":
- [
- { "id": "1001", "type": "Regular" },
- { "id": "1002", "type": "Chocolate" }
- ]
- },
- "topping":
- [
- { "id": "5001", "type": "None" },
- { "id": "5002", "type": "Glazed" },
- { "id": "5003", "type": "Chocolate" },
- { "id": "5004", "type": "Maple" }
- ]
- }
- {
- "id": "0004",
- "type": "donut",
- "name": "Filled",
- "ppu": 0.69,
- "sales": 14,
-
-
- "batters":
- {
- "batter":
- [
- { "id": "1001", "type": "Regular" },
- { "id": "1002", "type": "Chocolate" },
- { "id": "1003", "type": "Blueberry" },
- { "id": "1004", "type": "Devil's Food" }
- ]
- },
- "topping":
- [
- { "id": "5001", "type": "None" },
- { "id": "5002", "type": "Glazed" },
- { "id": "5005", "type": "Sugar" },
- { "id": "5007", "type": "Powdered Sugar" },
- { "id": "5006", "type": "Chocolate with Sprinkles" },
- { "id": "5003", "type": "Chocolate" },
- { "id": "5004", "type": "Maple" }
- ],
- "filling":
- [
- { "id": "6001", "type": "None" },
- { "id": "6002", "type": "Raspberry" },
- { "id": "6003", "type": "Lemon" },
- { "id": "6004", "type": "Chocolate" },
- { "id": "6005", "type": "Kreme" }
- ]
- }
- {
- "id": "0005",
- "type": "donut",
- "name": "Apple Fritter",
- "ppu": 1.00,
- "sales": 700,
-
-
- "batters":
- {
- "batter":
- [
- { "id": "1001", "type": "Regular" }
- ]
- },
- "topping":
- [
- { "id": "5002", "type": "Glazed" }
- ]
- }
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/employees.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/employees.json b/exec/ref/src/test/resources/employees.json
deleted file mode 100644
index 83f68bd..0000000
--- a/exec/ref/src/test/resources/employees.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "lastName": "Rafferty",
- "deptId": 31
-}
-{
- "lastName": "Jones",
- "deptId": 33
-}
-{
- "lastName": "Steinberg",
- "deptId": 33
-}
-{
- "lastName": "Robinson",
- "deptId": 34
-}
-{
- "lastName": "Smith",
- "deptId": 34
-}
-{
- "lastName": "John"
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/join/departments.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/join/departments.json b/exec/ref/src/test/resources/join/departments.json
deleted file mode 100644
index 3cf0a85..0000000
--- a/exec/ref/src/test/resources/join/departments.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "deptId": 31,
- "name": "Sales"
-}
-{
- "deptId": 33,
- "name": "Engineering"
-}
-{
- "deptId": 34,
- "name": "Clerical"
-}
-{
- "deptId": 35,
- "name": "Marketing"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/join/employees.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/join/employees.json b/exec/ref/src/test/resources/join/employees.json
deleted file mode 100644
index 562ea4d..0000000
--- a/exec/ref/src/test/resources/join/employees.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "lastName": "Rafferty",
- "deptId": 31
-}
-{
- "lastName": "Jones",
- "deptId": 33
-}
-{
- "lastName": "Steinberg",
- "deptId": 33
-}
-{
- "lastName": "Robinson",
- "deptId": 34
-}
-{
- "lastName": "Smith",
- "deptId": 34
-}
-{
- "deptId": null,
- "lastName": "John"
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/join/full_join.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/join/full_join.json b/exec/ref/src/test/resources/join/full_join.json
deleted file mode 100644
index 59a53f3..0000000
--- a/exec/ref/src/test/resources/join/full_join.json
+++ /dev/null
@@ -1,72 +0,0 @@
-{
- head: {
- type: "APACHE_DRILL_LOGICAL",
- version: "1",
- generator: {
- type: "manual",
- info: "na"
- }
- },
- "storage" : {
- "queue" : {
- "type" : "queue",
- "encoding" : "RECORD"
- },
- "cp" : {
- "type": "classpath"
- },
- "fs1" : {
- "type" : "fs",
- "root" : "file:///"
- },
- "console" : {
- "type" : "console"
- }
- },
-
- "query": [
- {
- @id: 1,
- op: "scan",
- memo: "initial_scan",
- ref: "employees",
- storageengine: "cp",
- selection: {
- path: "/join/employees.json",
- type: "JSON"
- }
- },
- {
- @id: 2,
- op: "scan",
- memo: "second_scan",
- ref: "departments",
- storageengine: "cp",
- selection: {
- path: "/join/departments.json",
- type: "JSON"
- }
- },
- {
- @id: 3,
- op: "join",
- left: 1,
- right: 2,
- type: "outer",
- conditions: [
- {
- relationship: "==",
- left: "employees.deptId",
- right: "departments.deptId"
- }
- ]
- },
- {
- input: 3,
- op: "store",
- memo: "output sink",
- storageengine: "queue",
- target: {number: 0}
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/join/left_join.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/join/left_join.json b/exec/ref/src/test/resources/join/left_join.json
deleted file mode 100644
index 96e6a79..0000000
--- a/exec/ref/src/test/resources/join/left_join.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
- head: {
- type: "APACHE_DRILL_LOGICAL",
- version: "1",
- generator: {
- type: "manual",
- info: "na"
- }
- },
- storage:{
- "queue" : {
- "type" : "queue",
- "encoding" : "RECORD"
- },
- "cp" : {
- "type": "classpath"
- },
- "fs1" : {
- "type" : "fs",
- "root" : "file:///"
- },
- "console" : {
- "type" : "console"
- }
- },
- query: [
- {
- @id: 1,
- op: "scan",
- memo: "initial_scan",
- ref: "employees",
- storageengine: "cp",
- selection: {
- path: "/join/employees.json",
- type: "JSON"
- }
- },
- {
- @id: 2,
- op: "scan",
- memo: "second_scan",
- ref: "departments",
- storageengine: "cp",
- selection: {
- path: "/join/departments.json",
- type: "JSON"
- }
- },
- {
- @id: 3,
- op: "join",
- left: 1,
- right: 2,
- type: "left",
- conditions: [
- {
- relationship: "==",
- left: "employees.deptId",
- right: "departments.deptId"
- }
- ]
- },
- {
- input: 3,
- op: "store",
- memo: "output sink",
- storageengine: "queue",
- target: {number: 0}
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/join/simple_join.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/join/simple_join.json b/exec/ref/src/test/resources/join/simple_join.json
deleted file mode 100644
index 1587a78..0000000
--- a/exec/ref/src/test/resources/join/simple_join.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
- head: {
- type: "APACHE_DRILL_LOGICAL",
- version: "1",
- generator: {
- type: "manual",
- info: "na"
- }
- },
- storage:{
- "queue" : {
- "type" : "queue",
- "encoding" : "RECORD"
- },
- "cp" : {
- "type": "classpath"
- },
- "fs1" : {
- "type" : "fs",
- "root" : "file:///"
- },
- "console" : {
- "type" : "console"
- }
- },
- query: [
- {
- @id: 1,
- op: "scan",
- memo: "initial_scan",
- ref: "employees",
- storageengine: "cp",
- selection: {
- path: "/join/employees.json",
- type: "JSON"
- }
- },
- {
- @id: 2,
- op: "scan",
- memo: "second_scan",
- ref: "departments",
- storageengine: "cp",
- selection: {
- path: "/join/departments.json",
- type: "JSON"
- }
- },
- {
- @id: 3,
- op: "join",
- left: 1,
- right: 2,
- type: "inner",
- conditions: [
- {
- relationship: "==",
- left: "employees.deptId",
- right: "departments.deptId"
- }
- ]
- },
- {
- input: 3,
- op: "store",
- memo: "output sink",
- storageengine: "queue",
- target: {number: 0}
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/logback.xml b/exec/ref/src/test/resources/logback.xml
deleted file mode 100644
index 39bb0c8..0000000
--- a/exec/ref/src/test/resources/logback.xml
+++ /dev/null
@@ -1,56 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
- license agreements. See the NOTICE file distributed with this work for additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
-<configuration>
-
- <appender name="SOCKET" class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
- <Compressing>true</Compressing>
- <ReconnectionDelay>10000</ReconnectionDelay>
- <IncludeCallerData>true</IncludeCallerData>
- <RemoteHosts>localhost</RemoteHosts>
- </appender>
-
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <!-- encoders are assigned the type
- ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>/logs/test-ref.log</file>
- <encoder>
- <pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n</pattern>
- </encoder>
- <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
- <!-- daily rollover -->
- <fileNamePattern>/logs/test-ref.%d{yyyy-MM-dd}.log</fileNamePattern>
- <!-- keep 30 days' worth of history -->
- <maxHistory>30</maxHistory>
- </rollingPolicy>
- </appender>
-
- <logger name="org.apache.drill" additivity="false">
- <level value="debug" />
- <appender-ref ref="SOCKET" />
- <appender-ref ref="STDOUT" />
- <appender-ref ref="FILE" />
- </logger>
-
- <root>
- <level value="error" />
- <appender-ref ref="SOCKET" />
- <appender-ref ref="STDOUT" />
- <appender-ref ref="FILE" />
- </root>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/order/nulls-first.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/order/nulls-first.json b/exec/ref/src/test/resources/order/nulls-first.json
deleted file mode 100644
index 6e541e9..0000000
--- a/exec/ref/src/test/resources/order/nulls-first.json
+++ /dev/null
@@ -1,79 +0,0 @@
-{
- "head" : {
- "type" : "APACHE_DRILL_LOGICAL",
- "version" : 1,
- "generator" : {
- "type" : "manual",
- "info" : "na"
- }
- },
- "storage" : {
- "queue" : {
- "type" : "queue",
- "encoding" : "RECORD"
- },
- "cp" : {
- "type": "classpath"
- },
- "fs1" : {
- "type" : "fs",
- "root" : "file:///"
- },
- "console" : {
- "type" : "console"
- }
- },
- "query" : [ {
- "op" : "scan",
- "@id" : 1,
- "memo" : "initial_scan",
- "storageengine" : "cp",
- "selection" : {
- "path" : "/employees.json",
- "type" : "JSON"
- },
- "ref" : "_MAP"
- }, {
- "op" : "project",
- "input" : 1,
- "@id" : 2,
- "projections" : [ {
- "ref" : "output.deptId",
- "expr" : "_MAP.deptId"
- } ]
- }, {
- op: "segment",
- "input" : 2,
- "@id" : 3,
- ref: "segment",
- exprs: ["deptId"]
- }, {
- "input" : 3,
- "@id" : 4,
- op: "collapsingaggregate",
- within: "segment",
- carryovers: [ "deptId" ],
- aggregations: [
- { ref: "typeCount", expr: "count(1)" }
- ]
- },
- {
- op: "order",
- "input" : 4,
- "@id" : 5,
- orderings: [
- {order: "asc", expr: "deptId", nullCollation: "first"}
- ]
- },
- {
- "op" : "store",
- "input" : 5,
- "@id" : 6,
- "memo" : "output sink",
- "target" : {
- "number" : 0
- },
- "partition" : null,
- "storageEngine" : "queue"
- } ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/order/nulls-last.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/order/nulls-last.json b/exec/ref/src/test/resources/order/nulls-last.json
deleted file mode 100644
index 83338b7..0000000
--- a/exec/ref/src/test/resources/order/nulls-last.json
+++ /dev/null
@@ -1,79 +0,0 @@
-{
- "head" : {
- "type" : "APACHE_DRILL_LOGICAL",
- "version" : 1,
- "generator" : {
- "type" : "manual",
- "info" : "na"
- }
- },
- "storage" : {
- "queue" : {
- "type" : "queue",
- "encoding" : "RECORD"
- },
- "cp" : {
- "type": "classpath"
- },
- "fs1" : {
- "type" : "fs",
- "root" : "file:///"
- },
- "console" : {
- "type" : "console"
- }
- },
- "query" : [ {
- "op" : "scan",
- "@id" : 1,
- "memo" : "initial_scan",
- "storageengine" : "cp",
- "selection" : {
- "path" : "/employees.json",
- "type" : "JSON"
- },
- "ref" : "_MAP"
- }, {
- "op" : "project",
- "input" : 1,
- "@id" : 2,
- "projections" : [ {
- "ref" : "output.deptId",
- "expr" : "_MAP.deptId"
- } ]
- }, {
- op: "segment",
- "input" : 2,
- "@id" : 3,
- ref: "segment",
- exprs: ["deptId"]
- }, {
- "input" : 3,
- "@id" : 4,
- op: "collapsingaggregate",
- within: "segment",
- carryovers: [ "deptId" ],
- aggregations: [
- { ref: "typeCount", expr: "count(1)" }
- ]
- },
- {
- op: "order",
- "input" : 4,
- "@id" : 5,
- orderings: [
- {order: "asc", expr: "deptId", nullCollation: "last"}
- ]
- },
- {
- "op" : "store",
- "input" : 5,
- "@id" : 6,
- "memo" : "output sink",
- "target" : {
- "number" : 0
- },
- "partition" : null,
- "storageEngine" : "queue"
- } ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/simple_join.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/simple_join.json b/exec/ref/src/test/resources/simple_join.json
deleted file mode 100644
index ba078d5..0000000
--- a/exec/ref/src/test/resources/simple_join.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
- head: {
- type: "APACHE_DRILL_LOGICAL",
- version: "1",
- generator: {
- type: "manual",
- info: "na"
- }
- },
- storage: {
- console: {type:"console"},
- fs1: {type:"fs", root:"file:///"},
- cp: {type:"classpath"}
- },
- query: [
- {
- @id: 1,
- op: "scan",
- memo: "initial_scan",
- ref: "employees",
- storageengine: "cp",
- selection: {
- path: "/employees.json",
- type: "JSON"
- }
- },
- {
- @id: 2,
- op: "scan",
- memo: "second_scan",
- ref: "departments",
- storageengine: "cp",
- selection: {
- path: "/departments.json",
- type: "JSON"
- }
- },
- {
- @id: 3,
- op: "join",
- left: 1,
- right: 2,
- type: "outer",
- conditions: [
- {
- relationship: "==",
- left: "employees.deptId",
- right: "departments.deptId"
- }
- ]
- },
- {
- input: 3,
- op: "store",
- memo: "output sink",
- storageengine: "console",
- target: {pipe: "STD_OUT"}
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/simple_plan.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/simple_plan.json b/exec/ref/src/test/resources/simple_plan.json
deleted file mode 100644
index ad3cdc7..0000000
--- a/exec/ref/src/test/resources/simple_plan.json
+++ /dev/null
@@ -1,94 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_LOGICAL",
- version:"1",
- generator:{
- type:"manual",
- info:"na"
- }
- },
- storage:{
- console: {type: "console"},
- fs1: {type: "fs", root:"file:///"},
- cp: {type: "classpath"}
- },
- query:[
-
- {
- @id:"1",
- op: "scan",
- memo: "initial_scan",
- ref: "donuts",
- storageengine: "cp",
- selection: {
- path: "/donuts.json",
- type: "JSON"
- }
- },
- {
- @id:"2",
- input:"1",
- op: "transform",
- transforms: [
- { ref: "quantity", expr: "donuts.sales"}
- ]
- },
- {
- @id:"3",
- input:"2",
- op: "filter",
- expr: "donuts.ppu < 1.00"
- },
- {
- @id:"4",
- input:"3",
- op: "segment",
- ref: "ppusegment",
- exprs: ["donuts.ppu"]
- },
- {
- @id:"5",
- input:"4",
- op: "collapsingaggregate",
- within: "ppusegment",
- carryovers: ["donuts.ppu"],
- aggregations: [
- { ref: "donuts.typeCount", expr: "count(1)" },
- { ref: "donuts.quantity", expr: "sum(quantity)" },
- { ref: "donuts.sales", expr: "sum(donuts.ppu * quantity)" }
- ]
- },
- {
- @id:"6",
- input:"5",
- op: "order",
- orderings: [
- {order: "DESC", expr: "donuts.ppu" }
- ]
- },
- {
- @id:"7",
- input:"6",
- op: "project",
- projections: [
- { ref: "output.output", expr: "donuts" }
- ]
- },
- {
- @id:"8",
- input:"7",
- op: "limit",
- first: 0,
- last: 100
- },
- {
- @id:"9",
- input:"8",
- op: "store",
- memo: "output sink",
- storageengine: "console",
- target: {pipe: "STD_OUT"}
- }
- ]
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/simple_plan_flattened.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/simple_plan_flattened.json b/exec/ref/src/test/resources/simple_plan_flattened.json
deleted file mode 100644
index 1601b4a..0000000
--- a/exec/ref/src/test/resources/simple_plan_flattened.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_LOGICAL",
- version:"1",
- generator:{
- type:"manual",
- info:"na"
- }
- },
- storage:{
- console: {type:"console"},
- fs1: {type:"fs", root:"file:///"},
- cp: {type:"classpath"}
- },
- query:[
- {
- op:"sequence",
- do:[
- {
- op: "scan",
- memo: "initial_scan",
- ref: "donuts",
- storageengine: "cp",
- selection: {
- path: "/donuts.json",
- type: "JSON"
- }
- },
- {
- op: "transform",
- transforms: [
- { ref: "donuts.quantity", expr: "donuts.sales"}
- ]
- },
- {
- op: "filter",
- expr: "donuts.ppu < 1.00"
- },
- {
- op: "flatten",
- expr: "donuts.topping",
- name: "donuts.t1",
- drop: 1
- },
- {
- op: "store",
- memo: "output sink",
- storageengine: "console",
- target: {pipe: "STD_OUT"}
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/union/distinct.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/union/distinct.json b/exec/ref/src/test/resources/union/distinct.json
deleted file mode 100644
index 9361009..0000000
--- a/exec/ref/src/test/resources/union/distinct.json
+++ /dev/null
@@ -1,82 +0,0 @@
-{
- "head": {
- "type": "APACHE_DRILL_LOGICAL",
- "version": 1,
- "generator": {
- "type": "manual",
- "info": "na"
- }
- },
- "storage": {
- "queue": {
- "type": "queue",
- "encoding": "RECORD"
- },
- "donuts-json": {
- "type": "classpath"
- }
- },
- "query": [
- {
- "op": "scan",
- "@id": 1,
- "memo": "initial_scan",
- "storageengine": "donuts-json",
- "selection": {
- "path": "/employees.json",
- "type": "JSON"
- },
- "ref": "_MAP"
- },
- {
- "op": "project",
- "input": 1,
- "@id": 2,
- "projections": [
- {
- "ref": "output.deptId",
- "expr": "_MAP.deptId"
- }
- ]
- },
- {
- "op": "scan",
- "@id": 3,
- "memo": "initial_scan",
- "storageengine": "donuts-json",
- "selection": {
- "path": "/departments.json",
- "type": "JSON"
- },
- "ref": "_MAP"
- },
- {
- "op": "project",
- "input": 3,
- "@id": 4,
- "projections": [
- {
- "ref": "output.deptId",
- "expr": "_MAP.deptId"
- }
- ]
- },
- {
- "op": "union",
- "@id": 5,
- "distinct": true,
- "inputs": [2, 4]
- },
- {
- "op": "store",
- "input": 5,
- "@id": 6,
- "memo": "output sink",
- "target": {
- "number": 0
- },
- "partition": null,
- "storageEngine": "queue"
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/exec/ref/src/test/resources/union/nondistinct.json
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/union/nondistinct.json b/exec/ref/src/test/resources/union/nondistinct.json
deleted file mode 100644
index 35ad70e..0000000
--- a/exec/ref/src/test/resources/union/nondistinct.json
+++ /dev/null
@@ -1,82 +0,0 @@
-{
- "head": {
- "type": "APACHE_DRILL_LOGICAL",
- "version": 1,
- "generator": {
- "type": "manual",
- "info": "na"
- }
- },
- "storage": {
- "queue": {
- "type": "queue",
- "encoding": "RECORD"
- },
- "donuts-json": {
- "type": "classpath"
- }
- },
- "query": [
- {
- "op": "scan",
- "@id": 1,
- "memo": "initial_scan",
- "storageengine": "donuts-json",
- "selection": {
- "path": "/employees.json",
- "type": "JSON"
- },
- "ref": "_MAP"
- },
- {
- "op": "project",
- "input": 1,
- "@id": 2,
- "projections": [
- {
- "ref": "output.deptId",
- "expr": "_MAP.deptId"
- }
- ]
- },
- {
- "op": "scan",
- "@id": 3,
- "memo": "initial_scan",
- "storageengine": "donuts-json",
- "selection": {
- "path": "/departments.json",
- "type": "JSON"
- },
- "ref": "_MAP"
- },
- {
- "op": "project",
- "input": 3,
- "@id": 4,
- "projections": [
- {
- "ref": "output.deptId",
- "expr": "_MAP.deptId"
- }
- ]
- },
- {
- "op": "union",
- "@id": 5,
- "distinct": false,
- "inputs": [2, 4]
- },
- {
- "op": "store",
- "input": 5,
- "@id": 6,
- "memo": "output sink",
- "target": {
- "number": 0
- },
- "partition": null,
- "storageEngine": "queue"
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f8ddbf2..235d737 100644
--- a/pom.xml
+++ b/pom.xml
@@ -9,7 +9,8 @@
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -314,7 +315,7 @@
</goals>
</pluginExecutionFilter>
<action>
- <ignore/>
+ <ignore />
</action>
</pluginExecution>
</pluginExecutions>
@@ -602,10 +603,27 @@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <version>2.11.0</version>
+ </dependency>
+ <dependency>
+ <groupId>xalan</groupId>
+ <artifactId>xalan</artifactId>
+ <version>2.7.1</version>
+ </dependency>
+ <dependency>
+ <groupId>sqlline</groupId>
+ <artifactId>sqlline</artifactId>
+ <version>1.1.6</version>
+ </dependency>
+
<dependency>
<groupId>net.hydromatic</groupId>
- <artifactId>optiq</artifactId>
- <version>0.4.11</version>
+ <artifactId>optiq-core</artifactId>
+ <version>0.4.18-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.jgrapht</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/protocol/src/main/protobuf/User.proto
----------------------------------------------------------------------
diff --git a/protocol/src/main/protobuf/User.proto b/protocol/src/main/protobuf/User.proto
index 0c1e588..fb4e093 100644
--- a/protocol/src/main/protobuf/User.proto
+++ b/protocol/src/main/protobuf/User.proto
@@ -20,9 +20,15 @@ enum RpcType {
CANCEL_QUERY = 4;
REQUEST_RESULTS = 5;
+
// bit to user
QUERY_RESULT = 6;
QUERY_HANDLE = 7;
+
+ REQ_META_FUNCTIONS = 8;
+ RESP_FUNCTION_LIST = 9;
+
+
}
message UserToBitHandshake {
@@ -42,6 +48,8 @@ message RunQuery {
optional string plan = 3;
}
+message
+
enum QueryType {
SQL = 1;
LOGICAL = 2;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/pom.xml
----------------------------------------------------------------------
diff --git a/sqlparser/pom.xml b/sqlparser/pom.xml
index c7ceb96..c1b114f 100644
--- a/sqlparser/pom.xml
+++ b/sqlparser/pom.xml
@@ -9,7 +9,8 @@
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>drill-root</artifactId>
@@ -22,7 +23,7 @@
<dependencies>
<dependency>
<groupId>net.hydromatic</groupId>
- <artifactId>optiq</artifactId>
+ <artifactId>optiq-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.drill</groupId>
@@ -42,14 +43,19 @@
</dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
- <artifactId>drill-ref</artifactId>
+ <artifactId>drill-java-exec</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.drill.exec</groupId>
- <artifactId>drill-java-exec</artifactId>
- <version>${project.version}</version>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>xalan</groupId>
+ <artifactId>xalan</artifactId>
</dependency>
+
+
<dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
@@ -68,13 +74,6 @@
<artifactId>mondrian-data-foodmart-json</artifactId>
<version>0.2</version>
</dependency>
- <dependency>
- <groupId>org.apache.drill.exec</groupId>
- <artifactId>drill-ref</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
@@ -84,7 +83,6 @@
<dependency>
<groupId>sqlline</groupId>
<artifactId>sqlline</artifactId>
- <version>1.1.0</version>
<scope>test</scope>
</dependency>
<!-- Specify xalan and xerces versions to avoid setXIncludeAware error. -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/AvaticaDrillSqlAccessor.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/AvaticaDrillSqlAccessor.java b/sqlparser/src/main/java/org/apache/drill/jdbc/AvaticaDrillSqlAccessor.java
new file mode 100644
index 0000000..48fb6fd
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/AvaticaDrillSqlAccessor.java
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.net.URL;
+import java.sql.Array;
+import java.sql.Blob;
+import java.sql.Clob;
+import java.sql.Date;
+import java.sql.NClob;
+import java.sql.Ref;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.sql.SQLXML;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Calendar;
+import java.util.Map;
+
+import net.hydromatic.avatica.Cursor.Accessor;
+
+import org.apache.drill.exec.vector.accessor.SqlAccessor;
+
+
+public class AvaticaDrillSqlAccessor implements Accessor{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AvaticaDrillSqlAccessor.class);
+
+ private SqlAccessor a;
+ private DrillCursor cursor;
+
+ public AvaticaDrillSqlAccessor(SqlAccessor drillSqlAccessor, DrillCursor cursor) {
+ super();
+ this.a = drillSqlAccessor;
+ this.cursor = cursor;
+ }
+
+ private int row(){
+ return cursor.currentRecord;
+ }
+
+ @Override
+ public boolean wasNull() {
+ return a.isNull(row());
+ }
+
+ @Override
+ public String getString() throws SQLException {
+ return a.getString(row());
+ }
+
+ @Override
+ public boolean getBoolean() throws SQLException {
+ return a.getBoolean(row());
+ }
+
+ @Override
+ public byte getByte() throws SQLException {
+ return a.getByte(row());
+ }
+
+ @Override
+ public short getShort() throws SQLException {
+ return a.getShort(row());
+ }
+
+ @Override
+ public int getInt() throws SQLException {
+ return a.getInt(row());
+ }
+
+ @Override
+ public long getLong() throws SQLException {
+ return a.getLong(row());
+ }
+
+ @Override
+ public float getFloat() throws SQLException {
+ return a.getFloat(row());
+ }
+
+ @Override
+ public double getDouble() throws SQLException {
+ return a.getDouble(row());
+ }
+
+ @Override
+ public BigDecimal getBigDecimal() throws SQLException {
+ return a.getBigDecimal(row());
+ }
+
+ @Override
+ public BigDecimal getBigDecimal(int scale) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public byte[] getBytes() throws SQLException {
+ return a.getBytes(row());
+ }
+
+ @Override
+ public InputStream getAsciiStream() throws SQLException {
+ return a.getStream(row());
+ }
+
+ @Override
+ public InputStream getUnicodeStream() throws SQLException {
+ return a.getStream(row());
+ }
+
+ @Override
+ public InputStream getBinaryStream() throws SQLException {
+ return a.getStream(row());
+ }
+
+ @Override
+ public Object getObject() throws SQLException {
+ return a.getObject(row());
+ }
+
+ @Override
+ public Reader getCharacterStream() throws SQLException {
+ return a.getReader(row());
+ }
+
+ @Override
+ public Object getObject(Map<String, Class<?>> map) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Ref getRef() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Blob getBlob() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Clob getClob() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Array getArray() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Date getDate(Calendar calendar) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Time getTime(Calendar calendar) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Timestamp getTimestamp(Calendar calendar) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public URL getURL() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public NClob getNClob() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public SQLXML getSQLXML() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public String getNString() throws SQLException {
+ return a.getString(row());
+ }
+
+ @Override
+ public Reader getNCharacterStream() throws SQLException {
+ return a.getReader(row());
+ }
+
+ @Override
+ public <T> T getObject(Class<T> type) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/BasicList.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/BasicList.java b/sqlparser/src/main/java/org/apache/drill/jdbc/BasicList.java
new file mode 100644
index 0000000..74fa72b
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/BasicList.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * Helper class to make simple changing lists for Avatica. Want to make sure Avatica doesn't try any nonsense other than getting with the list.
+ */
+abstract class BasicList<T> implements List<T>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicList.class);
+
+
+ @Override
+ public boolean isEmpty() {
+ return size() == 0;
+ }
+
+ @Override
+ public boolean contains(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object[] toArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> T[] toArray(T[] a) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean add(T e) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean containsAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends T> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean addAll(int index, Collection<? extends T> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+
+ public void clear(){
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public T set(int index, T element) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void add(int index, T element) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public T remove(int index) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int indexOf(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int lastIndexOf(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ListIterator<T> listIterator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ListIterator<T> listIterator(int index) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<T> subList(int fromIndex, int toIndex) {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/ConnectionConfig.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/ConnectionConfig.java b/sqlparser/src/main/java/org/apache/drill/jdbc/ConnectionConfig.java
new file mode 100644
index 0000000..5a81035
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/ConnectionConfig.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.util.Properties;
+import java.util.TimeZone;
+
+public class ConnectionConfig {
+ private final Properties props;
+
+ public ConnectionConfig(Properties p){
+ this.props = p;
+ }
+
+ public boolean isLocal(){
+ return "local".equals(props.getProperty("zk"));
+ }
+ public String getZookeeperConnectionString(){
+ return props.getProperty("zk");
+ }
+
+ public TimeZone getTimeZone(){
+ return TimeZone.getDefault();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/jdbc/DrillAccessorList.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillAccessorList.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillAccessorList.java
new file mode 100644
index 0000000..6394cb2
--- /dev/null
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillAccessorList.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc;
+
+import java.sql.SQLException;
+
+import net.hydromatic.avatica.Cursor.Accessor;
+
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.vector.accessor.SqlAccessor;
+
+public class DrillAccessorList extends BasicList<Accessor>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillAccessorList.class);
+
+ private Accessor[] accessors = new Accessor[0];
+ private int lastColumn = 1;
+
+ public void generateAccessors(DrillCursor cursor, RecordBatchLoader currentBatch){
+ int cnt = currentBatch.getSchema().getFieldCount();
+ accessors = new Accessor[cnt];
+ for(int i =0; i < cnt; i++){
+ SqlAccessor acc = TypeHelper.getSqlAccessor(currentBatch.getValueAccessorById(i, null).getValueVector());
+ accessors[i] = new AvaticaDrillSqlAccessor(acc, cursor);
+ }
+ }
+
+ @Override
+ public Accessor get(int index) {
+ lastColumn = index;
+ return accessors[index];
+ }
+
+ public boolean wasNull() throws SQLException{
+ return accessors[lastColumn].wasNull();
+ }
+
+ @Override
+ public int size() {
+ return accessors.length;
+ }
+
+}
[06/38] DRILL-257: Move SQL parsing to server side. Switch to Avatica
based JDBC driver. Update QuerySubmitter to support SQL queries. Update
SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes. Updating
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillAggregateRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillAggregateRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillAggregateRel.java
deleted file mode 100644
index 668ed33..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillAggregateRel.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import java.util.BitSet;
-import java.util.List;
-
-import net.hydromatic.linq4j.Ord;
-
-import org.eigenbase.rel.AggregateCall;
-import org.eigenbase.rel.AggregateRelBase;
-import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.util.Util;
-
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * Aggregation implemented in Drill.
- */
-public class DrillAggregateRel extends AggregateRelBase implements DrillRel {
- /** Creates a DrillAggregateRel. */
- public DrillAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, BitSet groupSet,
- List<AggregateCall> aggCalls) throws InvalidRelException {
- super(cluster, traits, child, groupSet, aggCalls);
- for (AggregateCall aggCall : aggCalls) {
- if (aggCall.isDistinct()) {
- throw new InvalidRelException("DrillAggregateRel does not support DISTINCT aggregates");
- }
- }
- }
-
- @Override
- public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- try {
- return new DrillAggregateRel(getCluster(), traitSet, sole(inputs), getGroupSet(), aggCalls);
- } catch (InvalidRelException e) {
- throw new AssertionError(e);
- }
- }
-
- @Override
- public int implement(DrillImplementor implementor) {
- int inputId = implementor.visitChild(this, 0, getChild());
- final List<String> childFields = getChild().getRowType().getFieldNames();
- final List<String> fields = getRowType().getFieldNames();
- /*
- * E.g. { op: "segment", ref: "segment", exprs: ["deptId"] }, { op: "collapsingaggregate", within: "segment",
- * carryovers: ["deptId"], aggregations: [ {ref: "c", expr: "count(1)"} ] }
- */
- final ObjectNode segment = implementor.mapper.createObjectNode();
- segment.put("op", "segment");
- segment.put("input", inputId);
- // TODO: choose different name for field if there is already a field
- // called "segment"
- segment.put("ref", "segment");
- final ArrayNode exprs = implementor.mapper.createArrayNode();
- segment.put("exprs", exprs);
- for (int group : Util.toIter(groupSet)) {
- exprs.add(childFields.get(group));
- }
-
- final int segmentId = implementor.add(segment);
-
- final ObjectNode aggregate = implementor.mapper.createObjectNode();
- aggregate.put("op", "collapsingaggregate");
- aggregate.put("input", segmentId);
- aggregate.put("within", "segment");
- final ArrayNode carryovers = implementor.mapper.createArrayNode();
- aggregate.put("carryovers", carryovers);
- for (int group : Util.toIter(groupSet)) {
- carryovers.add(childFields.get(group));
- }
- final ArrayNode aggregations = implementor.mapper.createArrayNode();
- aggregate.put("aggregations", aggregations);
- for (Ord<AggregateCall> aggCall : Ord.zip(aggCalls)) {
- final ObjectNode aggregation = implementor.mapper.createObjectNode();
- aggregation.put("ref", fields.get(groupSet.cardinality() + aggCall.i));
- aggregation.put("expr", toDrill(aggCall.e, childFields));
- aggregations.add(aggregation);
- }
-
- return implementor.add(aggregate);
- }
-
- private String toDrill(AggregateCall call, List<String> fn) {
- final StringBuilder buf = new StringBuilder();
- buf.append(call.getAggregation().getName().toLowerCase()).append("(");
- for (Ord<Integer> arg : Ord.zip(call.getArgList())) {
- if (arg.i > 0) {
- buf.append(", ");
- }
- buf.append(fn.get(arg.e));
- }
- if (call.getArgList().isEmpty()) {
- buf.append("1"); // dummy arg to implement COUNT(*)
- }
- buf.append(")");
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillAggregateRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillAggregateRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillAggregateRule.java
deleted file mode 100644
index 003ff61..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillAggregateRule.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.AggregateRel;
-import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
-import org.eigenbase.trace.EigenbaseTrace;
-
-import java.util.logging.Logger;
-
-/**
- * Rule that converts an {@link AggregateRel} to a {@link DrillAggregateRel}, implemented by a Drill "segment" operation
- * followed by a "collapseaggregate" operation.
- */
-public class DrillAggregateRule extends RelOptRule {
- public static final RelOptRule INSTANCE = new DrillAggregateRule();
- protected static final Logger tracer = EigenbaseTrace.getPlannerTracer();
-
- private DrillAggregateRule() {
- super(RelOptRule.some(AggregateRel.class, Convention.NONE, RelOptRule.any(RelNode.class)), "DrillAggregateRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final AggregateRel aggregate = (AggregateRel) call.rel(0);
- final RelNode input = call.rel(1);
- final RelTraitSet traits = aggregate.getTraitSet().plus(DrillRel.CONVENTION);
- final RelNode convertedInput = convert(input, traits);
- try {
- call.transformTo(new DrillAggregateRel(aggregate.getCluster(), traits, convertedInput, aggregate.getGroupSet(),
- aggregate.getAggCallList()));
- } catch (InvalidRelException e) {
- tracer.warning(e.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillFilterRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillFilterRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillFilterRel.java
deleted file mode 100644
index e01d645..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillFilterRel.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.eigenbase.rel.FilterRelBase;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
-import org.eigenbase.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Filter implemented in Drill.
- */
-public class DrillFilterRel extends FilterRelBase implements DrillRel {
- protected DrillFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) {
- super(cluster, traits, child, condition);
- assert getConvention() == CONVENTION;
- }
-
- @Override
- public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- return new DrillFilterRel(getCluster(), traitSet, sole(inputs), getCondition());
- }
-
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(0.1);
- }
-
- @Override
- public int implement(DrillImplementor implementor) {
- final int inputId = implementor.visitChild(this, 0, getChild());
- final ObjectNode filter = implementor.mapper.createObjectNode();
- /*
- * E.g. { op: "filter", expr: "donuts.ppu < 1.00" }
- */
- filter.put("op", "filter");
- filter.put("input", inputId);
- filter.put("expr", DrillOptiq.toDrill(getChild(), getCondition()));
- return implementor.add(filter);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillFilterRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillFilterRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillFilterRule.java
deleted file mode 100644
index eff0715..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillFilterRule.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.FilterRel;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
-
-/**
- * Rule that converts a {@link org.eigenbase.rel.FilterRel} to a Drill "filter" operation.
- */
-public class DrillFilterRule extends RelOptRule {
- public static final RelOptRule INSTANCE = new DrillFilterRule();
-
- private DrillFilterRule() {
- super(RelOptRule.some(FilterRel.class, Convention.NONE, RelOptRule.any(RelNode.class)), "DrillFilterRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final FilterRel filter = (FilterRel) call.getRels()[0];
- final RelNode input = call.getRels()[1];
- final RelTraitSet traits = filter.getTraitSet().plus(DrillRel.CONVENTION);
- final RelNode convertedInput = convert(input, traits);
- call.transformTo(new DrillFilterRel(filter.getCluster(), traits, convertedInput, filter.getCondition()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
deleted file mode 100644
index 715e64f..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.exec.ref.rse.QueueRSE.QueueOutputInfo;
-import org.apache.drill.jdbc.DrillTable;
-import org.eigenbase.rel.RelNode;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Sets;
-
-/**
- * Context for converting a tree of {@link DrillRel} nodes into a Drill logical plan.
- */
-public class DrillImplementor {
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillImplementor.class);
-
- final ObjectMapper mapper = new ObjectMapper();
- {
- mapper.enable(SerializationFeature.INDENT_OUTPUT);
- mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
- mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
- }
- private final ObjectNode rootNode = mapper.createObjectNode();
- private final ArrayNode operatorsNode;
- private final ObjectNode sourcesNode;
- private Set<DrillTable> tables = Sets.newHashSet();
- private final boolean isRef;
-
- public DrillImplementor(boolean isRef) {
- this.isRef = isRef;
- final ObjectNode headNode = mapper.createObjectNode();
- rootNode.put("head", headNode);
- headNode.put("type", "APACHE_DRILL_LOGICAL");
- headNode.put("version", "1");
-
- final ObjectNode generatorNode = mapper.createObjectNode();
- headNode.put("generator", generatorNode);
- generatorNode.put("type", "optiq");
- generatorNode.put("info", "na");
-
- // TODO: populate sources based on the sources of scans that occur in
- // the query
- sourcesNode = mapper.createObjectNode();
- rootNode.put("storage", sourcesNode);
-
- if(isRef){
- {
- final ObjectNode sourceNode = mapper.createObjectNode();
- sourceNode.put("type", "classpath");
- sourcesNode.put("donuts-json", sourceNode);
- }
- {
- final ObjectNode sourceNode = mapper.createObjectNode();
- sourceNode.put("type", "queue");
- sourcesNode.put("queue", sourceNode);
- }
- }
-
-
- final ArrayNode queryNode = mapper.createArrayNode();
- rootNode.put("query", queryNode);
-
- this.operatorsNode = queryNode;
- }
-
- public void registerSource(DrillTable table){
- if(!table.isRefEngine() && tables.add(table)){
- sourcesNode.put(table.getStorageEngineName(), mapper.convertValue(table.getStorageEngineConfig(), JsonNode.class));
- }
- }
-
- public int go(DrillRel root) {
- int inputId = root.implement(this);
-
- // Add a last node, to write to the output queue.
- final ObjectNode writeOp = mapper.createObjectNode();
- writeOp.put("op", "store");
- writeOp.put("input", inputId);
- writeOp.put("storageengine", "queue");
- writeOp.put("memo", "output sink");
- QueueOutputInfo output = new QueueOutputInfo();
- output.number = 0;
- writeOp.put("target", mapper.convertValue(output, JsonNode.class));
- return add(writeOp);
- }
-
- public int add(LogicalOperator operator, Integer inputId) {
- return add(operator.nodeBuilder().convert(mapper, operator, inputId));
- }
-
- public int add(ObjectNode operator) {
- operatorsNode.add(operator);
- final int id = operatorsNode.size();
- operator.put("@id", id);
- return id;
- }
-
- /** Returns the generated plan. */
- public String getJsonString() {
- String s = rootNode.toString();
-
- if(logger.isDebugEnabled()){
- JsonNode node;
- try {
- ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
- node = mapper.readValue(s, JsonNode.class);
- logger.debug("Optiq Generated Logical Plan: {}", mapper.writeValueAsString(node));
- } catch (IOException e) {
- logger.error("Failure while trying to parse logical plan string of {}", s, e);
- }
-
- }
- return s;
- }
-
- public int visitChild(DrillRel parent, int ordinal, RelNode child) {
- ((DrillRel) child).implement(this);
- return operatorsNode.size();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
deleted file mode 100644
index acdc5c7..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-
-import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.JoinRelBase;
-import org.eigenbase.rel.JoinRelType;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptUtil;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.rex.RexNode;
-import org.eigenbase.util.Pair;
-
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * Join implemented in Drill.
- */
-public class DrillJoinRel extends JoinRelBase implements DrillRel {
- private final List<Integer> leftKeys = new ArrayList<>();
- private final List<Integer> rightKeys = new ArrayList<>();
-
- /** Creates a DrillJoinRel. */
- public DrillJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition,
- JoinRelType joinType) throws InvalidRelException {
- super(cluster, traits, left, right, condition, joinType, Collections.<String> emptySet());
-
- RexNode remaining = RelOptUtil.splitJoinCondition(left, right, condition, leftKeys, rightKeys);
- if (!remaining.isAlwaysTrue()) {
- throw new InvalidRelException("DrillJoinRel only supports equi-join");
- }
- }
-
- @Override
- public DrillJoinRel copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right) {
- try {
- return new DrillJoinRel(getCluster(), traitSet, left, right, condition, joinType);
- } catch (InvalidRelException e) {
- throw new AssertionError(e);
- }
- }
-
- @Override
- public int implement(DrillImplementor implementor) {
- final List<String> fields = getRowType().getFieldNames();
- assert isUnique(fields);
- final int leftCount = left.getRowType().getFieldCount();
- final List<String> leftFields = fields.subList(0, leftCount);
- final List<String> rightFields = fields.subList(leftCount, fields.size());
-
- final int leftId = implementInput(implementor, 0, 0, left);
- final int rightId = implementInput(implementor, 1, leftCount, right);
-
- /*
- * E.g. { op: "join", left: 2, right: 4, conditions: [ {relationship: "==", left: "deptId", right: "deptId"} ] }
- */
- final ObjectNode join = implementor.mapper.createObjectNode();
- join.put("op", "join");
- join.put("left", leftId);
- join.put("right", rightId);
- join.put("type", toDrill(joinType));
- final ArrayNode conditions = implementor.mapper.createArrayNode();
- join.put("conditions", conditions);
- for (Pair<Integer, Integer> pair : Pair.zip(leftKeys, rightKeys)) {
- final ObjectNode condition = implementor.mapper.createObjectNode();
- condition.put("relationship", "==");
- condition.put("left", leftFields.get(pair.left));
- condition.put("right", rightFields.get(pair.right));
- conditions.add(condition);
- }
- return implementor.add(join);
- }
-
- private int implementInput(DrillImplementor implementor, int i, int offset, RelNode input) {
- final int inputId = implementor.visitChild(this, i, input);
- assert uniqueFieldNames(input.getRowType());
- final List<String> fields = getRowType().getFieldNames();
- final List<String> inputFields = input.getRowType().getFieldNames();
- final List<String> outputFields = fields.subList(offset, offset + inputFields.size());
- if (!outputFields.equals(inputFields)) {
- // Ensure that input field names are the same as output field names.
- // If there are duplicate field names on left and right, fields will get
- // lost.
- return rename(implementor, inputId, inputFields, outputFields);
- } else {
- return inputId;
- }
- }
-
- private int rename(DrillImplementor implementor, int inputId, List<String> inputFields, List<String> outputFields) {
- final ObjectNode project = implementor.mapper.createObjectNode();
- project.put("op", "project");
- project.put("input", inputId);
- final ArrayNode transforms = implementor.mapper.createArrayNode();
- project.put("projections", transforms);
- for (Pair<String, String> pair : Pair.zip(inputFields, outputFields)) {
- final ObjectNode objectNode = implementor.mapper.createObjectNode();
- transforms.add(objectNode);
- objectNode.put("expr", pair.left);
- objectNode.put("ref", "output." + pair.right);
-// objectNode.put("ref", pair.right);
- }
- return implementor.add(project);
- }
-
- /**
- * Returns whether there are any elements in common between left and right.
- */
- private static <T> boolean intersects(List<T> left, List<T> right) {
- return new HashSet<>(left).removeAll(right);
- }
-
- private boolean uniqueFieldNames(RelDataType rowType) {
- return isUnique(rowType.getFieldNames());
- }
-
- private static <T> boolean isUnique(List<T> list) {
- return new HashSet<>(list).size() == list.size();
- }
-
- private static String toDrill(JoinRelType joinType) {
- switch (joinType) {
- case LEFT:
- return "left";
- case RIGHT:
- return "right";
- case INNER:
- return "inner";
- case FULL:
- return "outer";
- default:
- throw new AssertionError(joinType);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRule.java
deleted file mode 100644
index 3c5e072..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRule.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.JoinRel;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
-import org.eigenbase.trace.EigenbaseTrace;
-
-import java.util.logging.Logger;
-
-/**
- * Rule that converts a {@link JoinRel} to a {@link DrillJoinRel}, which is implemented by Drill "join" operation.
- */
-public class DrillJoinRule extends RelOptRule {
- public static final RelOptRule INSTANCE = new DrillJoinRule();
- protected static final Logger tracer = EigenbaseTrace.getPlannerTracer();
-
- private DrillJoinRule() {
- super(
- RelOptRule.some(JoinRel.class, Convention.NONE, RelOptRule.any(RelNode.class), RelOptRule.any(RelNode.class)),
- "DrillJoinRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final JoinRel join = (JoinRel) call.rel(0);
- final RelNode left = call.rel(1);
- final RelNode right = call.rel(2);
- final RelTraitSet traits = join.getTraitSet().plus(DrillRel.CONVENTION);
-
- final RelNode convertedLeft = convert(left, traits);
- final RelNode convertedRight = convert(right, traits);
- try {
- call.transformTo(new DrillJoinRel(join.getCluster(), traits, convertedLeft, convertedRight, join.getCondition(),
- join.getJoinType()));
- } catch (InvalidRelException e) {
- tracer.warning(e.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillLimitRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillLimitRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillLimitRel.java
deleted file mode 100644
index c1bf3d2..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillLimitRel.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.drill.common.logical.data.Limit;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.SingleRel;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.rex.RexLiteral;
-import org.eigenbase.rex.RexNode;
-
-import java.util.List;
-
-public class DrillLimitRel extends SingleRel implements DrillRel {
- private RexNode offset;
- private RexNode fetch;
-
- public DrillLimitRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, RexNode offset, RexNode fetch) {
- super(cluster, traitSet, child);
- this.offset = offset;
- this.fetch = fetch;
- }
-
- @Override
- public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- return new DrillLimitRel(getCluster(), traitSet, sole(inputs), offset, fetch);
- }
-
- @Override
- public int implement(DrillImplementor implementor) {
- int inputId = implementor.visitChild(this, 0, getChild());
-
-
- // First offset to include into results (inclusive). Null implies it is starting from offset 0
- int first = offset != null ? Math.max(0, RexLiteral.intValue(offset)) : 0;
-
- // Last offset to stop including into results (exclusive), translating fetch row counts into an offset.
- // Null value implies including entire remaining result set from first offset
- Integer last = fetch != null ? Math.max(0, RexLiteral.intValue(fetch)) + first : null;
- return implementor.add(new Limit(first, last), inputId);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillLimitRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillLimitRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillLimitRule.java
deleted file mode 100644
index 0b835c9..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillLimitRule.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.SortRel;
-import org.eigenbase.relopt.Convention;
-import org.eigenbase.relopt.RelOptRule;
-import org.eigenbase.relopt.RelOptRuleCall;
-import org.eigenbase.relopt.RelTraitSet;
-
-/**
- * This rule converts a SortRel that has either a offset and fetch into a Drill Sort and Limit Rel
- */
-public class DrillLimitRule extends RelOptRule {
- public static DrillLimitRule INSTANCE = new DrillLimitRule();
-
- private DrillLimitRule() {
- super(RelOptRule.some(SortRel.class, Convention.NONE, RelOptRule.any(RelNode.class)), "DrillLimitRule");
- }
-
- @Override
- public boolean matches(RelOptRuleCall call) {
- final SortRel sort = call.rel(0);
- return sort.offset != null || sort.fetch != null;
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final SortRel incomingSort = call.rel(0);
- final RelTraitSet incomingTraits = incomingSort.getTraitSet();
- RelNode input = incomingSort.getChild();
-
- // if the Optiq sort rel includes a collation and a limit, we need to create a copy the sort rel that excludes the
- // limit information.
- if (!incomingSort.getCollation().getFieldCollations().isEmpty()) {
- input = incomingSort.copy(incomingTraits, input, incomingSort.getCollation(), null, null);
- }
-
- RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.CONVENTION));
- call.transformTo(new DrillLimitRel(incomingSort.getCluster(), incomingTraits.plus(DrillRel.CONVENTION), convertedInput, incomingSort.offset, incomingSort.fetch));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
deleted file mode 100644
index 1897151..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import net.hydromatic.linq4j.Ord;
-
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.work.fragment.StatusReporter;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.reltype.RelDataTypeField;
-import org.eigenbase.rex.RexCall;
-import org.eigenbase.rex.RexCorrelVariable;
-import org.eigenbase.rex.RexDynamicParam;
-import org.eigenbase.rex.RexFieldAccess;
-import org.eigenbase.rex.RexInputRef;
-import org.eigenbase.rex.RexLiteral;
-import org.eigenbase.rex.RexLocalRef;
-import org.eigenbase.rex.RexNode;
-import org.eigenbase.rex.RexOver;
-import org.eigenbase.rex.RexRangeRef;
-import org.eigenbase.rex.RexVisitorImpl;
-import org.eigenbase.sql.SqlSyntax;
-import org.eigenbase.sql.fun.SqlStdOperatorTable;
-
-/**
- * Utilities for Drill's planner.
- */
-public class DrillOptiq {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillOptiq.class);
-
- static void registerStandardPlannerRules(RelOptPlanner planner, DrillClient client) {
- planner.addRule(EnumerableDrillRule.getInstance(client));
-
- // planner.addRule(DrillTableModificationConverterRule.INSTANCE);
- // planner.addRule(DrillCalcConverterRule.INSTANCE);
-
- planner.addRule(DrillFilterRule.INSTANCE);
- planner.addRule(DrillProjectRule.INSTANCE);
- planner.addRule(DrillAggregateRule.INSTANCE);
-
- // Enable when https://issues.apache.org/jira/browse/DRILL-57 fixed
- if (false) planner.addRule(DrillValuesRule.INSTANCE);
- planner.addRule(DrillLimitRule.INSTANCE);
- planner.addRule(DrillSortRule.INSTANCE);
- planner.addRule(DrillJoinRule.INSTANCE);
- planner.addRule(DrillUnionRule.INSTANCE);
- // planner.addRule(AbstractConverter.ExpandConversionRule.instance);
- }
-
- /**
- * Converts a tree of {@link RexNode} operators into a scalar expression in Drill syntax.
- */
- static String toDrill(RelNode input, RexNode expr) {
- final RexToDrill visitor = new RexToDrill(input);
- expr.accept(visitor);
- return visitor.buf.toString();
- }
-
- private static class RexToDrill extends RexVisitorImpl<StringBuilder> {
- final StringBuilder buf = new StringBuilder();
- private final RelNode input;
-
- RexToDrill(RelNode input) {
- super(true);
- this.input = input;
- }
-
- @Override
- public StringBuilder visitCall(RexCall call) {
- logger.debug("RexCall {}, {}", call);
- final SqlSyntax syntax = call.getOperator().getSyntax();
- switch (syntax) {
- case Binary:
- logger.debug("Binary");
- buf.append("(");
- call.getOperands().get(0).accept(this).append(" ").append(call.getOperator().getName()).append(" ");
- return call.getOperands().get(1).accept(this).append(")");
- case Function:
- logger.debug("Function");
- buf.append(call.getOperator().getName().toLowerCase()).append("(");
- for (Ord<RexNode> operand : Ord.zip(call.getOperands())) {
- buf.append(operand.i > 0 ? ", " : "");
- operand.e.accept(this);
- }
- return buf.append(")");
- case Special:
- logger.debug("Special");
- switch (call.getKind()) {
- case Cast:
- logger.debug("Cast {}", buf);
- // Ignore casts. Drill is type-less.
- logger.debug("Ignoring cast {}, {}", call.getOperands().get(0), call.getOperands().get(0).getClass());
- return call.getOperands().get(0).accept(this);
- }
- if (call.getOperator() == SqlStdOperatorTable.itemOp) {
- final RexNode left = call.getOperands().get(0);
- final RexLiteral literal = (RexLiteral) call.getOperands().get(1);
- final String field = (String) literal.getValue2();
-// buf.append('\'');
- final int length = buf.length();
- left.accept(this);
- if (buf.length() > length) {
- // check before generating empty LHS if inputName is null
- buf.append('.');
- }
- buf.append(field);
-// buf.append('\'');
- return buf;
- }
- logger.debug("Not cast.");
- // fall through
- default:
- throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
- }
- }
-
- private StringBuilder doUnknown(Object o){
- logger.warn("Doesn't currently support consumption of {}.", o);
- return buf;
- }
- @Override
- public StringBuilder visitLocalRef(RexLocalRef localRef) {
- return doUnknown(localRef);
- }
-
- @Override
- public StringBuilder visitOver(RexOver over) {
- return doUnknown(over);
- }
-
- @Override
- public StringBuilder visitCorrelVariable(RexCorrelVariable correlVariable) {
- return doUnknown(correlVariable);
- }
-
- @Override
- public StringBuilder visitDynamicParam(RexDynamicParam dynamicParam) {
- return doUnknown(dynamicParam);
- }
-
- @Override
- public StringBuilder visitRangeRef(RexRangeRef rangeRef) {
- return doUnknown(rangeRef);
- }
-
- @Override
- public StringBuilder visitFieldAccess(RexFieldAccess fieldAccess) {
- return super.visitFieldAccess(fieldAccess);
- }
-
- @Override
- public StringBuilder visitInputRef(RexInputRef inputRef) {
- final int index = inputRef.getIndex();
- final RelDataTypeField field = input.getRowType().getFieldList().get(index);
- buf.append(field.getName());
- return buf;
- }
-
- @Override
- public StringBuilder visitLiteral(RexLiteral literal) {
- return buf.append(literal);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
deleted file mode 100644
index af7bbc7..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import net.hydromatic.optiq.prepare.OptiqPrepareImpl;
-import net.hydromatic.optiq.rules.java.JavaRules;
-
-import org.apache.drill.jdbc.Driver;
-import org.eigenbase.relopt.RelOptPlanner;
-
-/**
- * Implementation of {@link net.hydromatic.optiq.jdbc.OptiqPrepare} for Drill.
- */
-public class DrillPrepareImpl extends OptiqPrepareImpl {
- final Driver driver;
- public DrillPrepareImpl(Driver driver) {
- super();
- this.driver = driver;
- }
-
- @Override
- protected RelOptPlanner createPlanner() {
- final RelOptPlanner planner = super.createPlanner();
- planner.addRule(EnumerableDrillRule.getInstance(driver == null ? null : driver.getClient()));
- planner.addRule(DrillValuesRule.INSTANCE);
-
- planner.removeRule(JavaRules.ENUMERABLE_JOIN_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_CALC_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_AGGREGATE_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_SORT_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_LIMIT_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_UNION_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_INTERSECT_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_MINUS_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_TABLE_MODIFICATION_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_VALUES_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_WINDOW_RULE);
- planner.removeRule(JavaRules.ENUMERABLE_ONE_ROW_RULE);
- return planner;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
deleted file mode 100644
index fabfb77..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.eigenbase.rel.*;
-import org.eigenbase.relopt.*;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.rex.RexNode;
-import org.eigenbase.util.Pair;
-
-import java.util.*;
-
-/**
- * Project implemented in Drill.
- */
-public class DrillProjectRel extends ProjectRelBase implements DrillRel {
- protected DrillProjectRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, List<RexNode> exps,
- RelDataType rowType) {
- super(cluster, traits, child, exps, rowType, Flags.Boxed, Collections.<RelCollation> emptyList());
- assert getConvention() == CONVENTION;
- }
-
- public DrillProjectRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, List<RexNode> exps,
- RelDataType rowType, int flags, List<RelCollation> collationList) {
- super(cluster, traits, child, exps, rowType, flags, collationList);
-
- }
-
- @Override
- public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- return new DrillProjectRel(getCluster(), traitSet, sole(inputs), new ArrayList<RexNode>(exps), rowType);
- }
-
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(0.1);
- }
-
- private List<Pair<RexNode, String>> projects() {
- return Pair.zip(exps, getRowType().getFieldNames());
- }
-
- @Override
- public int implement(DrillImplementor implementor) {
- int inputId = implementor.visitChild(this, 0, getChild());
- final ObjectNode project = implementor.mapper.createObjectNode();
- /*
- * E.g. { op: "project", projections: [ { ref: "output.quantity", expr: "donuts.sales"} ]
- */
- project.put("op", "project");
- project.put("input", inputId);
- final ArrayNode transforms = implementor.mapper.createArrayNode();
- project.put("projections", transforms);
- for (Pair<RexNode, String> pair : projects()) {
- final ObjectNode objectNode = implementor.mapper.createObjectNode();
- transforms.add(objectNode);
- String expr = DrillOptiq.toDrill(getChild(), pair.left);
- objectNode.put("expr", expr);
- String ref = "output." + pair.right;
-// String ref = pair.right;
- objectNode.put("ref", ref);
- }
- return implementor.add(project);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRule.java
deleted file mode 100644
index 36b272f..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRule.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.ProjectRel;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.Convention;
-import org.eigenbase.relopt.RelOptRule;
-import org.eigenbase.relopt.RelOptRuleCall;
-import org.eigenbase.relopt.RelTraitSet;
-
-/**
- * Rule that converts a {@link org.eigenbase.rel.ProjectRel} to a Drill "project" operation.
- */
-public class DrillProjectRule extends RelOptRule {
- public static final RelOptRule INSTANCE = new DrillProjectRule();
-
- private DrillProjectRule() {
- super(RelOptRule.some(ProjectRel.class, Convention.NONE, RelOptRule.any(RelNode.class)), "DrillProjectRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final ProjectRel project = (ProjectRel) call.rel(0);
- final RelNode input = call.rel(1);
- final RelTraitSet traits = project.getTraitSet().plus(DrillRel.CONVENTION);
- final RelNode convertedInput = convert(input, traits);
- call.transformTo(new DrillProjectRel(project.getCluster(), traits, convertedInput, project.getProjects(), project
- .getRowType()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillRel.java
deleted file mode 100644
index 4c9ece9..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillRel.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.Convention;
-
-/**
- * Relational expression that is implemented in Drill.
- */
-public interface DrillRel extends RelNode {
- /** Calling convention for relational expressions that are "implemented" by
- * generating Drill logical plans. */
- Convention CONVENTION = new Convention.Impl("DRILL", DrillRel.class);
-
- int implement(DrillImplementor implementor);
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java
deleted file mode 100644
index a165fea..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.apache.drill.jdbc.DrillTable;
-import org.eigenbase.rel.TableAccessRelBase;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelOptTable;
-import org.eigenbase.relopt.RelTraitSet;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * GroupScan of a Drill table.
- */
-public class DrillScan extends TableAccessRelBase implements DrillRel {
- private final DrillTable drillTable;
-
- /** Creates a DrillScan. */
- public DrillScan(RelOptCluster cluster, RelTraitSet traits, RelOptTable table) {
- super(cluster, traits, table);
- assert getConvention() == CONVENTION;
- this.drillTable = table.unwrap(DrillTable.class);
- assert drillTable != null;
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- super.register(planner);
- DrillOptiq.registerStandardPlannerRules(planner, drillTable.client);
- }
-
- public int implement(DrillImplementor implementor) {
- final ObjectNode node = implementor.mapper.createObjectNode();
- node.put("op", "scan");
- node.put("memo", "initial_scan");
- node.put("ref", "_MAP"); // output is a record with a single field, '_MAP'
- node.put("storageengine", drillTable.getStorageEngineName());
- node.put("selection", implementor.mapper.convertValue(drillTable.getSelection(), JsonNode.class));
- implementor.registerSource(drillTable);
- return implementor.add(node);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillSortRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillSortRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillSortRel.java
deleted file mode 100644
index 929d381..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillSortRel.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import java.util.List;
-
-import net.hydromatic.linq4j.Ord;
-
-import org.eigenbase.rel.RelCollation;
-import org.eigenbase.rel.RelFieldCollation;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.SortRel;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelTraitSet;
-
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.eigenbase.rex.RexLiteral;
-import org.eigenbase.rex.RexNode;
-
-/**
- * Sort implemented in Drill.
- */
-public class DrillSortRel extends SortRel implements DrillRel {
-
- /** Creates a DrillSortRel. */
- public DrillSortRel(RelOptCluster cluster, RelTraitSet traits, RelNode input, RelCollation collation) {
- super(cluster, traits, input, collation);
- }
-
- /** Creates a DrillSortRel with offset and fetch. */
- public DrillSortRel(RelOptCluster cluster, RelTraitSet traits, RelNode input, RelCollation collation, RexNode offset, RexNode fetch) {
- super(cluster, traits, input, collation, offset, fetch);
- }
-
- @Override
- public DrillSortRel copy(RelTraitSet traitSet, RelNode input, RelCollation collation, RexNode offset, RexNode fetch) {
- return new DrillSortRel(getCluster(), traitSet, input, collation, offset, fetch);
- }
-
- @Override
- public int implement(DrillImplementor implementor) {
- int inputId = implementor.visitChild(this, 0, getChild());
- final List<String> childFields = getChild().getRowType().getFieldNames();
- /*
- * E.g. { op: "order", input: 4, ordering: [ {order: "asc", expr: "deptId"} ] }
- */
- final ObjectNode order = implementor.mapper.createObjectNode();
- order.put("op", "order");
- order.put("input", inputId);
- final ArrayNode orderings = implementor.mapper.createArrayNode();
- order.put("orderings", orderings);
- for (Ord<RelFieldCollation> fieldCollation : Ord.zip(this.collation.getFieldCollations())) {
- final ObjectNode ordering = implementor.mapper.createObjectNode();
- ordering.put("order", toDrill(fieldCollation.e));
- ordering.put("expr", childFields.get(fieldCollation.e.getFieldIndex()));
- switch (fieldCollation.e.nullDirection) {
- case FIRST:
- ordering.put("nullCollation", "first");
- break;
- default:
- ordering.put("nullCollation", "last");
- break;
- }
- orderings.add(ordering);
- }
-
- return implementor.add(order);
- }
-
-
-
- private static String toDrill(RelFieldCollation collation) {
- switch (collation.getDirection()) {
- case Ascending:
- return "asc";
- case Descending:
- return "desc";
- default:
- throw new AssertionError(collation.getDirection());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillSortRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillSortRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillSortRule.java
deleted file mode 100644
index 93ffb62..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillSortRule.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.SortRel;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
-
-/**
- * Rule that converts an {@link SortRel} to a {@link DrillSortRel}, implemented by a Drill "order" operation.
- */
-public class DrillSortRule extends RelOptRule {
- public static final RelOptRule INSTANCE = new DrillSortRule();
-
- private DrillSortRule() {
- super(RelOptRule.some(SortRel.class, Convention.NONE, RelOptRule.any(RelNode.class)), "DrillSortRule");
- }
-
- @Override
- public boolean matches(RelOptRuleCall call) {
- final SortRel sort = call.rel(0);
- return sort.offset == null && sort.fetch == null;
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
-
- final SortRel sort = call.rel(0);
-
- final RelNode input = call.rel(1);
- final RelTraitSet traits = sort.getTraitSet().plus(DrillRel.CONVENTION);
-
- final RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.CONVENTION));
- call.transformTo(new DrillSortRel(sort.getCluster(), traits, convertedInput, sort.getCollation()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillUnionRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillUnionRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillUnionRel.java
deleted file mode 100644
index 90cda2e..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillUnionRel.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import net.hydromatic.linq4j.Ord;
-import org.eigenbase.rel.UnionRelBase;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Union implemented in Drill.
- */
-public class DrillUnionRel extends UnionRelBase implements DrillRel {
- /** Creates a DrillUnionRel. */
- public DrillUnionRel(RelOptCluster cluster, RelTraitSet traits,
- List<RelNode> inputs, boolean all) {
- super(cluster, traits, inputs, all);
- }
-
- @Override
- public DrillUnionRel copy(RelTraitSet traitSet, List<RelNode> inputs,
- boolean all) {
- return new DrillUnionRel(getCluster(), traitSet, inputs, all);
- }
-
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- // divide cost by two to ensure cheaper than EnumerableDrillRel
- return super.computeSelfCost(planner).multiplyBy(.5);
- }
-
- @Override
- public int implement(DrillImplementor implementor) {
- List<Integer> inputIds = new ArrayList<>();
- for (Ord<RelNode> input : Ord.zip(inputs)) {
- inputIds.add(implementor.visitChild(this, input.i, input.e));
- }
-/*
- E.g. {
- op: "union",
- distinct: true,
- inputs: [2, 4]
- }
-*/
- final ObjectNode union = implementor.mapper.createObjectNode();
- union.put("op", "union");
- union.put("distinct", !all);
- final ArrayNode inputs = implementor.mapper.createArrayNode();
- union.put("inputs", inputs);
- for (Integer inputId : inputIds) {
- inputs.add(inputId);
- }
- return implementor.add(union);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillUnionRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillUnionRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillUnionRule.java
deleted file mode 100644
index 0d70443..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillUnionRule.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.UnionRel;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Rule that converts a {@link UnionRel} to a {@link DrillUnionRel}, implemented by a "union" operation.
- */
-public class DrillUnionRule extends RelOptRule {
- public static final RelOptRule INSTANCE = new DrillUnionRule();
-
- private DrillUnionRule() {
- super(RelOptRule.any(UnionRel.class, Convention.NONE), "DrillUnionRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final UnionRel union = (UnionRel) call.rel(0);
- final RelTraitSet traits = union.getTraitSet().plus(DrillRel.CONVENTION);
- final List<RelNode> convertedInputs = new ArrayList<>();
- for (RelNode input : union.getInputs()) {
- final RelNode convertedInput = convert(input, traits);
- convertedInputs.add(convertedInput);
- }
- call.transformTo(new DrillUnionRel(union.getCluster(), traits, convertedInputs, union.all));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillValuesRel.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillValuesRel.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillValuesRel.java
deleted file mode 100644
index 5bab4f8..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillValuesRel.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.ValuesRelBase;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.*;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.rex.RexLiteral;
-
-import java.util.List;
-
-/**
- * Values implemented in Drill.
- */
-public class DrillValuesRel extends ValuesRelBase implements DrillRel {
- protected DrillValuesRel(RelOptCluster cluster, RelDataType rowType, List<List<RexLiteral>> tuples, RelTraitSet traits) {
- super(cluster, rowType, tuples, traits);
- assert getConvention() == CONVENTION;
- }
-
- @Override
- public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- assert inputs.isEmpty();
- return new DrillValuesRel(getCluster(), rowType, tuples, traitSet);
- }
-
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner) {
- return super.computeSelfCost(planner).multiplyBy(0.1);
- }
-
- @Override
- public int implement(DrillImplementor implementor) {
- // Update when https://issues.apache.org/jira/browse/DRILL-57 fixed
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/DrillValuesRule.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/DrillValuesRule.java b/sqlparser/src/main/java/org/apache/drill/optiq/DrillValuesRule.java
deleted file mode 100644
index b5e3eb2..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/DrillValuesRule.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import org.eigenbase.rel.ValuesRel;
-import org.eigenbase.relopt.*;
-
-/**
- * Rule that converts a {@link ValuesRel} to a Drill "values" operation.
- */
-public class DrillValuesRule extends RelOptRule {
- public static final RelOptRule INSTANCE = new DrillValuesRule();
-
- private DrillValuesRule() {
- super(RelOptRule.any(ValuesRel.class, Convention.NONE), "DrillValuesRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final ValuesRel values = (ValuesRel) call.rel(0);
- final RelTraitSet traits = values.getTraitSet().plus(DrillRel.CONVENTION);
- call.transformTo(new DrillValuesRel(values.getCluster(), values.getRowType(), values.getTuples(), traits));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java b/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
deleted file mode 100644
index 7b0b24d..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import net.hydromatic.linq4j.AbstractEnumerable;
-import net.hydromatic.linq4j.Enumerable;
-import net.hydromatic.linq4j.Enumerator;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.exec.ref.IteratorRegistry;
-import org.apache.drill.exec.ref.ReferenceInterpreter;
-import org.apache.drill.exec.ref.RunOutcome;
-import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
-import org.apache.drill.exec.ref.rse.RSERegistry;
-import org.apache.drill.exec.store.json.JSONRecordReader;
-import org.apache.drill.jdbc.DrillHandler;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * Runtime helper that executes a Drill query and converts it into an {@link Enumerable}.
- */
-public class EnumerableDrill<E> extends AbstractEnumerable<E> implements Enumerable<E> {
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EnumerableDrill.class);
-
- private final LogicalPlan plan;
- final BlockingQueue<Object> queue = new ArrayBlockingQueue<>(100);
- final DrillConfig config;
- private final String holder;
- private final List<String> fields;
-
- private static final ObjectMapper mapper = createMapper();
-
- /**
- * Creates a DrillEnumerable.
- *
- * @param plan
- * Logical plan
- * @param clazz
- * Type of elements returned from enumerable
- * @param fields
- * Names of fields, or null to return the whole blob
- */
- public EnumerableDrill(DrillConfig config, LogicalPlan plan, Class<E> clazz, List<String> fields) {
- this.plan = plan;
- this.config = config;
- this.holder = null;
- this.fields = fields;
- config.setSinkQueues(0, queue);
- }
-
- /**
- * Creates a DrillEnumerable from a plan represented as a string. Each record returned is a {@link JsonNode}.
- */
- public static <E> EnumerableDrill<E> of(DrillHandler handler, String plan, final List<String> fieldNames, Class<E> clazz) {
- DrillConfig config = DrillConfig.create();
- final LogicalPlan parse = LogicalPlan.parse(config, plan);
- return new EnumerableDrill<>(config, parse, clazz, fieldNames);
- }
-
- /** Runs the plan as a background task. */
- Future<Collection<RunOutcome>> runPlan(CompletionService<Collection<RunOutcome>> service) {
- IteratorRegistry ir = new IteratorRegistry();
- DrillConfig config = DrillConfig.create();
- config.setSinkQueues(0, queue);
- final ReferenceInterpreter i = new ReferenceInterpreter(plan, ir, new BasicEvaluatorFactory(ir), new RSERegistry(
- config));
- try {
- i.setup();
- } catch (IOException e) {
- logger.error("IOException during query", e);
- throw new RuntimeException(e);
- }
- return service.submit(new Callable<Collection<RunOutcome>>() {
- @Override
- public Collection<RunOutcome> call() throws Exception {
- Collection<RunOutcome> outcomes = i.run();
-
- for (RunOutcome outcome : outcomes) {
- System.out.println("============");
- System.out.println(outcome);
- if (outcome.outcome == RunOutcome.OutcomeType.FAILED && outcome.exception != null) {
- outcome.exception.printStackTrace();
- }
- }
- return outcomes;
- }
- });
- }
-
- @Override
- public Enumerator<E> enumerator() {
- // TODO: use a completion service from the container
- final ExecutorCompletionService<Collection<RunOutcome>> service = new ExecutorCompletionService<Collection<RunOutcome>>(
- new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingDeque<Runnable>(10)));
-
- // Run the plan using an executor. It runs in a different thread, writing
- // results to our queue.
- //
- // TODO: use the result of task, and check for exceptions
- final Future<Collection<RunOutcome>> task = runPlan(service);
-
- return new JsonEnumerator(queue, fields);
- }
-
- private static ObjectMapper createMapper() {
- return new ObjectMapper();
- }
-
- /**
- * Converts a JSON document, represented as an array of bytes, into a Java object (consisting of DistributedMap, List, String,
- * Integer, Double, Boolean).
- */
- static Object parseJson(byte[] bytes) {
- try {
- return wrapper(mapper.readTree(bytes));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Converts a JSON node to Java objects ({@link List}, {@link Map}, {@link String}, {@link Integer}, {@link Double},
- * {@link Boolean}.
- */
- static Object wrapper(JsonNode node) {
- switch (node.asToken()) {
- case START_OBJECT:
- return map((ObjectNode) node);
- case START_ARRAY:
- return array((ArrayNode) node);
- case VALUE_STRING:
- return node.asText();
- case VALUE_NUMBER_INT:
- return node.asInt();
- case VALUE_NUMBER_FLOAT:
- return node.asDouble();
- case VALUE_TRUE:
- return Boolean.TRUE;
- case VALUE_FALSE:
- return Boolean.FALSE;
- case VALUE_NULL:
- return null;
- default:
- throw new AssertionError("unexpected: " + node + ": " + node.asToken());
- }
- }
-
- private static List<Object> array(ArrayNode node) {
- final List<Object> list = new ArrayList<>();
- for (JsonNode jsonNode : node) {
- list.add(wrapper(jsonNode));
- }
- return Collections.unmodifiableList(list);
- }
-
- private static SortedMap<String, Object> map(ObjectNode node) {
- // TreeMap makes the results deterministic.
- final TreeMap<String, Object> map = new TreeMap<>();
- final Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
- while (fields.hasNext()) {
- Map.Entry<String, JsonNode> next = fields.next();
- map.put(next.getKey(), wrapper(next.getValue()));
- }
- return Collections.unmodifiableSortedMap(map);
- }
-
- private static class JsonEnumerator implements Enumerator {
- private final BlockingQueue<Object> queue;
- private final String holder;
- private final List<String> fields;
- private Object current;
-
- public JsonEnumerator(BlockingQueue<Object> queue, List<String> fields) {
- this.queue = queue;
- this.holder = null;
- this.fields = fields;
- }
-
- public Object current() {
- return current;
- }
-
- public void close() {
-
- }
-
- public boolean moveNext() {
- try {
- Object o = queue.take();
- if (o instanceof RunOutcome.OutcomeType) {
- switch ((RunOutcome.OutcomeType) o) {
- case SUCCESS:
- return false; // end of data
- case CANCELED:
- throw new RuntimeException("canceled");
- case FAILED:
- default:
- throw new RuntimeException("failed");
- }
- } else {
- Object o1 = parseJson((byte[]) o);
- if (holder != null) {
- o1 = ((Map<String, Object>) o1).get(holder);
- }
- if (fields == null) {
- current = o1;
- } else {
- final Map<String, Object> map = (Map<String, Object>) o1;
- if (fields.size() == 1) {
- current = map.get(fields.get(0));
- } else {
- Object[] os = new Object[fields.size()];
- for (int i = 0; i < os.length; i++) {
- os[i] = map.get(fields.get(i));
- }
- current = os;
- }
- }
- return true;
- }
- } catch (InterruptedException e) {
- Thread.interrupted();
- throw new RuntimeException(e);
- }
- }
-
- public void reset() {
- throw new UnsupportedOperationException();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java b/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
deleted file mode 100644
index 318f982..0000000
--- a/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.optiq;
-
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-
-import net.hydromatic.linq4j.AbstractEnumerable;
-import net.hydromatic.linq4j.Enumerable;
-import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.optiq.DataContext;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.jdbc.DrillHandler.FakeSchema;
-import org.apache.drill.sql.client.full.DrillFullImpl;
-import org.apache.drill.sql.client.ref.DrillRefImpl;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-/**
- * Runtime helper that executes a Drill query and converts it into an {@link Enumerable}.
- */
-public class EnumerableDrillFullEngine<E> extends AbstractEnumerable<E> implements Enumerable<E> {
- private final String plan;
- final BlockingQueue<Object> queue = new ArrayBlockingQueue<>(100);
- final DrillConfig config;
- private final List<String> fields;
- private DrillClient client;
-
- /**
- * Creates a DrillEnumerable.
- *
- * @param plan
- * Logical plan
- * @param clazz
- * Type of elements returned from enumerable
- * @param fields
- * Names of fields, or null to return the whole blob
- */
- public EnumerableDrillFullEngine(DrillConfig config, String plan, Class<E> clazz, List<String> fields,
- DrillClient client) {
- this.plan = plan;
- this.config = config;
- this.fields = fields;
- this.client = client;
- config.setSinkQueues(0, queue);
- }
-
- /**
- * Creates a DrillEnumerable from a plan represented as a string. Each record returned is a {@link JsonNode}.
- */
- public static <E> EnumerableDrillFullEngine<E> of(String plan, final List<String> fieldNames, Class<E> clazz,
- DataContext context) {
- DrillConfig config = DrillConfig.create();
- FakeSchema s = (FakeSchema) context.getSubSchema("--FAKE--");
- return new EnumerableDrillFullEngine<>(config, plan, clazz, fieldNames, s == null ? null : s.getClient());
- }
-
- @Override
- public Enumerator<E> enumerator() {
- if (client == null) {
- DrillRefImpl<E> impl = new DrillRefImpl<E>(plan, config, fields, queue);
- return impl.enumerator();
- } else {
- DrillFullImpl<E> impl = new DrillFullImpl<E>(plan, config, fields);
- return impl.enumerator(client);
- }
- }
-
-}
[29/38] git commit: DRILL-375: VarBinary hash function computing hash
for entire buffer
Posted by ja...@apache.org.
DRILL-375: VarBinary hash function computing hash for entire buffer
Signed-off-by: Jacques Nadeau <ja...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/08bc16db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/08bc16db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/08bc16db
Branch: refs/heads/master
Commit: 08bc16dbd4a6886fb9d6c103f45f67573e83a7ea
Parents: b7f9e60
Author: Steven Phillips <sp...@maprtech.com>
Authored: Tue Feb 11 15:04:13 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/08bc16db/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
index 83ae35e..3cc82b4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
@@ -167,7 +167,7 @@ public class HashFunctions {
}
public void eval() {
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(), 0);
+ out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
}
}
[16/38] git commit: DRILL-257: Move SQL parsing to server side.
Switch to Avatica based JDBC driver. Update QuerySubmitter to support SQL
queries. Update SqlAccesors to support getObject() Remove ref,
clean up SQL packages some. Various performance fixes
Posted by ja...@apache.org.
DRILL-257: Move SQL parsing to server side.
Switch to Avatica based JDBC driver.
Update QuerySubmitter to support SQL queries.
Update SqlAccesors to support getObject()
Remove ref, clean up SQL packages some.
Various performance fixes. Updating result set so first set of results must be returned before control is return to client to allow metadata to populate for aggressive tools like sqlline
Move timeout functionality to TestTools.
Update Expression materializer so that it will return a nullable int if a field is not found.
Update Project record batch to support simple wildcard queries.
Updates to move JSON record reader test to expecting VarCharVector.getObject to return a String rather than a byte[].
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b3460af8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b3460af8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b3460af8
Branch: refs/heads/master
Commit: b3460af8f3d47ce3a1ac4289998d255a9307b1ea
Parents: cdf46fd
Author: Jacques Nadeau <ja...@apache.org>
Authored: Thu Jan 30 10:56:27 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:16 2014 -0800
----------------------------------------------------------------------
common/pom.xml | 39 +-
.../org/apache/drill/common/JSONOptions.java | 48 +-
.../org/apache/drill/common/defs/OrderDef.java | 57 --
.../apache/drill/common/defs/PartitionDef.java | 58 --
.../drill/common/expression/FieldReference.java | 3 +
.../common/expression/FunctionRegistry.java | 3 +
.../drill/common/expression/SchemaPath.java | 2 +-
.../common/expression/ValueExpressions.java | 31 +-
.../drill/common/logical/LogicalPlan.java | 12 +-
.../common/logical/LogicalPlanBuilder.java | 11 +-
.../drill/common/logical/PlanProperties.java | 22 +-
.../common/logical/data/AbstractBuilder.java | 37 ++
.../logical/data/AbstractSingleBuilder.java | 41 ++
.../common/logical/data/GroupingAggregate.java | 100 ++++
.../apache/drill/common/logical/data/Join.java | 98 +++-
.../apache/drill/common/logical/data/Order.java | 82 ++-
.../drill/common/logical/data/Project.java | 53 +-
.../apache/drill/common/logical/data/Scan.java | 32 +-
.../drill/common/logical/data/ScanBuilder.java | 49 --
.../logical/data/SingleInputOperator.java | 8 +-
.../apache/drill/common/logical/data/Store.java | 39 +-
.../drill/common/logical/data/StoreBuilder.java | 49 --
.../apache/drill/common/logical/data/Union.java | 34 +-
.../data/visitors/AbstractLogicalVisitor.java | 5 +
.../logical/data/visitors/LogicalVisitor.java | 22 +-
.../org/apache/drill/common/types/Types.java | 79 +++
.../org/apache/drill/common/util/TestTools.java | 34 ++
contrib/sqlline/pom.xml | 1 -
distribution/pom.xml | 54 +-
distribution/src/assemble/bin.xml | 47 +-
distribution/src/resources/sqlline | 2 +-
exec/bufferl/pom.xml | 4 +-
exec/java-exec/pom.xml | 14 +-
.../src/main/codegen/includes/vv_imports.ftl | 3 +
.../codegen/templates/FixedValueVectors.java | 4 +
.../codegen/templates/RepeatedValueVectors.java | 5 +
.../main/codegen/templates/SqlAccessors.java | 117 ++++
.../src/main/codegen/templates/TypeHelper.java | 21 +-
.../templates/VariableLengthVectors.java | 21 +
.../org/apache/drill/exec/cache/HazelCache.java | 5 +
.../apache/drill/exec/client/DrillClient.java | 35 +-
.../drill/exec/client/QuerySubmitter.java | 112 ++--
.../apache/drill/exec/compile/MergeAdapter.java | 3 -
.../exec/expr/ExpressionTreeMaterializer.java | 13 +-
.../apache/drill/exec/ops/FragmentContext.java | 7 +
.../org/apache/drill/exec/ops/QueryContext.java | 11 +
.../apache/drill/exec/opt/BasicOptimizer.java | 22 +-
.../exec/physical/config/MergeJoinPOP.java | 13 +-
.../physical/config/MergingReceiverPOP.java | 18 +-
.../config/OrderedPartitionExchange.java | 18 +-
.../physical/config/OrderedPartitionSender.java | 25 +-
.../physical/config/SingleMergeExchange.java | 16 +-
.../apache/drill/exec/physical/config/Sort.java | 9 +-
.../exec/physical/impl/join/JoinTemplate.java | 5 +-
.../exec/physical/impl/join/MergeJoinBatch.java | 3 +-
.../physical/impl/join/MergeJoinCreator.java | 8 +-
.../impl/mergereceiver/MergingRecordBatch.java | 10 +-
.../OrderedPartitionRecordBatch.java | 23 +-
.../impl/project/ProjectRecordBatch.java | 70 ++-
.../exec/physical/impl/sort/SortBatch.java | 11 +-
.../exec/planner/logical/DrillAggregateRel.java | 108 ++++
.../planner/logical/DrillAggregateRule.java | 53 ++
.../exec/planner/logical/DrillFilterRel.java | 66 +++
.../exec/planner/logical/DrillFilterRule.java | 42 ++
.../exec/planner/logical/DrillImplementor.java | 88 +++
.../exec/planner/logical/DrillJoinRel.java | 165 ++++++
.../exec/planner/logical/DrillJoinRule.java | 57 ++
.../exec/planner/logical/DrillLimitRel.java | 73 +++
.../exec/planner/logical/DrillLimitRule.java | 59 ++
.../drill/exec/planner/logical/DrillOptiq.java | 200 +++++++
.../exec/planner/logical/DrillParseContext.java | 36 ++
.../exec/planner/logical/DrillProjectRel.java | 98 ++++
.../exec/planner/logical/DrillProjectRule.java | 46 ++
.../drill/exec/planner/logical/DrillRel.java | 33 ++
.../exec/planner/logical/DrillRuleSets.java | 92 +++
.../exec/planner/logical/DrillScanRel.java | 62 ++
.../exec/planner/logical/DrillScanRule.java | 43 ++
.../exec/planner/logical/DrillScreenRel.java | 76 +++
.../exec/planner/logical/DrillSortRel.java | 100 ++++
.../exec/planner/logical/DrillSortRule.java | 51 ++
.../exec/planner/logical/DrillStoreRel.java | 45 ++
.../drill/exec/planner/logical/DrillTable.java | 107 ++++
.../exec/planner/logical/DrillUnionRel.java | 71 +++
.../exec/planner/logical/DrillUnionRule.java | 48 ++
.../exec/planner/logical/DrillValuesRel.java | 57 ++
.../exec/planner/logical/DrillValuesRule.java | 39 ++
.../planner/logical/EnumerableDrillRule.java | 47 ++
.../drill/exec/planner/logical/ExprHelper.java | 41 ++
.../planner/logical/RelDataTypeDrillImpl.java | 142 +++++
.../exec/planner/logical/RelOptHelper.java | 44 ++
.../planner/logical/ScanFieldDeterminer.java | 217 +++++++
.../exec/planner/logical/StorageEngines.java | 74 +++
.../exec/planner/sql/DrillSchemaFactory.java | 96 ++++
.../drill/exec/planner/sql/DrillSqlWorker.java | 123 ++++
.../planner/sql/ExpandingConcurrentMap.java | 223 ++++++++
.../exec/planner/sql/FileSystemSchema.java | 117 ++++
.../exec/planner/torel/ConversionContext.java | 167 ++++++
.../apache/drill/exec/record/BatchSchema.java | 9 +
.../drill/exec/record/VectorContainer.java | 9 +-
.../org/apache/drill/exec/rpc/BasicClient.java | 6 +-
.../exec/rpc/BasicClientWithConnection.java | 5 +-
.../apache/drill/exec/rpc/RemoteConnection.java | 3 +-
.../drill/exec/rpc/user/ConnectionThrottle.java | 22 +
.../drill/exec/rpc/user/QueryResultHandler.java | 13 +-
.../apache/drill/exec/rpc/user/UserClient.java | 4 +-
.../exec/rpc/user/UserResultsListener.java | 2 +-
.../apache/drill/exec/rpc/user/UserServer.java | 2 +-
.../drill/exec/server/DrillbitContext.java | 35 ++
.../exec/store/parquet/ParquetGroupScan.java | 2 +-
.../store/parquet/ParquetSchemaProvider.java | 2 +-
.../store/parquet/ParquetStorageEngine.java | 2 +-
.../org/apache/drill/exec/vector/BitVector.java | 4 +
.../apache/drill/exec/vector/ValueVector.java | 2 +
.../vector/accessor/AbstractSqlAccessor.java | 123 ++++
.../drill/exec/vector/accessor/SqlAccessor.java | 67 +++
.../org/apache/drill/exec/work/WorkManager.java | 3 +-
.../apache/drill/exec/work/foreman/Foreman.java | 131 ++++-
.../drill/exec/work/foreman/QueryManager.java | 11 +-
.../src/main/resources/storage-engines.json | 29 +
.../physical/impl/TestSimpleFragmentRun.java | 7 +-
.../drill/exec/store/JSONRecordReaderTest.java | 22 +-
.../exec/store/ParquetRecordReaderTest.java | 22 +-
.../exec/store/TestParquetPhysicalPlan.java | 17 +-
.../store/parquet/ParquetRecordReaderTest.java | 3 +-
exec/pom.xml | 1 -
exec/ref/pom.xml | 107 ----
.../drill/exec/ref/BasicStatusHandle.java | 44 --
.../apache/drill/exec/ref/ExecRefConstants.java | 27 -
.../apache/drill/exec/ref/IteratorRegistry.java | 70 ---
.../org/apache/drill/exec/ref/ROPConverter.java | 154 -----
.../apache/drill/exec/ref/RecordIterator.java | 67 ---
.../apache/drill/exec/ref/RecordPointer.java | 37 --
.../drill/exec/ref/ReferenceInterpreter.java | 155 -----
.../drill/exec/ref/ReferenceTransform.java | 34 --
.../org/apache/drill/exec/ref/RunOutcome.java | 53 --
.../apache/drill/exec/ref/UnbackedRecord.java | 131 -----
.../drill/exec/ref/eval/BaseBasicEvaluator.java | 42 --
.../exec/ref/eval/BasicEvaluatorFactory.java | 106 ----
.../drill/exec/ref/eval/BoundarySubscriber.java | 22 -
.../drill/exec/ref/eval/EvaluatorFactory.java | 59 --
.../drill/exec/ref/eval/EvaluatorTypes.java | 58 --
.../drill/exec/ref/eval/FieldEvaluator.java | 46 --
.../apache/drill/exec/ref/eval/IfEvaluator.java | 81 ---
.../exec/ref/eval/SimpleEvaluationVisitor.java | 145 -----
.../exec/ref/eval/fn/ComparisonEvaluators.java | 137 -----
.../exec/ref/eval/fn/FunctionArguments.java | 87 ---
.../exec/ref/eval/fn/FunctionEvaluator.java | 26 -
.../ref/eval/fn/FunctionEvaluatorRegistry.java | 110 ----
.../drill/exec/ref/eval/fn/MathEvaluators.java | 106 ----
.../exec/ref/eval/fn/StringEvaluators.java | 52 --
.../fn/agg/AggregatingWrapperEvaluator.java | 54 --
.../exec/ref/eval/fn/agg/CountAggregator.java | 75 ---
.../exec/ref/eval/fn/agg/SumAggregator.java | 103 ----
.../exec/ref/exceptions/MajorException.java | 46 --
.../exec/ref/exceptions/RecordException.java | 48 --
.../exec/ref/exceptions/SetupException.java | 46 --
.../exec/ref/rops/AbstractBlockingOperator.java | 83 ---
.../apache/drill/exec/ref/rops/BaseSinkROP.java | 113 ----
.../drill/exec/ref/rops/BoundaryListener.java | 25 -
.../exec/ref/rops/CollapsingAggregateROP.java | 269 ---------
.../apache/drill/exec/ref/rops/ConstantROP.java | 109 ----
.../apache/drill/exec/ref/rops/DataWriter.java | 49 --
.../apache/drill/exec/ref/rops/FilterROP.java | 81 ---
.../apache/drill/exec/ref/rops/FlattenROP.java | 140 -----
.../org/apache/drill/exec/ref/rops/JoinROP.java | 315 -----------
.../apache/drill/exec/ref/rops/LimitROP.java | 87 ---
.../apache/drill/exec/ref/rops/OrderROP.java | 201 -------
.../apache/drill/exec/ref/rops/ProjectROP.java | 95 ----
.../drill/exec/ref/rops/ProxySimpleRecord.java | 79 ---
.../org/apache/drill/exec/ref/rops/ROP.java | 28 -
.../org/apache/drill/exec/ref/rops/ROPBase.java | 61 --
.../drill/exec/ref/rops/ReferenceOperator.java | 26 -
.../exec/ref/rops/ReferenceOperatorBase.java | 39 --
.../org/apache/drill/exec/ref/rops/ScanROP.java | 70 ---
.../apache/drill/exec/ref/rops/SegmentROP.java | 118 ----
.../drill/exec/ref/rops/SingleInputROPBase.java | 49 --
.../org/apache/drill/exec/ref/rops/SinkROP.java | 30 -
.../apache/drill/exec/ref/rops/StoreROP.java | 63 ---
.../drill/exec/ref/rops/TransformROP.java | 89 ---
.../apache/drill/exec/ref/rops/UnionROP.java | 107 ----
.../drill/exec/ref/rops/WindowFrameROP.java | 321 -----------
.../apache/drill/exec/ref/rse/ClasspathRSE.java | 93 ---
.../apache/drill/exec/ref/rse/ConsoleRSE.java | 66 ---
.../drill/exec/ref/rse/FileSystemRSE.java | 161 ------
.../drill/exec/ref/rse/JSONDataWriter.java | 142 -----
.../drill/exec/ref/rse/JSONRecordReader.java | 183 ------
.../drill/exec/ref/rse/OutputStreamWriter.java | 78 ---
.../org/apache/drill/exec/ref/rse/QueueRSE.java | 146 -----
.../org/apache/drill/exec/ref/rse/RSEBase.java | 71 ---
.../apache/drill/exec/ref/rse/RSERegistry.java | 87 ---
.../apache/drill/exec/ref/rse/RecordReader.java | 28 -
.../drill/exec/ref/rse/RecordRecorder.java | 32 --
.../exec/ref/rse/ReferenceStorageEngine.java | 45 --
.../drill/exec/ref/util/BatchBreaker.java | 46 --
.../apache/drill/exec/ref/util/ByteRange.java | 34 --
.../drill/exec/ref/values/BaseArrayValue.java | 86 ---
.../drill/exec/ref/values/BaseDataValue.java | 82 ---
.../drill/exec/ref/values/BaseMapValue.java | 111 ----
.../drill/exec/ref/values/BooleanValue.java | 22 -
.../drill/exec/ref/values/BytesValue.java | 28 -
.../drill/exec/ref/values/ComparableValue.java | 38 --
.../drill/exec/ref/values/ContainerValue.java | 25 -
.../apache/drill/exec/ref/values/DataValue.java | 45 --
.../drill/exec/ref/values/DataValueSet.java | 80 ---
.../drill/exec/ref/values/NumericValue.java | 175 ------
.../drill/exec/ref/values/ScalarValues.java | 560 -------------------
.../drill/exec/ref/values/SimpleArrayValue.java | 147 -----
.../drill/exec/ref/values/SimpleMapValue.java | 130 -----
.../drill/exec/ref/values/StringValue.java | 22 -
.../drill/exec/ref/values/ValueReader.java | 62 --
.../drill/exec/ref/values/ValueUtils.java | 80 ---
exec/ref/src/main/resources/drill-module.conf | 4 -
exec/ref/src/main/resources/logback-test.xml | 43 --
.../apache/drill/exec/ref/RSERegistryTest.java | 40 --
.../apache/drill/exec/ref/RunSimplePlan.java | 77 ---
.../org/apache/drill/exec/ref/TestUtils.java | 122 ----
.../exec/ref/rops/CollapsingAggregateTest.java | 51 --
.../drill/exec/ref/rops/ConstantROPTest.java | 111 ----
.../apache/drill/exec/ref/rops/JoinROPTest.java | 39 --
.../drill/exec/ref/rops/LimitROPTest.java | 84 ---
.../drill/exec/ref/rops/OrderROPTest.java | 61 --
.../drill/exec/ref/rops/UnionROPTest.java | 63 ---
.../drill/exec/ref/rops/WindowFrameROPTest.java | 259 ---------
exec/ref/src/test/resources/collapse/test1.json | 73 ---
exec/ref/src/test/resources/constant.json | 4 -
exec/ref/src/test/resources/constant2.json | 42 --
exec/ref/src/test/resources/departments.json | 16 -
exec/ref/src/test/resources/donuts.json | 132 -----
exec/ref/src/test/resources/employees.json | 23 -
.../src/test/resources/join/departments.json | 16 -
exec/ref/src/test/resources/join/employees.json | 24 -
exec/ref/src/test/resources/join/full_join.json | 72 ---
exec/ref/src/test/resources/join/left_join.json | 71 ---
.../src/test/resources/join/simple_join.json | 71 ---
exec/ref/src/test/resources/logback.xml | 56 --
.../src/test/resources/order/nulls-first.json | 79 ---
.../src/test/resources/order/nulls-last.json | 79 ---
exec/ref/src/test/resources/simple_join.json | 60 --
exec/ref/src/test/resources/simple_plan.json | 94 ----
.../test/resources/simple_plan_flattened.json | 54 --
exec/ref/src/test/resources/union/distinct.json | 82 ---
.../src/test/resources/union/nondistinct.json | 82 ---
pom.xml | 26 +-
protocol/src/main/protobuf/User.proto | 8 +
sqlparser/pom.xml | 26 +-
.../drill/jdbc/AvaticaDrillSqlAccessor.java | 214 +++++++
.../java/org/apache/drill/jdbc/BasicList.java | 136 +++++
.../org/apache/drill/jdbc/ConnectionConfig.java | 41 ++
.../apache/drill/jdbc/DrillAccessorList.java | 58 ++
.../drill/jdbc/DrillColumnMetaDataList.java | 81 +++
.../org/apache/drill/jdbc/DrillConnection.java | 41 ++
.../apache/drill/jdbc/DrillConnectionImpl.java | 166 ++++++
.../java/org/apache/drill/jdbc/DrillCursor.java | 127 +++++
.../apache/drill/jdbc/DrillDriverVersion.java | 42 --
.../org/apache/drill/jdbc/DrillFactory.java | 57 ++
.../org/apache/drill/jdbc/DrillHandler.java | 190 +------
.../apache/drill/jdbc/DrillJdbc40Factory.java | 29 +
.../apache/drill/jdbc/DrillJdbc41Factory.java | 186 ++++++
.../apache/drill/jdbc/DrillPrepareResult.java | 52 ++
.../drill/jdbc/DrillPreparedStatement.java | 50 ++
.../apache/drill/jdbc/DrillRemoteStatement.java | 22 +
.../org/apache/drill/jdbc/DrillResultSet.java | 196 +++++++
.../org/apache/drill/jdbc/DrillStatement.java | 39 ++
.../drill/jdbc/DrillStatementRegistry.java | 28 +
.../java/org/apache/drill/jdbc/DrillTable.java | 225 --------
.../main/java/org/apache/drill/jdbc/Driver.java | 72 ++-
.../drill/jdbc/GlobalServiceSetReference.java | 29 +
.../java/org/apache/drill/jdbc/MetaImpl.java | 180 ++++++
.../java/org/apache/drill/jdbc/RefDriver.java | 71 ---
.../apache/drill/jdbc/SchemaChangeListener.java | 24 +
.../apache/drill/jdbc/SqlTimeoutException.java | 34 ++
.../org/apache/drill/jdbc/StorageEngines.java | 74 ---
.../org/apache/drill/jdbc/package-info.java | 22 +
.../apache/drill/optiq/DrillAggregateRel.java | 119 ----
.../apache/drill/optiq/DrillAggregateRule.java | 53 --
.../org/apache/drill/optiq/DrillFilterRel.java | 60 --
.../org/apache/drill/optiq/DrillFilterRule.java | 42 --
.../apache/drill/optiq/DrillImplementor.java | 146 -----
.../org/apache/drill/optiq/DrillJoinRel.java | 158 ------
.../org/apache/drill/optiq/DrillJoinRule.java | 57 --
.../org/apache/drill/optiq/DrillLimitRel.java | 60 --
.../org/apache/drill/optiq/DrillLimitRule.java | 59 --
.../java/org/apache/drill/optiq/DrillOptiq.java | 180 ------
.../apache/drill/optiq/DrillPrepareImpl.java | 56 --
.../org/apache/drill/optiq/DrillProjectRel.java | 83 ---
.../apache/drill/optiq/DrillProjectRule.java | 46 --
.../java/org/apache/drill/optiq/DrillRel.java | 32 --
.../java/org/apache/drill/optiq/DrillScan.java | 60 --
.../org/apache/drill/optiq/DrillSortRel.java | 98 ----
.../org/apache/drill/optiq/DrillSortRule.java | 51 --
.../org/apache/drill/optiq/DrillUnionRel.java | 75 ---
.../org/apache/drill/optiq/DrillUnionRule.java | 48 --
.../org/apache/drill/optiq/DrillValuesRel.java | 53 --
.../org/apache/drill/optiq/DrillValuesRule.java | 39 --
.../org/apache/drill/optiq/EnumerableDrill.java | 271 ---------
.../drill/optiq/EnumerableDrillFullEngine.java | 87 ---
.../apache/drill/optiq/EnumerableDrillRel.java | 130 -----
.../apache/drill/optiq/EnumerableDrillRule.java | 53 --
.../drill/sql/client/full/BatchListener.java | 79 ---
.../drill/sql/client/full/BatchLoaderMap.java | 233 --------
.../drill/sql/client/full/DrillFullImpl.java | 76 ---
.../sql/client/full/DrillMutableSchema.java | 91 ---
.../drill/sql/client/full/FileSystemSchema.java | 154 -----
.../drill/sql/client/full/JsonHelper.java | 57 --
.../drill/sql/client/full/ResultEnumerator.java | 72 ---
.../drill/sql/client/ref/DrillRefImpl.java | 271 ---------
.../src/main/resources/storage-engines.json | 29 -
.../apache/drill/jdbc/test/FullEngineTest.java | 87 ---
.../org/apache/drill/jdbc/test/JdbcAssert.java | 11 +-
.../org/apache/drill/jdbc/test/JdbcTest.java | 50 +-
.../apache/drill/jdbc/test/TestJdbcQuery.java | 108 ++++
311 files changed, 6944 insertions(+), 14542 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 7b3c697..d76f7a1 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -36,6 +36,19 @@
</dependency>
<dependency>
+ <!-- add as provided scope so that we can compile TestTools. Should only be ever used in a test scenario where someone else is bringing JUnit in. -->
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${dep.junit.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>net.hydromatic</groupId>
+ <artifactId>optiq-core</artifactId>
+ </dependency>
+
+
+ <dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.5.0</version>
@@ -68,7 +81,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>2.1.1</version>
+ <version>2.2.0</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
@@ -84,7 +97,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.1.1</version>
+ <version>2.2.0</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
@@ -99,12 +112,12 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.15</version>
- <configuration>
- <useSystemClassLoader>false</useSystemClassLoader>
- </configuration>
- </plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.15</version>
+ <configuration>
+ <useSystemClassLoader>false</useSystemClassLoader>
+ </configuration>
+ </plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<executions>
@@ -131,7 +144,7 @@
</execution>
</executions>
</plugin>
-
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -144,13 +157,13 @@
</goals>
<configuration>
<artifactSet>
- <includes>
- <include>com.google.protobuf:protobuf-java</include>
- </includes>
+ <includes>
+ <include>com.google.protobuf:protobuf-java</include>
+ </includes>
</artifactSet>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>rebuffed</shadedClassifierName>
-
+
<relocations>
<relocation>
<pattern>com.google.protobuf</pattern>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/JSONOptions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/JSONOptions.java b/common/src/main/java/org/apache/drill/common/JSONOptions.java
index 23de124..c1f9a85 100644
--- a/common/src/main/java/org/apache/drill/common/JSONOptions.java
+++ b/common/src/main/java/org/apache/drill/common/JSONOptions.java
@@ -18,12 +18,8 @@
package org.apache.drill.common;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.lang.reflect.ParameterizedType;
-import com.fasterxml.jackson.core.*;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.drill.common.JSONOptions.De;
import org.apache.drill.common.JSONOptions.Se;
import org.apache.drill.common.config.DrillConfig;
@@ -31,8 +27,16 @@ import org.apache.drill.common.exceptions.LogicalPlanParsingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonLocation;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.TreeNode;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@@ -47,14 +51,28 @@ public class JSONOptions {
private JsonNode root;
private JsonLocation location;
+ private Object opaque;
+
+ public JSONOptions(Object opaque){
+ this.opaque = opaque;
+ }
public JSONOptions(JsonNode n, JsonLocation location){
this.root = n;
this.location = location;
}
+ @SuppressWarnings("unchecked")
public <T> T getWith(DrillConfig config, Class<T> c){
try {
+ if(opaque != null){
+ if (opaque.getClass().equals(c)){
+ return (T) opaque;
+ }else{
+ throw new IllegalArgumentException(String.format("Attmpted to retrieve a option with type of %s. However, the JSON options carried an opaque value of type %s.", c.getName(), opaque.getClass().getName()));
+ }
+ }
+
//logger.debug("Read tree {}", root);
return config.getMapper().treeToValue(root, c);
} catch (JsonProcessingException e) {
@@ -63,11 +81,20 @@ public class JSONOptions {
}
public <T> T getListWith(DrillConfig config, TypeReference<T> t) throws IOException {
- ObjectMapper mapper = config.getMapper();
- return mapper.treeAsTokens(root).readValueAs(t);
+ return getListWith(config.getMapper(), t);
}
+ @SuppressWarnings("unchecked")
public <T> T getListWith(ObjectMapper mapper, TypeReference<T> t) throws IOException {
+ if(opaque != null){
+ ParameterizedType pt = (ParameterizedType) t.getType();
+ if ( pt.getRawType().equals(opaque.getClass())){
+ return (T) opaque;
+ }else{
+ throw new IOException(String.format("Attmpted to retrieve a list with type of %s. However, the JSON options carried an opaque value of type %s.", t.getType(), opaque.getClass().getName()));
+ }
+ }
+ if(root == null) return null;
return mapper.treeAsTokens(root).readValueAs(t);
}
@@ -111,7 +138,12 @@ public class JSONOptions {
@Override
public void serialize(JSONOptions value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
JsonGenerationException {
- jgen.writeTree(value.root);
+ if(value.opaque != null){
+ jgen.writeObject(value.opaque);
+ }else{
+ jgen.writeTree(value.root);
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/defs/OrderDef.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/defs/OrderDef.java b/common/src/main/java/org/apache/drill/common/defs/OrderDef.java
deleted file mode 100644
index e33de94..0000000
--- a/common/src/main/java/org/apache/drill/common/defs/OrderDef.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.defs;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.Order.Direction;
-import org.apache.drill.common.logical.data.Order.Ordering;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class OrderDef {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OrderDef.class);
-
- private final Direction direction;
- private final LogicalExpression expr;
-
- @JsonCreator
- public OrderDef(@JsonProperty("order") Direction direction, @JsonProperty("expr") LogicalExpression expr) {
- this.expr = expr;
- // default to ascending unless desc is provided.
- this.direction = direction == null ? Direction.ASC : direction;
- }
-
- @JsonIgnore
- public Direction getDirection() {
- return direction;
- }
-
- public LogicalExpression getExpr() {
- return expr;
- }
-
- public String getOrder() {
- return direction.description;
- }
-
- public static OrderDef create(Ordering o){
- return new OrderDef(o.getDirection(), o.getExpr());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/defs/PartitionDef.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/defs/PartitionDef.java b/common/src/main/java/org/apache/drill/common/defs/PartitionDef.java
deleted file mode 100644
index f763fd2..0000000
--- a/common/src/main/java/org/apache/drill/common/defs/PartitionDef.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.defs;
-
-import org.apache.drill.common.expression.LogicalExpression;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class PartitionDef {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PartitionDef.class);
-
- private final PartitionType partitionType;
- private final LogicalExpression expr;
- private final LogicalExpression[] starts;
-
- @JsonCreator
- public PartitionDef(@JsonProperty("mode") PartitionType partitionType, @JsonProperty("expr") LogicalExpression expr, @JsonProperty("starts") LogicalExpression[] starts) {
- this.partitionType = partitionType;
- this.expr = expr;
- this.starts = starts;
- }
-
- @JsonProperty("mode")
- public PartitionType getPartitionType() {
- return partitionType;
- }
-
- @JsonProperty("expr")
- public LogicalExpression getExpr() {
- return expr;
- }
-
- @JsonProperty("starts")
- public LogicalExpression[] getStarts() {
- return starts;
- }
-
-
- public static enum PartitionType{
- DUPLICATE, RANDOM, HASH, RANGE;
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
index f01d525..358a865 100644
--- a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
+++ b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
@@ -43,6 +43,9 @@ public class FieldReference extends SchemaPath {
super(sp);
}
+ public FieldReference(CharSequence value){
+ super(value, ExpressionPosition.UNKNOWN);
+ }
public FieldReference(CharSequence value, ExpressionPosition pos) {
super(value, pos);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java b/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java
index 8ffc07a..7cb386a 100644
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java
+++ b/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java
@@ -87,6 +87,9 @@ public class FunctionRegistry {
return new FunctionCall(castFuncDef, newArgs, ep);
}
+ public LogicalExpression createExpression(String functionName, List<LogicalExpression> args){
+ return createExpression(functionName, ExpressionPosition.UNKNOWN, args);
+ }
public LogicalExpression createExpression(String functionName, ExpressionPosition ep, List<LogicalExpression> args){
FunctionDefinition d = funcMap.get(functionName);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 31bac80..7c2dce6 100644
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -112,7 +112,7 @@ public class SchemaPath extends LogicalExpressionBase {
return visitor.visitSchemaPath(this, value);
}
- public SchemaPath getChild(String[] childPaths){
+ public SchemaPath getChild(String... childPaths){
return new SchemaPath(this, childPaths);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
index 9137356..49f3773 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -29,6 +29,30 @@ import com.google.common.collect.Iterators;
public class ValueExpressions {
+ public static LogicalExpression getBigInt(long l){
+ return new LongExpression(l);
+ }
+
+ public static LogicalExpression getInt(int i){
+ return new LongExpression(i);
+ }
+
+ public static LogicalExpression getFloat8(double d){
+ return new DoubleExpression(d, ExpressionPosition.UNKNOWN);
+ }
+ public static LogicalExpression getFloat4(float f){
+ return new DoubleExpression(f, ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getBit(boolean b){
+ return new BooleanExpression(Boolean.toString(b), ExpressionPosition.UNKNOWN);
+ }
+
+ public static LogicalExpression getChar(String s){
+ return new QuotedString(s, ExpressionPosition.UNKNOWN);
+ }
+
+
public static LogicalExpression getNumericExpression(String s, ExpressionPosition ep) {
try {
long l = Long.parseLong(s);
@@ -74,6 +98,7 @@ public class ValueExpressions {
super(value, pos);
}
+
@Override
protected Boolean parseValue(String s) {
return Boolean.parseBoolean(s);
@@ -132,7 +157,11 @@ public class ValueExpressions {
private long l;
- public LongExpression(long l, ExpressionPosition pos) {
+ public LongExpression(long l) {
+ this(l, ExpressionPosition.UNKNOWN);
+ }
+
+ public LongExpression(long l, ExpressionPosition pos) {
super(pos);
this.l = l;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
index 8be44e9..a9db251 100644
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
+++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
@@ -46,7 +46,8 @@ public class LogicalPlan {
private final PlanProperties properties;
private final Map<String, StorageEngineConfig> storageEngineMap;
private final Graph<LogicalOperator, SinkOperator, SourceOperator> graph;
-
+
+
@JsonCreator
public LogicalPlan(@JsonProperty("head") PlanProperties head,
@JsonProperty("storage") Map<String, StorageEngineConfig> storageEngineMap,
@@ -84,6 +85,15 @@ public class LogicalPlan {
return config.getMapper().writeValueAsString(this);
}
+ public String toJsonStringSafe(DrillConfig config){
+ try{
+ return toJsonString(config);
+ }catch(JsonProcessingException e){
+ logger.error("Failure while trying to get JSON representation of plan.", e);
+ return "Unable to generate plan.";
+ }
+ }
+
/** Parses a logical plan. */
public static LogicalPlan parse(DrillConfig config, String planString) {
ObjectMapper mapper = config.getMapper();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
index 675a5c8..369cd58 100644
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
+++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
@@ -17,9 +17,12 @@
******************************************************************************/
package org.apache.drill.common.logical;
+import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
+import org.apache.drill.common.logical.PlanProperties.PlanType;
+import org.apache.drill.common.logical.data.LogicalOperator;
+
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import org.apache.drill.common.logical.data.LogicalOperator;
/**
* A programmatic builder for logical plans.
@@ -33,6 +36,12 @@ public class LogicalPlanBuilder {
this.planProperties = planProperties;
return this;
}
+
+ public LogicalPlanBuilder planProperties(PlanType type, int version, String generatorType, String generatorInfo, ResultMode mode){
+ this.planProperties = PlanProperties.builder().generator(generatorType, generatorInfo).type(type).version(version).resultMode(mode).build();
+ return this;
+ }
+
public LogicalPlanBuilder addStorageEngine(String name, StorageEngineConfig config) {
this.storageEngines.put(name, config);
return this;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java b/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java
index 5980bf8..606f2df 100644
--- a/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java
+++ b/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java
@@ -17,6 +17,8 @@
******************************************************************************/
package org.apache.drill.common.logical;
+import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
+
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -30,11 +32,16 @@ public class PlanProperties {
public PlanType type;
public int version;
public Generator generator;
+ public ResultMode resultMode;
- @JsonInclude(Include.NON_NULL)
+// @JsonInclude(Include.NON_NULL)
public static class Generator{
public String type;
public String info;
+
+ public static enum ResultMode{
+ EXEC, LOGICAL, PHYSICAL;
+ }
private Generator(@JsonProperty("type") String type, @JsonProperty("info") String info) {
this.type = type;
@@ -44,10 +51,13 @@ public class PlanProperties {
private PlanProperties(@JsonProperty("version") int version,
@JsonProperty("generator") Generator generator,
- @JsonProperty("type") PlanType type) {
+ @JsonProperty("type") PlanType type,
+ @JsonProperty("mode") ResultMode resultMode
+ ) {
this.version = version;
this.generator = generator;
this.type = type;
+ this.resultMode = resultMode == null ? ResultMode.EXEC : resultMode;
}
public static PlanPropertiesBuilder builder() {
@@ -58,6 +68,7 @@ public class PlanProperties {
private int version;
private Generator generator;
private PlanType type;
+ private ResultMode mode = ResultMode.EXEC;
public PlanPropertiesBuilder type(PlanType type) {
this.type = type;
@@ -73,6 +84,11 @@ public class PlanProperties {
this.generator = new Generator(type, info);
return this;
}
+
+ public PlanPropertiesBuilder resultMode(ResultMode mode){
+ this.mode = mode;
+ return this;
+ }
public PlanPropertiesBuilder generator(Generator generator) {
this.generator = generator;
@@ -80,7 +96,7 @@ public class PlanProperties {
}
public PlanProperties build() {
- return new PlanProperties(version, generator, type);
+ return new PlanProperties(version, generator, type, mode);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java b/common/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
new file mode 100644
index 0000000..69cb4eb
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/data/AbstractBuilder.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.List;
+
+import org.apache.drill.common.expression.LogicalExpression;
+
+public abstract class AbstractBuilder<T extends LogicalOperator> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBuilder.class);
+
+ public abstract T build();
+
+ protected LogicalExpression[] aL(List<LogicalExpression> exprs){
+ return exprs.toArray(new LogicalExpression[exprs.size()]);
+ }
+
+ protected NamedExpression[] aN(List<NamedExpression> exprs){
+ return exprs.toArray(new NamedExpression[exprs.size()]);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java b/common/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
new file mode 100644
index 0000000..e733fdd
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/data/AbstractSingleBuilder.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import com.google.common.base.Preconditions;
+
+public abstract class AbstractSingleBuilder<T extends SingleInputOperator, X extends AbstractSingleBuilder<T, X>> extends AbstractBuilder<T> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSingleBuilder.class);
+
+ private LogicalOperator input;
+
+ public final T build(){
+ Preconditions.checkNotNull(input);
+ T out = internalBuild();
+ out.setInput(input);
+ return out;
+ }
+
+ public X setInput(LogicalOperator input){
+ this.input = input;
+ return (X) this;
+ }
+
+ public abstract T internalBuild();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java b/common/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
new file mode 100644
index 0000000..4aabfba
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/data/GroupingAggregate.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical.data;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+
+@JsonTypeName("groupingaggregate")
+public class GroupingAggregate extends SingleInputOperator{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GroupingAggregate.class);
+
+ private final NamedExpression[] keys;
+ private final NamedExpression[] exprs;
+
+ public GroupingAggregate(@JsonProperty("keys") NamedExpression[] keys, @JsonProperty("exprs") NamedExpression[] exprs) {
+ super();
+ this.keys = keys;
+ this.exprs = exprs;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitGroupingAggregate(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(getInput());
+ }
+
+ public static Builder builder(){
+ return new Builder();
+ }
+
+ public NamedExpression[] getKeys(){
+ return keys;
+ }
+
+ public NamedExpression[] getExprs(){
+ return exprs;
+ }
+
+ public static class Builder extends AbstractSingleBuilder<GroupingAggregate, Builder>{
+ private List<NamedExpression> keys = Lists.newArrayList();
+ private List<NamedExpression> exprs = Lists.newArrayList();
+
+ public Builder addKey(FieldReference ref, LogicalExpression expr){
+ keys.add(new NamedExpression(expr, ref));
+ return this;
+ }
+
+ public Builder addKey(NamedExpression expr){
+ keys.add(expr);
+ return this;
+ }
+
+ public Builder addExpr(NamedExpression expr){
+ exprs.add(expr);
+ return this;
+ }
+
+ public Builder addExpr(FieldReference ref, LogicalExpression expr){
+ exprs.add(new NamedExpression(expr, ref));
+ return this;
+ }
+
+ public GroupingAggregate internalBuild(){
+ GroupingAggregate ga = new GroupingAggregate(aN(keys), aN(exprs));
+ return ga;
+ }
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/Join.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Join.java b/common/src/main/java/org/apache/drill/common/logical/data/Join.java
index 06f276b..3e68b09 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Join.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Join.java
@@ -17,47 +17,55 @@
*/
package org.apache.drill.common.logical.data;
-import com.google.common.collect.Iterators;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.drill.common.exceptions.ExpressionParsingException;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+import org.eigenbase.rel.JoinRelType;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
@JsonTypeName("join")
public class Join extends LogicalOperatorBase {
private final LogicalOperator left;
private final LogicalOperator right;
- private final JoinType type;
+ private final JoinRelType type;
private final JoinCondition[] conditions;
- public static enum JoinType{
- LEFT, INNER, OUTER, RIGHT;
-
- public static JoinType resolve(String val){
- for(JoinType jt : JoinType.values()){
- if(jt.name().equalsIgnoreCase(val)) return jt;
- }
- throw new ExpressionParsingException(String.format("Unable to determine join type for value '%s'.", val));
+ public static JoinRelType resolve(String val){
+ for (JoinRelType jt : JoinRelType.values()) {
+ if (jt.name().equalsIgnoreCase(val))
+ return jt;
}
+ throw new ExpressionParsingException(String.format("Unable to determine join type for value '%s'.", val));
}
@JsonCreator
- public Join(@JsonProperty("left") LogicalOperator left, @JsonProperty("right") LogicalOperator right, @JsonProperty("conditions") JoinCondition[] conditions, @JsonProperty("type") String type) {
+ public Join(@JsonProperty("left") LogicalOperator left, @JsonProperty("right") LogicalOperator right,
+ @JsonProperty("conditions") JoinCondition[] conditions, @JsonProperty("type") String type) {
+ this(left, right, conditions, resolve(type));
+ }
+
+
+ public Join(LogicalOperator left, @JsonProperty("right") LogicalOperator right, JoinCondition[] conditions, JoinRelType type) {
super();
this.conditions = conditions;
this.left = left;
this.right = right;
left.registerAsSubscriber(this);
right.registerAsSubscriber(this);
- this.type = JoinType.resolve(type);
+ this.type = type;
}
-
+
public LogicalOperator getLeft() {
return left;
}
@@ -71,21 +79,61 @@ public class Join extends LogicalOperatorBase {
}
@JsonIgnore
- public JoinType getJointType(){
+ public JoinRelType getJoinType() {
return type;
}
-
- public String getType(){
+
+ public String getType() {
return type.name();
}
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitJoin(this, value);
- }
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitJoin(this, value);
+ }
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.forArray(getLeft(), getRight());
+ }
+
+ public static Builder builder(){
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractBuilder<Join>{
+ private LogicalOperator left;
+ private LogicalOperator right;
+ private JoinRelType type;
+ private List<JoinCondition> conditions = Lists.newArrayList();
+
+ public Builder type(JoinRelType type){
+ this.type = type;
+ return this;
+ }
+
+ public Builder left(LogicalOperator left){
+ this.left = left;
+ return this;
+ }
+ public Builder right(LogicalOperator right){
+ this.right = right;
+ return this;
+ }
+
+ public Builder addCondition(String relationship, LogicalExpression left, LogicalExpression right){
+ conditions.add(new JoinCondition(relationship, left, right));
+ return this;
+ }
+
+
@Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.forArray(getLeft(), getRight());
+ public Join build() {
+ Preconditions.checkNotNull(left);
+ Preconditions.checkNotNull(right);
+ Preconditions.checkNotNull(type);
+ return new Join(left, right, conditions.toArray(new JoinCondition[conditions.size()]), type);
}
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/Order.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Order.java b/common/src/main/java/org/apache/drill/common/logical/data/Order.java
index 7b77218..3c864b0 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Order.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Order.java
@@ -17,17 +17,22 @@
*/
package org.apache.drill.common.logical.data;
-import com.google.common.collect.Iterators;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+import org.eigenbase.rel.RelFieldCollation;
+import org.eigenbase.rel.RelFieldCollation.Direction;
+import org.eigenbase.rel.RelFieldCollation.NullDirection;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
@JsonTypeName("order")
public class Order extends SingleInputOperator {
@@ -62,16 +67,25 @@ public class Order extends SingleInputOperator {
public static class Ordering {
- private final Direction direction;
+ private final RelFieldCollation.Direction direction;
private final LogicalExpression expr;
- private final NullCollation nulls;
+ private final RelFieldCollation.NullDirection nulls;
@JsonCreator
- public Ordering(@JsonProperty("order") String strOrder, @JsonProperty("expr") LogicalExpression expr, @JsonProperty("nullCollation") String nullCollation) {
+ public Ordering(@JsonProperty("order") String strOrder, @JsonProperty("expr") LogicalExpression expr, @JsonProperty("nullDirection") String nullCollation) {
this.expr = expr;
- this.nulls = NullCollation.NULLS_LAST.description.equals(nullCollation) ? NullCollation.NULLS_LAST : NullCollation.NULLS_FIRST; // default first
- this.direction = Direction.DESC.description.equalsIgnoreCase(strOrder) ? Direction.DESC : Direction.ASC; // default asc
-
+ this.nulls = NullDirection.LAST.name().equalsIgnoreCase(nullCollation) ? NullDirection.LAST : NullDirection.FIRST; // default first
+ this.direction = Order.getDirectionFromString(strOrder);
+ }
+
+ public Ordering(Direction direction, LogicalExpression e, NullDirection nullCollation) {
+ this.expr = e;
+ this.nulls = nullCollation;
+ this.direction = direction;
+ }
+
+ public Ordering(Direction direction, LogicalExpression e) {
+ this(direction, e, NullDirection.FIRST);
}
@JsonIgnore
@@ -84,34 +98,52 @@ public class Order extends SingleInputOperator {
}
public String getOrder() {
- return direction.description;
+
+ switch(direction){
+ case Descending: return "DESC";
+ default: return "ASC";
+ }
}
- public NullCollation getNullCollation() {
+ public NullDirection getNullDirection() {
return nulls;
}
}
- public static enum NullCollation {
- NULLS_FIRST("first"), NULLS_LAST("last");
+
+ public static Builder builder(){
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractSingleBuilder<Order, Builder>{
+ private List<Ordering> orderings = Lists.newArrayList();
+ private FieldReference within;
- public final String description;
-
- NullCollation(String d) {
- description = d;
+ public Builder setWithin(FieldReference within){
+ this.within = within;
+ return this;
+ }
+
+ public Builder addOrdering(Direction direction, LogicalExpression e, NullDirection collation){
+ orderings.add(new Ordering(direction, e, collation));
+ return this;
}
- }
-
- public static enum Direction {
- ASC("ASC"), DESC("DESC");
- public final String description;
- Direction(String d) {
- description = d;
+ @Override
+ public Order internalBuild() {
+ return new Order(within, orderings.toArray(new Ordering[orderings.size()]));
}
+
+
}
+ public static Direction getDirectionFromString(String direction){
+ return "DESC".equalsIgnoreCase(direction) ? Direction.Descending : Direction.Ascending;
+ }
+ public static String getStringFromDirection(Direction direction){
+ return direction == Direction.Descending ? "DESC" : "ASC";
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/Project.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Project.java b/common/src/main/java/org/apache/drill/common/logical/data/Project.java
index 5c0716d..645dc32 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Project.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Project.java
@@ -17,16 +17,20 @@
*/
package org.apache.drill.common.logical.data;
-import com.google.common.collect.Iterators;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.drill.common.exceptions.ExpressionParsingException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
@JsonTypeName("project")
public class Project extends SingleInputOperator {
@@ -36,13 +40,18 @@ public class Project extends SingleInputOperator {
@JsonCreator
public Project(@JsonProperty("projections") NamedExpression[] selections) {
this.selections = selections;
- if(selections == null || selections.length == 0) throw new ExpressionParsingException("Project did not provide any projection selections. At least one projection must be provided.");
+ if (selections == null || selections.length == 0)
+ throw new ExpressionParsingException(
+ "Project did not provide any projection selections. At least one projection must be provided.");
for (int i = 0; i < selections.length; i++) {
PathSegment segment = selections[i].getRef().getRootSegment();
CharSequence path = segment.getNameSegment().getPath();
if (!segment.isNamed() || !path.equals("output"))
- throw new ExpressionParsingException(String.format(
- "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]", path, segment.isNamed()));
+ throw new ExpressionParsingException(
+ String
+ .format(
+ "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]",
+ path, segment.isNamed()));
}
}
@@ -52,15 +61,33 @@ public class Project extends SingleInputOperator {
return selections;
}
- @Override
- public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
- return logicalVisitor.visitProject(this, value);
+ @Override
+ public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitProject(this, value);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractSingleBuilder<Project, Builder> {
+
+ private List<NamedExpression> exprs = Lists.newArrayList();
+
+ public Builder addExpr(NamedExpression expr) {
+ exprs.add(expr);
+ return this;
}
- @Override
- public Iterator<LogicalOperator> iterator() {
- return Iterators.singletonIterator(getInput());
+ public Builder addExpr(FieldReference ref, LogicalExpression expr) {
+ exprs.add(new NamedExpression(expr, ref));
+ return this;
}
+ @Override
+ public Project internalBuild() {
+ return new Project(aN(exprs));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/Scan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Scan.java b/common/src/main/java/org/apache/drill/common/logical/data/Scan.java
index 294c0e8..c36c2d1 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Scan.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Scan.java
@@ -19,11 +19,11 @@ package org.apache.drill.common.logical.data;
import org.apache.drill.common.JSONOptions;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
@JsonTypeName("scan")
@@ -59,8 +59,34 @@ public class Scan extends SourceOperator{
return logicalVisitor.visitScan(this, value);
}
- public static ScanBuilder builder() {
- return new ScanBuilder();
+ public static Builder builder() {
+ return new Builder();
+ }
+
+
+ public static class Builder extends AbstractBuilder<Scan>{
+ private String storageEngine;
+ private JSONOptions selection;
+ private FieldReference outputReference;
+
+ public Builder storageEngine(String storageEngine) {
+ this.storageEngine = storageEngine;
+ return this;
+ }
+
+ public Builder selection(JSONOptions selection) {
+ this.selection = selection;
+ return this;
+ }
+
+ public Builder outputReference(FieldReference outputReference) {
+ this.outputReference = outputReference;
+ return this;
+ }
+
+ public Scan build() {
+ return new Scan(storageEngine, selection, outputReference);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/ScanBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/ScanBuilder.java b/common/src/main/java/org/apache/drill/common/logical/data/ScanBuilder.java
deleted file mode 100644
index 3eaa91f..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/ScanBuilder.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.common.logical.data;
-
-import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.expression.FieldReference;
-
-/**
- * Builder for the scan operator
- */
-public class ScanBuilder {
- private String storageEngine;
- private JSONOptions selection;
- private FieldReference outputReference;
-
- public ScanBuilder storageEngine(String storageEngine) {
- this.storageEngine = storageEngine;
- return this;
- }
-
- public ScanBuilder selection(JSONOptions selection) {
- this.selection = selection;
- return this;
- }
-
- public ScanBuilder outputReference(FieldReference outputReference) {
- this.outputReference = outputReference;
- return this;
- }
-
- public Scan build() {
- return new Scan(storageEngine, selection, outputReference);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java b/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
index 279e13c..0a5015c 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/SingleInputOperator.java
@@ -17,9 +17,12 @@
*/
package org.apache.drill.common.logical.data;
+import java.util.Iterator;
+
import org.apache.drill.common.logical.UnexpectedOperatorType;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Iterators;
/**
* SimpleOperator is an operator that has one inputs at most.
@@ -41,6 +44,9 @@ public abstract class SingleInputOperator extends LogicalOperatorBase {
}
-
+ @Override
+ public Iterator<LogicalOperator> iterator() {
+ return Iterators.singletonIterator(input);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/Store.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Store.java b/common/src/main/java/org/apache/drill/common/logical/data/Store.java
index 99d1d50..171599f 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Store.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Store.java
@@ -17,16 +17,15 @@
*/
package org.apache.drill.common.logical.data;
-import com.google.common.collect.Iterators;
+import java.util.Iterator;
+
import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.defs.PartitionDef;
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
+import com.google.common.collect.Iterators;
@JsonTypeName("store")
public class Store extends SinkOperator{
@@ -34,14 +33,12 @@ public class Store extends SinkOperator{
private final String storageEngine;
private final JSONOptions target;
- private final PartitionDef partition;
@JsonCreator
- public Store(@JsonProperty("storageengine") String storageEngine, @JsonProperty("target") JSONOptions target, @JsonProperty("partition") PartitionDef partition) {
+ public Store(@JsonProperty("storageengine") String storageEngine, @JsonProperty("target") JSONOptions target) {
super();
this.storageEngine = storageEngine;
this.target = target;
- this.partition = partition;
}
public String getStorageEngine() {
@@ -52,10 +49,6 @@ public class Store extends SinkOperator{
return target;
}
- public PartitionDef getPartition() {
- return partition;
- }
-
@Override
public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
return logicalVisitor.visitStore(this, value);
@@ -66,8 +59,26 @@ public class Store extends SinkOperator{
return Iterators.singletonIterator(getInput());
}
- public static StoreBuilder builder() {
- return new StoreBuilder();
+ public static Builder builder() {
+ return new Builder();
}
+ public static class Builder extends AbstractSingleBuilder<Store, Builder>{
+ private String storageEngine;
+ private JSONOptions target;
+
+ public Builder storageEngine(String storageEngine) {
+ this.storageEngine = storageEngine;
+ return this;
+ }
+
+ public Builder target(JSONOptions target) {
+ this.target = target;
+ return this;
+ }
+
+ public Store internalBuild() {
+ return new Store(storageEngine, target);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/StoreBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/StoreBuilder.java b/common/src/main/java/org/apache/drill/common/logical/data/StoreBuilder.java
deleted file mode 100644
index 09d20b6..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/StoreBuilder.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.common.logical.data;
-
-import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.defs.PartitionDef;
-
-/**
- * Builder for the scan operator
- */
-public class StoreBuilder {
- private String storageEngine;
- private JSONOptions target;
- private PartitionDef partition;
-
- public StoreBuilder storageEngine(String storageEngine) {
- this.storageEngine = storageEngine;
- return this;
- }
-
- public StoreBuilder target(JSONOptions target) {
- this.target = target;
- return this;
- }
-
- public StoreBuilder partition(PartitionDef partition) {
- this.partition = partition;
- return this;
- }
-
- public Store build() {
- return new Store(storageEngine, target, partition);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/Union.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Union.java b/common/src/main/java/org/apache/drill/common/logical/data/Union.java
index 4db48f8..40c43cd 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Union.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Union.java
@@ -17,13 +17,16 @@
*/
package org.apache.drill.common.logical.data;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
+
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.collect.Iterators;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
+import com.google.common.collect.Lists;
@JsonTypeName("union")
public class Union extends LogicalOperatorBase {
@@ -63,6 +66,29 @@ public class Union extends LogicalOperatorBase {
}
-
+ public static Builder builder(){
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractBuilder<Union>{
+ private List<LogicalOperator> inputs = Lists.newArrayList();
+ private boolean distinct;
+
+ public Builder addInput(LogicalOperator o){
+ inputs.add(o);
+ return this;
+ }
+
+ public Builder setDistinct(boolean distinct){
+ this.distinct = distinct;
+ return this;
+ }
+
+ @Override
+ public Union build() {
+ return new Union(inputs.toArray(new LogicalOperator[inputs.size()]), distinct);
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
index cd519c6..bb37a14 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
@@ -79,6 +79,11 @@ public abstract class AbstractLogicalVisitor<T, X, E extends Throwable> implemen
}
@Override
+ public T visitGroupingAggregate(GroupingAggregate groupBy, X value) throws E {
+ return visitOp(groupBy, value);
+ }
+
+ @Override
public T visitSequence(Sequence sequence, X value) throws E {
return visitOp(sequence, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
index 8308cc7..f409453 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
@@ -18,8 +18,23 @@
package org.apache.drill.common.logical.data.visitors;
-import org.apache.drill.common.graph.GraphVisitor;
-import org.apache.drill.common.logical.data.*;
+import org.apache.drill.common.logical.data.CollapsingAggregate;
+import org.apache.drill.common.logical.data.Constant;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.Flatten;
+import org.apache.drill.common.logical.data.GroupingAggregate;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.Limit;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.RunningAggregate;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Segment;
+import org.apache.drill.common.logical.data.Sequence;
+import org.apache.drill.common.logical.data.Store;
+import org.apache.drill.common.logical.data.Transform;
+import org.apache.drill.common.logical.data.Union;
+import org.apache.drill.common.logical.data.WindowFrame;
/**
* Visitor class designed to traversal of a operator tree. Basis for a number of operator manipulations including fragmentation and materialization.
@@ -31,11 +46,10 @@ public interface LogicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalVisitor.class);
- //public RETURN visitExchange(Exchange exchange, EXTRA value) throws EXCEP;
public RETURN visitScan(Scan scan, EXTRA value) throws EXCEP;
public RETURN visitStore(Store store, EXTRA value) throws EXCEP;
public RETURN visitCollapsingAggregate(CollapsingAggregate collapsingAggregate, EXTRA value) throws EXCEP;
-
+ public RETURN visitGroupingAggregate(GroupingAggregate groupBy, EXTRA value) throws EXCEP;
public RETURN visitFilter(Filter filter, EXTRA value) throws EXCEP;
public RETURN visitFlatten(Flatten flatten, EXTRA value) throws EXCEP;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/types/Types.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java
index efb7bf4..fd8991d 100644
--- a/common/src/main/java/org/apache/drill/common/types/Types.java
+++ b/common/src/main/java/org/apache/drill/common/types/Types.java
@@ -58,6 +58,85 @@ public class Types {
}
}
+ public static int getSqlType(MajorType type){
+ if(type.getMode() == DataMode.REPEATED) return java.sql.Types.ARRAY;
+
+ switch(type.getMinorType()){
+ case BIGINT:
+ return java.sql.Types.BIGINT;
+ case BIT:
+ return java.sql.Types.BOOLEAN;
+ case DATE:
+ return java.sql.Types.DATE;
+ case DATETIME:
+ return java.sql.Types.DATE;
+ case DECIMAL12:
+ case DECIMAL16:
+ case DECIMAL4:
+ case DECIMAL8:
+ return java.sql.Types.DECIMAL;
+ case FIXED16CHAR:
+ return java.sql.Types.NCHAR;
+ case FIXEDBINARY:
+ return java.sql.Types.BINARY;
+ case FIXEDCHAR:
+ return java.sql.Types.NCHAR;
+ case FLOAT4:
+ return java.sql.Types.FLOAT;
+ case FLOAT8:
+ return java.sql.Types.DOUBLE;
+ case INT:
+ return java.sql.Types.INTEGER;
+ case MAP:
+ return java.sql.Types.STRUCT;
+ case MONEY:
+ return java.sql.Types.DECIMAL;
+ case NULL:
+ case INTERVAL:
+ case LATE:
+ case REPEATMAP:
+ return java.sql.Types.OTHER;
+ case SMALLINT:
+ return java.sql.Types.SMALLINT;
+ case TIME:
+ return java.sql.Types.TIME;
+ case TIMESTAMP:
+ return java.sql.Types.TIMESTAMP;
+ case TIMETZ:
+ return java.sql.Types.DATE;
+ case TINYINT:
+ return java.sql.Types.TINYINT;
+ case UINT1:
+ return java.sql.Types.TINYINT;
+ case UINT2:
+ return java.sql.Types.SMALLINT;
+ case UINT4:
+ return java.sql.Types.INTEGER;
+ case UINT8:
+ return java.sql.Types.BIGINT;
+ case VAR16CHAR:
+ return java.sql.Types.NVARCHAR;
+ case VARBINARY:
+ return java.sql.Types.VARBINARY;
+ case VARCHAR:
+ return java.sql.Types.NVARCHAR;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static boolean isUnSigned(MajorType type){
+ switch(type.getMinorType()){
+ case UINT1:
+ case UINT2:
+ case UINT4:
+ case UINT8:
+ return true;
+ default:
+ return false;
+ }
+
+ }
public static boolean usesHolderForGet(MajorType type){
if(type.getMode() == REPEATED) return true;
switch(type.getMinorType()){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/common/src/main/java/org/apache/drill/common/util/TestTools.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/TestTools.java b/common/src/main/java/org/apache/drill/common/util/TestTools.java
new file mode 100644
index 0000000..0a82826
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/util/TestTools.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.util;
+
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+
+public class TestTools {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTools.class);
+
+ static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
+
+
+ public static TestRule getTimeoutRule(){
+ return IS_DEBUG ? new TestName() : new Timeout(10000);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b3460af8/contrib/sqlline/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/sqlline/pom.xml b/contrib/sqlline/pom.xml
index d4963a6..6376e28 100644
--- a/contrib/sqlline/pom.xml
+++ b/contrib/sqlline/pom.xml
@@ -37,7 +37,6 @@
<dependency>
<groupId>sqlline</groupId>
<artifactId>sqlline</artifactId>
- <version>1.1.0</version>
</dependency>
<!-- Specify xalan and xerces versions to avoid setXIncludeAware error. -->
<dependency>
[26/38] git commit: Integrate new SQL changes with Hive storage
engine,
moving to automatic file detection. Rename storage engine to storage plugin.
Separate storage plugins from format plugins,
updating Parquet and JSON to format engines. Refactor dist
Posted by ja...@apache.org.
Integrate new SQL changes with Hive storage engine, moving to automatic file detection. Rename storage engine to storage plugin. Separate storage plugins from format plugins, updating Parquet and JSON to format engines. Refactor distribution logic and enable use within JSON format and all EasyFormatPlugins.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/8b90cae3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/8b90cae3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/8b90cae3
Branch: refs/heads/master
Commit: 8b90cae3fe3c5aa48b7d82147452fda043305bd4
Parents: 2dfadc9
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Feb 17 12:06:06 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.gitignore | 3 +
INSTALL.md | 10 +-
common/pom.xml | 7 -
.../org/apache/drill/common/JSONOptions.java | 6 +-
.../drill/common/config/CommonConstants.java | 2 +-
.../apache/drill/common/config/DrillConfig.java | 7 +-
.../common/logical/FormatPluginConfig.java | 27 +
.../common/logical/FormatPluginConfigBase.java | 41 +
.../drill/common/logical/LogicalPlan.java | 10 +-
.../common/logical/LogicalPlanBuilder.java | 4 +-
.../common/logical/StorageEngineConfig.java | 28 -
.../common/logical/StorageEngineConfigBase.java | 41 -
.../common/logical/StoragePluginConfig.java | 28 +
.../common/logical/StoragePluginConfigBase.java | 41 +
.../drill/common/util/ConstructorChecker.java | 55 ++
.../org/apache/drill/common/util/TestTools.java | 6 +-
.../drill/storage/CheckStorageConfig.java | 4 +-
contrib/sqlline/pom.xml | 2 -
distribution/pom.xml | 1 +
distribution/src/resources/storage-engines.json | 44 +-
exec/java-exec/pom.xml | 13 -
.../org/apache/drill/exec/ops/QueryContext.java | 22 +-
.../apache/drill/exec/opt/BasicOptimizer.java | 24 +-
.../apache/drill/exec/physical/ReadEntry.java | 33 -
.../drill/exec/physical/ReadEntryFromHDFS.java | 54 --
.../drill/exec/physical/ReadEntryWithPath.java | 49 --
.../exec/physical/base/AbstractGroupScan.java | 2 -
.../exec/physical/base/AbstractSubScan.java | 70 ++
.../drill/exec/physical/base/GroupScan.java | 3 +-
.../apache/drill/exec/physical/base/Scan.java | 1 -
.../drill/exec/physical/base/SubScan.java | 1 -
.../drill/exec/planner/PhysicalPlanReader.java | 6 +-
.../drill/exec/planner/logical/DrillTable.java | 47 +-
.../exec/planner/logical/DynamicDrillTable.java | 35 +
.../exec/planner/logical/StorageEngines.java | 10 +-
.../exec/planner/sql/DrillSchemaFactory.java | 96 ---
.../drill/exec/planner/sql/DrillSqlWorker.java | 32 +-
.../exec/planner/sql/FileSystemSchema.java | 117 ---
.../drill/exec/server/DrillbitContext.java | 40 +-
.../apache/drill/exec/store/AbstractSchema.java | 101 +++
.../drill/exec/store/AbstractStorageEngine.java | 89 ---
.../drill/exec/store/AbstractStoragePlugin.java | 54 ++
.../drill/exec/store/AffinityCalculator.java | 150 ----
.../drill/exec/store/ClassPathFileSystem.java | 21 +-
.../exec/store/NamedStoragePluginConfig.java | 27 +
.../apache/drill/exec/store/SchemaFactory.java | 27 +
.../apache/drill/exec/store/SchemaHolder.java | 44 ++
.../apache/drill/exec/store/SchemaProvider.java | 25 -
.../exec/store/SchemaProviderRegistry.java | 82 --
.../apache/drill/exec/store/StorageEngine.java | 122 ---
.../drill/exec/store/StorageEngineRegistry.java | 84 ---
.../apache/drill/exec/store/StoragePlugin.java | 48 ++
.../drill/exec/store/StoragePluginRegistry.java | 175 +++++
.../exec/store/dfs/BasicFormatMatcher.java | 130 ++++
.../drill/exec/store/dfs/FileSelection.java | 127 ++++
.../drill/exec/store/dfs/FileSystemConfig.java | 35 +
.../exec/store/dfs/FileSystemFormatConfig.java | 29 +
.../drill/exec/store/dfs/FileSystemPlugin.java | 125 ++++
.../exec/store/dfs/FileSystemSchemaFactory.java | 116 +++
.../drill/exec/store/dfs/FormatCreator.java | 100 +++
.../drill/exec/store/dfs/FormatMatcher.java | 28 +
.../drill/exec/store/dfs/FormatPlugin.java | 52 ++
.../drill/exec/store/dfs/FormatSelection.java | 64 ++
.../drill/exec/store/dfs/MagicString.java | 42 ++
.../exec/store/dfs/NamedFormatPluginConfig.java | 28 +
.../drill/exec/store/dfs/ReadEntryFromHDFS.java | 44 ++
.../drill/exec/store/dfs/ReadEntryWithPath.java | 37 +
.../exec/store/dfs/WorkspaceSchemaFactory.java | 123 +++
.../exec/store/dfs/easy/EasyBatchCreator.java | 38 +
.../exec/store/dfs/easy/EasyFormatPlugin.java | 142 ++++
.../exec/store/dfs/easy/EasyGroupScan.java | 183 +++++
.../drill/exec/store/dfs/easy/EasySubScan.java | 103 +++
.../drill/exec/store/dfs/easy/FileWork.java | 27 +
.../exec/store/dfs/shim/DrillFileSystem.java | 43 ++
.../exec/store/dfs/shim/DrillInputStream.java | 32 +
.../exec/store/dfs/shim/DrillOutputStream.java | 29 +
.../exec/store/dfs/shim/FileSystemCreator.java | 36 +
.../dfs/shim/fallback/FallbackFileSystem.java | 146 ++++
.../exec/store/easy/json/JSONFormatPlugin.java | 73 ++
.../exec/store/easy/json/JSONRecordReader.java | 541 ++++++++++++++
.../drill/exec/store/hive/HiveReadEntry.java | 27 +-
.../drill/exec/store/hive/HiveReadEntryOld.java | 50 --
.../apache/drill/exec/store/hive/HiveScan.java | 32 +-
.../exec/store/hive/HiveStorageEngine.java | 194 -----
.../store/hive/HiveStorageEngineConfig.java | 71 --
.../exec/store/hive/HiveStoragePlugin.java | 81 ++
.../store/hive/HiveStoragePluginConfig.java | 71 ++
.../exec/store/hive/schema/DrillHiveTable.java | 177 +++++
.../store/hive/schema/HiveDatabaseSchema.java | 56 ++
.../store/hive/schema/HiveSchemaFactory.java | 274 +++++++
.../drill/exec/store/json/JSONGroupScan.java | 162 ----
.../drill/exec/store/json/JSONRecordReader.java | 541 --------------
.../exec/store/json/JSONScanBatchCreator.java | 46 --
.../exec/store/json/JSONStorageEngine.java | 78 --
.../store/json/JSONStorageEngineConfig.java | 54 --
.../drill/exec/store/json/JSONSubScan.java | 122 ---
.../exec/store/json/JsonSchemaProvider.java | 62 --
.../drill/exec/store/mock/MockGroupScanPOP.java | 7 +-
.../exec/store/mock/MockStorageEngine.java | 15 +-
.../store/mock/MockStorageEngineConfig.java | 4 +-
.../drill/exec/store/mock/MockSubScanPOP.java | 26 +-
.../drill/exec/store/parquet/BitReader.java | 2 +-
.../exec/store/parquet/ColumnDataReader.java | 2 +-
.../drill/exec/store/parquet/ColumnReader.java | 2 +-
.../store/parquet/FixedByteAlignedReader.java | 2 +-
.../exec/store/parquet/NullableBitReader.java | 2 +-
.../store/parquet/NullableColumnReader.java | 2 +-
.../parquet/NullableFixedByteAlignedReader.java | 2 +-
.../exec/store/parquet/PageReadStatus.java | 8 +-
.../exec/store/parquet/ParquetFormatConfig.java | 37 +
.../exec/store/parquet/ParquetFormatPlugin.java | 178 +++++
.../exec/store/parquet/ParquetGroupScan.java | 356 ++++-----
.../exec/store/parquet/ParquetRecordReader.java | 2 +-
.../exec/store/parquet/ParquetRowGroupScan.java | 93 +--
.../store/parquet/ParquetScanBatchCreator.java | 14 +-
.../store/parquet/ParquetSchemaProvider.java | 63 --
.../store/parquet/ParquetStorageEngine.java | 123 ---
.../parquet/ParquetStorageEngineConfig.java | 68 --
.../exec/store/parquet/RowGroupReadEntry.java | 44 ++
.../exec/store/parquet/VarLenBinaryReader.java | 4 +
.../exec/store/schedule/AffinityCreator.java | 62 ++
.../exec/store/schedule/AssignmentCreator.java | 135 ++++
.../exec/store/schedule/BlockMapBuilder.java | 215 ++++++
.../exec/store/schedule/CompleteFileWork.java | 105 +++
.../drill/exec/store/schedule/CompleteWork.java | 29 +
.../exec/store/schedule/EndpointByteMap.java | 34 +
.../store/schedule/EndpointByteMapImpl.java | 61 ++
.../drill/exec/store/schedule/PartialWork.java | 43 ++
.../apache/drill/exec/work/foreman/Foreman.java | 2 +-
.../src/main/resources/storage-engines.json | 29 -
.../java/org/apache/drill/exec/TestPlan.java | 48 --
.../exec/physical/impl/TestOptiqPlans.java | 4 +-
.../exec/physical/impl/join/TestMergeJoin.java | 8 +-
.../exec/store/CachedSingleFileSystem.java | 171 +++++
.../drill/exec/store/JSONRecordReaderTest.java | 390 ----------
.../exec/store/ParquetRecordReaderTest.java | 742 -------------------
.../exec/store/TestAffinityCalculator.java | 113 +--
.../drill/exec/store/TestOutputMutator.java | 74 ++
.../exec/store/TestParquetPhysicalPlan.java | 136 ----
.../exec/store/json/JSONRecordReaderTest.java | 390 ++++++++++
.../drill/exec/store/parquet/FieldInfo.java | 40 +
.../store/parquet/ParquetRecordReaderTest.java | 450 ++++++-----
.../store/parquet/ParquetRecordReaderTest2.java | 742 +++++++++++++++++++
.../store/parquet/ParquetResultListener.java | 203 +++++
.../store/parquet/ParquetTestProperties.java | 37 +
.../exec/store/parquet/TestFileGenerator.java | 142 ++--
.../store/parquet/TestParquetPhysicalPlan.java | 136 ++++
.../exec/store/parquet/WrapAroundCounter.java | 40 +
.../join/merge_inner_single_batch.json | 28 +-
.../test/resources/join/merge_multi_batch.json | 30 +-
.../test/resources/join/merge_single_batch.json | 28 +-
.../src/test/resources/logical_json_scan.json | 8 +-
.../resources/parquet/parquet_scan_screen.json | 23 +-
.../parquet_scan_screen_read_entry_replace.json | 15 +-
.../parquet_scan_union_screen_physical.json | 6 +-
.../src/test/resources/parquet_scan_screen.json | 17 +-
.../parquet_scan_screen_read_entry_replace.json | 15 +-
.../parquet_selective_column_read.json | 9 +-
.../resources/physical_json_scan_test1.json | 14 +-
.../resources/sender/broadcast_exchange.json | 30 +-
.../src/test/resources/storage-engines.json | 20 +
pom.xml | 5 +-
sqlparser/pom.xml | 25 +-
.../sql/client/full/HiveDatabaseSchema.java | 314 --------
.../drill/sql/client/full/HiveSchema.java | 162 ----
.../org/apache/drill/jdbc/test/JdbcTest.java | 10 +-
.../apache/drill/jdbc/test/TestJdbcQuery.java | 39 +-
.../src/test/resources/storage-engines.json | 47 +-
168 files changed, 7613 insertions(+), 5511 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 6bda406..c27a41b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,10 @@
.classpath
.settings/
.idea/
+*.log
+*.lck
*.iml
target/
*.DS_Store
+*.patch
*~
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/INSTALL.md
----------------------------------------------------------------------
diff --git a/INSTALL.md b/INSTALL.md
index 3faf7e4..e89da01 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -4,13 +4,9 @@
Currently, the Apache Drill build process is known to work on Linux, Windows and OSX. To build, you need to have the following software installed on your system to successfully complete a build.
* Java 7+
- * protoc 2.5.x compiler
* Maven 3.0+
## Confirm settings
- # protoc --version
- libprotoc 2.5.0
-
# java -version
java version "1.7.0_09"
Java(TM) SE Runtime Environment (build 1.7.0_09-b05)
@@ -30,13 +26,13 @@ Currently, the Apache Drill build process is known to work on Linux, Windows and
## Start SQLLine
- ./sqlline -u jdbc:drill:schema=parquet-local -n admin -p admin
+ ./sqlline -u jdbc:drill:zk=local -n admin -p admin
## Run a query
SELECT
- _MAP['R_REGIONKEY'] AS region_key,
- _MAP['R_NAME'] AS name, _MAP['R_COMMENT'] AS comment
+ region_key,
+ r_comment
FROM "sample-data/region.parquet";
## More information
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index d76f7a1..afcf7a3 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -47,13 +47,6 @@
<artifactId>optiq-core</artifactId>
</dependency>
-
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>2.5.0</version>
- </dependency>
-
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/JSONOptions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/JSONOptions.java b/common/src/main/java/org/apache/drill/common/JSONOptions.java
index c1f9a85..c77412d 100644
--- a/common/src/main/java/org/apache/drill/common/JSONOptions.java
+++ b/common/src/main/java/org/apache/drill/common/JSONOptions.java
@@ -19,6 +19,7 @@ package org.apache.drill.common;
import java.io.IOException;
import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
import org.apache.drill.common.JSONOptions.De;
import org.apache.drill.common.JSONOptions.Se;
@@ -87,8 +88,9 @@ public class JSONOptions {
@SuppressWarnings("unchecked")
public <T> T getListWith(ObjectMapper mapper, TypeReference<T> t) throws IOException {
if(opaque != null){
- ParameterizedType pt = (ParameterizedType) t.getType();
- if ( pt.getRawType().equals(opaque.getClass())){
+ Type c = t.getType();
+ if(c instanceof ParameterizedType) c = ((ParameterizedType)c).getRawType();
+ if ( c.equals(opaque.getClass())){
return (T) opaque;
}else{
throw new IOException(String.format("Attmpted to retrieve a list with type of %s. However, the JSON options carried an opaque value of type %s.", t.getType(), opaque.getClass().getName()));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java b/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
index f36a0e9..234c9b2 100644
--- a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
+++ b/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
@@ -23,7 +23,7 @@ public interface CommonConstants {
public static final String LOGICAL_OPERATOR_SCAN_PACKAGES = "drill.logical.operator.packages";
public static final String PHYSICAL_OPERATOR_SCAN_PACKAGES = "drill.physical.operator.packages";
- public static final String STORAGE_ENGINE_CONFIG_SCAN_PACKAGES = "drill.logical.storage.packages";
+ public static final String STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES = "drill.logical.storage.packages";
public static final String DRILL_JAR_MARKER_FILE = "drill-module.conf";
public static final String LOGICAL_FUNCTION_SCAN_PACKAGES = "drill.logical.function.packages";
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index a0e261f..6fc048d 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -27,7 +27,8 @@ import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.drill.common.exceptions.DrillConfigurationException;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
+import org.apache.drill.common.logical.FormatPluginConfigBase;
+import org.apache.drill.common.logical.StoragePluginConfigBase;
import org.apache.drill.common.logical.data.LogicalOperatorBase;
import org.apache.drill.common.util.PathScanner;
@@ -61,13 +62,15 @@ public final class DrillConfig extends NestedConfig{
SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
.addDeserializer(LogicalExpression.class, new LogicalExpression.De(this));
+
mapper.registerModule(deserModule);
mapper.enable(SerializationFeature.INDENT_OUTPUT);
mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
mapper.configure(Feature.ALLOW_COMMENTS, true);
mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this));
- mapper.registerSubtypes(StorageEngineConfigBase.getSubTypes(this));
+ mapper.registerSubtypes(StoragePluginConfigBase.getSubTypes(this));
+ mapper.registerSubtypes(FormatPluginConfigBase.getSubTypes(this));
RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
this.startupArguments = ImmutableList.copyOf(bean.getInputArguments());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
new file mode 100644
index 0000000..20b4d66
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
+public interface FormatPluginConfig {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfig.class);
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
new file mode 100644
index 0000000..e640359
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import java.util.List;
+
+import org.apache.drill.common.config.CommonConstants;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.util.PathScanner;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public abstract class FormatPluginConfigBase implements FormatPluginConfig{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfigBase.class);
+
+
+ public synchronized static Class<?>[] getSubTypes(DrillConfig config){
+ List<String> packages = config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES);
+ Class<?>[] sec = PathScanner.scanForImplementationsArr(FormatPluginConfig.class, packages);
+ logger.debug("Adding Format Plugin Configs including {}", (Object) sec );
+ return sec;
+ }
+
+ public abstract boolean equals(Object o);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
index a9db251..06ea117 100644
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
+++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
@@ -44,15 +44,15 @@ public class LogicalPlan {
static final Logger logger = LoggerFactory.getLogger(LogicalPlan.class);
private final PlanProperties properties;
- private final Map<String, StorageEngineConfig> storageEngineMap;
+ private final Map<String, StoragePluginConfig> storageEngineMap;
private final Graph<LogicalOperator, SinkOperator, SourceOperator> graph;
@JsonCreator
public LogicalPlan(@JsonProperty("head") PlanProperties head,
- @JsonProperty("storage") Map<String, StorageEngineConfig> storageEngineMap,
+ @JsonProperty("storage") Map<String, StoragePluginConfig> storageEngineMap,
@JsonProperty("query") List<LogicalOperator> operators) {
- this.storageEngineMap = storageEngineMap != null ? storageEngineMap : new HashMap<String, StorageEngineConfig>();
+ this.storageEngineMap = storageEngineMap != null ? storageEngineMap : new HashMap<String, StoragePluginConfig>();
this.properties = head;
this.graph = Graph.newGraph(operators, SinkOperator.class, SourceOperator.class);
}
@@ -62,7 +62,7 @@ public class LogicalPlan {
return GraphAlgos.TopoSorter.sortLogical(graph);
}
- public StorageEngineConfig getStorageEngineConfig(String name) {
+ public StoragePluginConfig getStorageEngineConfig(String name) {
return storageEngineMap.get(name);
}
@@ -77,7 +77,7 @@ public class LogicalPlan {
}
@JsonProperty("storage")
- public Map<String, StorageEngineConfig> getStorageEngines() {
+ public Map<String, StoragePluginConfig> getStorageEngines() {
return storageEngineMap;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
index 369cd58..1941a92 100644
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
+++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
@@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableMap;
*/
public class LogicalPlanBuilder {
private PlanProperties planProperties;
- private ImmutableMap.Builder<String, StorageEngineConfig> storageEngines = ImmutableMap.builder();
+ private ImmutableMap.Builder<String, StoragePluginConfig> storageEngines = ImmutableMap.builder();
private ImmutableList.Builder<LogicalOperator> operators = ImmutableList.builder();
public LogicalPlanBuilder planProperties(PlanProperties planProperties) {
@@ -42,7 +42,7 @@ public class LogicalPlanBuilder {
return this;
}
- public LogicalPlanBuilder addStorageEngine(String name, StorageEngineConfig config) {
+ public LogicalPlanBuilder addStorageEngine(String name, StoragePluginConfig config) {
this.storageEngines.put(name, config);
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfig.java b/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfig.java
deleted file mode 100644
index 343c5db..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfig.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
-public interface StorageEngineConfig{
-
- public boolean equals(Object o);
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfigBase.java
deleted file mode 100644
index 21468bc..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfigBase.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-import java.util.List;
-
-import org.apache.drill.common.config.CommonConstants;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.PathScanner;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public abstract class StorageEngineConfigBase implements StorageEngineConfig{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StorageEngineConfigBase.class);
-
-
- public synchronized static Class<?>[] getSubTypes(DrillConfig config){
- List<String> packages = config.getStringList(CommonConstants.STORAGE_ENGINE_CONFIG_SCAN_PACKAGES);
- Class<?>[] sec = PathScanner.scanForImplementationsArr(StorageEngineConfig.class, packages);
- logger.debug("Adding Storage Engine Configs including {}", (Object) sec );
- return sec;
- }
-
- public abstract boolean equals(Object o);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
new file mode 100644
index 0000000..7d7cf38
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
+public interface StoragePluginConfig{
+
+ public boolean equals(Object o);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
new file mode 100644
index 0000000..8e01493
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import java.util.List;
+
+import org.apache.drill.common.config.CommonConstants;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.util.PathScanner;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public abstract class StoragePluginConfigBase implements StoragePluginConfig{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginConfigBase.class);
+
+
+ public synchronized static Class<?>[] getSubTypes(DrillConfig config){
+ List<String> packages = config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES);
+ Class<?>[] sec = PathScanner.scanForImplementationsArr(StoragePluginConfig.class, packages);
+ logger.debug("Adding Storage Engine Configs including {}", (Object) sec );
+ return sec;
+ }
+
+ public abstract boolean equals(Object o);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/util/ConstructorChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/ConstructorChecker.java b/common/src/main/java/org/apache/drill/common/util/ConstructorChecker.java
new file mode 100644
index 0000000..a68c5ba
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/util/ConstructorChecker.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.util;
+
+import java.lang.reflect.Constructor;
+
+public class ConstructorChecker {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ConstructorChecker.class);
+
+ private final String requirementString;
+ private final Class<?>[] classes;
+
+
+ public ConstructorChecker(Class<?>... classes) {
+ super();
+ this.classes = classes;
+ StringBuffer sb = new StringBuffer();
+ sb.append("The required constructor is (");
+ for(int i =0; i < classes.length; i++){
+ if(i != 0) sb.append(", ");
+ sb.append(classes[i].getName());
+ }
+
+ this.requirementString = sb.toString();
+ }
+
+ public boolean check(Constructor<?> c){
+ Class<?>[] params = c.getParameterTypes();
+ if(params.length != classes.length) return false;
+ for(int i =0; i < classes.length; i++){
+ if( !classes[i].isAssignableFrom(params[i])) return false;
+ }
+
+ return true;
+ }
+
+ public String getRequirementString(){
+ return requirementString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/util/TestTools.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/TestTools.java b/common/src/main/java/org/apache/drill/common/util/TestTools.java
index 0a82826..500be59 100644
--- a/common/src/main/java/org/apache/drill/common/util/TestTools.java
+++ b/common/src/main/java/org/apache/drill/common/util/TestTools.java
@@ -28,7 +28,11 @@ public class TestTools {
public static TestRule getTimeoutRule(){
- return IS_DEBUG ? new TestName() : new Timeout(10000);
+ return getTimeoutRule(10000);
+ }
+
+ public static TestRule getTimeoutRule(int timeout){
+ return IS_DEBUG ? new TestName() : new Timeout(timeout);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
index 5bcdc8a..1dc6c77 100644
--- a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
+++ b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
@@ -23,7 +23,7 @@ import java.util.Collection;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.common.util.PathScanner;
import org.junit.Test;
@@ -37,7 +37,7 @@ public class CheckStorageConfig {
@Test
public void ensureStorageEnginePickup() {
- Collection<?> engines = PathScanner.scanForImplementations(StorageEngineConfig.class, Lists.newArrayList("org"));
+ Collection<?> engines = PathScanner.scanForImplementations(StoragePluginConfig.class, Lists.newArrayList("org"));
assertEquals(engines.size(), 1);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/contrib/sqlline/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/sqlline/pom.xml b/contrib/sqlline/pom.xml
index 6376e28..0029b61 100644
--- a/contrib/sqlline/pom.xml
+++ b/contrib/sqlline/pom.xml
@@ -42,12 +42,10 @@
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
- <version>2.9.1</version>
</dependency>
<dependency>
<groupId>xalan</groupId>
<artifactId>xalan</artifactId>
- <version>2.7.1</version>
</dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 15b2f53..3ad42cb 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -82,6 +82,7 @@
<exclude>**/control</exclude>
<exclude>**/*.json</exclude>
<exclude>**/hadoop-excludes.txt</exclude>
+ <exclude>**/drill_hive_db/*</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/distribution/src/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/distribution/src/resources/storage-engines.json b/distribution/src/resources/storage-engines.json
index d1d0413..a22e7d0 100644
--- a/distribution/src/resources/storage-engines.json
+++ b/distribution/src/resources/storage-engines.json
@@ -1,29 +1,23 @@
{
"storage":{
- "parquet-local" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
- },
- "parquet-cp" :
- {
- "type":"parquet",
- "dfsName" : "classpath:///"
- },
- "jsonl" :
- {
- "type":"json",
- "dfsName" : "file:///"
- },
- "json-cp" :
- {
- "type":"json",
- "dfsName" : "classpath:///"
- },
- "parquet" :
- {
- "type":"parquet",
- "dfsName" : "file:///"
+ dfs: {
+ type: "file",
+ connection: "file:///"
+ },
+ cp: {
+ type: "file",
+ connection: "classpath:///"
+ },
+ hive : {
+ type:"hive",
+ config :
+ {
+ "hive.metastore.uris" : "",
+ "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true",
+ "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
+ "fs.default.name" : "file:///",
+ "hive.metastore.sasl.enabled" : "false"
+ }
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 5c6fcfe..b4c282c 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -272,7 +272,6 @@
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
<executions>
<execution>
<id>copy-resources</id>
@@ -369,18 +368,6 @@
</executions>
</plugin>
- <!-- <plugin> -->
- <!-- <groupId>com.github.igor-petruk.protobuf</groupId> -->
- <!-- <artifactId>protobuf-maven-plugin</artifactId> -->
- <!-- <version>0.6.2</version> -->
- <!-- <executions> -->
- <!-- <execution> -->
- <!-- <goals> -->
- <!-- <goal>run</goal> -->
- <!-- </goals> -->
- <!-- </execution> -->
- <!-- </executions> -->
- <!-- </plugin> -->
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index 11658e6..36cfb53 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -20,18 +20,16 @@ package org.apache.drill.exec.ops;
import java.util.Collection;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FunctionRegistry;
-import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.planner.sql.DrillSchemaFactory;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.rpc.control.WorkEventBus;
import org.apache.drill.exec.rpc.data.DataConnectionCreator;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.StorageEngine;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory;
public class QueryContext {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class);
@@ -55,10 +53,11 @@ public class QueryContext {
return queryId;
}
- public StorageEngine getStorageEngine(StorageEngineConfig config) throws ExecutionSetupException {
- return drillbitContext.getStorageEngine(config);
+ public StoragePluginRegistry getStorage(){
+ return drillbitContext.getStorage();
}
-
+
+
public DistributedCache getCache(){
return drillbitContext.getCache();
}
@@ -82,14 +81,13 @@ public class QueryContext {
public WorkEventBus getWorkBus(){
return workBus;
}
-
- public DrillSchemaFactory getSchemaFactory(){
- return drillbitContext.getSchemaFactory();
- }
public FunctionRegistry getFunctionRegistry(){
return drillbitContext.getFunctionRegistry();
-
+ }
+
+ public DrillSchemaFactory getFactory(){
+ return drillbitContext.getSchemaFactory();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index cd59428..aed4802 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -30,9 +30,19 @@ import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.logical.data.*;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.common.logical.data.CollapsingAggregate;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.JoinCondition;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.logical.data.Order;
import org.apache.drill.common.logical.data.Order.Ordering;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Segment;
+import org.apache.drill.common.logical.data.SinkOperator;
+import org.apache.drill.common.logical.data.Store;
import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.DataMode;
@@ -41,13 +51,13 @@ import org.apache.drill.exec.exception.OptimizerException;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.Limit;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.config.Screen;
import org.apache.drill.exec.physical.config.SelectionVectorRemover;
import org.apache.drill.exec.physical.config.Sort;
-import org.apache.drill.exec.physical.config.Limit;
import org.apache.drill.exec.physical.config.StreamingAggregate;
-import org.apache.drill.exec.store.StorageEngine;
+import org.apache.drill.exec.store.StoragePlugin;
import org.eigenbase.rel.RelFieldCollation.Direction;
import org.eigenbase.rel.RelFieldCollation.NullDirection;
@@ -192,11 +202,11 @@ public class BasicOptimizer extends Optimizer{
@Override
public PhysicalOperator visitScan(Scan scan, Object obj) throws OptimizerException {
- StorageEngineConfig config = logicalPlan.getStorageEngineConfig(scan.getStorageEngine());
+ StoragePluginConfig config = logicalPlan.getStorageEngineConfig(scan.getStorageEngine());
if(config == null) throw new OptimizerException(String.format("Logical plan referenced the storage engine config %s but the logical plan didn't have that available as a config.", scan.getStorageEngine()));
- StorageEngine engine;
+ StoragePlugin engine;
try {
- engine = context.getStorageEngine(config);
+ engine = context.getStorage().getEngine(config);
return engine.getPhysicalScan(scan);
} catch (IOException | ExecutionSetupException e) {
throw new OptimizerException("Failure while attempting to retrieve storage engine.", e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java
deleted file mode 100644
index f99ad0e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical;
-
-import org.apache.drill.exec.physical.base.Size;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-/**
- * Describes a chunk of read work that will be done.
- */
-public interface ReadEntry {
- @JsonIgnore
- public OperatorCost getCost();
-
- @JsonIgnore
- public Size getSize();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java
deleted file mode 100644
index e748d4c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.drill.exec.physical.base.Size;
-
-public class ReadEntryFromHDFS extends ReadEntryWithPath {
-
- private long start;
- private long length;
-
- @JsonCreator
- public ReadEntryFromHDFS(@JsonProperty("path") String path,@JsonProperty("start") long start, @JsonProperty("length") long length) {
- this.path = path;
- this.start = start;
- this.length = length;
- }
-
- @Override
- public OperatorCost getCost() {
- return new OperatorCost(1, 2, 1, 1);
- }
-
- @Override
- public Size getSize() {
- // TODO - these values are wrong, I cannot know these until after I read a file
- return new Size(10, 10);
- }
-
- public long getStart() {
- return start;
- }
-
- public long getLength() {
- return length;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
deleted file mode 100644
index a24869b..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical;
-
-import org.apache.drill.exec.physical.base.Size;
-
-public class ReadEntryWithPath implements ReadEntry {
-
- protected String path;
-
-
- public ReadEntryWithPath(String path) {
- super();
- this.path = path;
- }
-
- public ReadEntryWithPath(){}
-
- public String getPath(){
- return path;
- }
-
- @Override
- public OperatorCost getCost() {
- throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " +
- "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost.");
- }
-
- @Override
- public Size getSize() {
- throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " +
- "selections on a scan node from a logical plan, it cannot be used in an executing plan and has no size.");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
index 0b486e9..343435b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
@@ -19,8 +19,6 @@ package org.apache.drill.exec.physical.base;
import java.util.Iterator;
-import org.apache.drill.exec.physical.ReadEntry;
-
import com.google.common.collect.Iterators;
public abstract class AbstractGroupScan extends AbstractBase implements GroupScan {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
new file mode 100644
index 0000000..c6e11d1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.base;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.graph.GraphVisitor;
+import org.apache.drill.exec.physical.OperatorCost;
+
+import com.google.common.collect.Iterators;
+
+public abstract class AbstractSubScan implements SubScan{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSubScan.class);
+
+ @Override
+ public OperatorCost getCost() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Size getSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isExecutable() {
+ return true;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
+ return physicalVisitor.visitSubScan(this, value);
+ }
+
+ @Override
+ public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
+ assert children == null || children.isEmpty();
+ return this;
+ }
+
+ @Override
+ public void accept(GraphVisitor<PhysicalOperator> visitor) {
+ visitor.enter(this);
+ visitor.visit(this);
+ visitor.leave(this);
+ }
+
+ @Override
+ public Iterator<PhysicalOperator> iterator() {
+ return Iterators.emptyIterator();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
index b4a85df..e51ebb6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.physical.base;
import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -30,7 +31,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
*/
public interface GroupScan extends Scan, HasAffinity{
- public abstract void applyAssignments(List<DrillbitEndpoint> endpoints);
+ public abstract void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException;
public abstract SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java
index 63af5eb..9795433 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.physical.base;
-import org.apache.drill.exec.physical.ReadEntry;
public interface Scan extends Leaf {
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java
index 5fe3b99..e6c58a3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.physical.base;
-import org.apache.drill.exec.physical.ReadEntry;
/**
* A SubScan operator represents the data scanned by a particular major/minor fragment. This is in contrast to
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
index e270b79..a100163 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
@@ -37,7 +37,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
public class PhysicalPlanReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PhysicalPlanReader.class);
@@ -48,7 +48,7 @@ public class PhysicalPlanReader {
private final ObjectReader logicalPlanReader;
public PhysicalPlanReader(DrillConfig config, ObjectMapper mapper, final DrillbitEndpoint endpoint,
- final StorageEngineRegistry engineRegistry) {
+ final StoragePluginRegistry engineRegistry) {
// Endpoint serializer/deserializer.
SimpleModule deserModule = new SimpleModule("PhysicalOperatorModule") //
@@ -61,7 +61,7 @@ public class PhysicalPlanReader {
mapper.registerModule(deserModule);
mapper.registerSubtypes(PhysicalOperatorUtil.getSubTypes(config));
InjectableValues injectables = new InjectableValues.Std() //
- .addValue(StorageEngineRegistry.class, engineRegistry) //
+ .addValue(StoragePluginRegistry.class, engineRegistry) //
.addValue(DrillbitEndpoint.class, endpoint); //
this.mapper = mapper;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
index 30dd48d..02ca0ad 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
@@ -17,42 +17,31 @@
*/
package org.apache.drill.exec.planner.logical;
-import java.util.Collections;
-
import net.hydromatic.optiq.Schema.TableType;
import net.hydromatic.optiq.Statistic;
import net.hydromatic.optiq.Statistics;
import net.hydromatic.optiq.Table;
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
import org.eigenbase.rel.RelNode;
import org.eigenbase.relopt.RelOptTable;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeFactory;
-import org.eigenbase.sql.type.SqlTypeName;
-/** Optiq Table used by Drill. */
-public class DrillTable implements Table{
+
+public abstract class DrillTable implements Table{
- private final String name;
private final String storageEngineName;
- public final StorageEngineConfig storageEngineConfig;
+ public final StoragePluginConfig storageEngineConfig;
private Object selection;
/** Creates a DrillTable. */
- public DrillTable(String name, String storageEngineName, Object selection, StorageEngineConfig storageEngineConfig) {
- this.name = name;
+ public DrillTable(String storageEngineName, Object selection, StoragePluginConfig storageEngineConfig) {
this.selection = selection;
this.storageEngineConfig = storageEngineConfig;
this.storageEngineName = storageEngineName;
}
-
- public String getName() {
- return name;
- }
- public StorageEngineConfig getStorageEngineConfig(){
+ public StoragePluginConfig getStorageEngineConfig(){
return storageEngineConfig;
}
@@ -76,32 +65,10 @@ public class DrillTable implements Table{
}
@Override
- public RelDataType getRowType(RelDataTypeFactory typeFactory) {
- return new RelDataTypeDrillImpl(typeFactory);
- }
-
- @Override
public TableType getJdbcTableType() {
- return null;
+ return TableType.TABLE;
}
-// /** Factory for custom tables in Optiq schema. */
-// @SuppressWarnings("UnusedDeclaration")
-// public static class Factory implements TableFactory<DrillTable> {
-//
-// @Override
-// public DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) {
-//
-// final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
-// final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
-// inputConfig.path = "/" + name.toLowerCase() + ".json";
-// inputConfig.type = DataWriter.ConverterType.JSON;
-// return createTable(schema.getTypeFactory(), (MutableSchema) schema, name, "donuts-json", rseConfig, inputConfig);
-// }
-// }
-
-
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java
new file mode 100644
index 0000000..c50e67c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+
+public class DynamicDrillTable extends DrillTable{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DynamicDrillTable.class);
+
+ public DynamicDrillTable(String storageEngineName, Object selection, StoragePluginConfig storageEngineConfig) {
+ super(storageEngineName, selection, storageEngineConfig);
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ return new RelDataTypeDrillImpl(typeFactory);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
index a020ab3..d298040 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
@@ -23,19 +23,19 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.io.Resources;
-public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineConfig>>{
+public class StorageEngines implements Iterable<Map.Entry<String, StoragePluginConfig>>{
- private Map<String, StorageEngineConfig> storage;
+ private Map<String, StoragePluginConfig> storage;
@JsonCreator
- public StorageEngines(@JsonProperty("storage") Map<String, StorageEngineConfig> storage){
+ public StorageEngines(@JsonProperty("storage") Map<String, StoragePluginConfig> storage){
this.storage = storage;
}
@@ -53,7 +53,7 @@ public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineC
}
@Override
- public Iterator<Entry<String, StorageEngineConfig>> iterator() {
+ public Iterator<Entry<String, StoragePluginConfig>> iterator() {
return storage.entrySet().iterator();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
deleted file mode 100644
index 23d03b8..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner.sql;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import net.hydromatic.linq4j.function.Function1;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.SchemaPlus;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.exception.SetupException;
-import org.apache.drill.exec.planner.logical.StorageEngines;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.drill.exec.store.SchemaProviderRegistry;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class DrillSchemaFactory implements Function1<SchemaPlus, Schema>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSchemaFactory.class);
-
- private final SchemaProviderRegistry registry;
- private final Map<String, StorageEngineEntry> preEntries = Maps.newHashMap();
-
- public static DrillSchemaFactory createEmpty(){
- return new DrillSchemaFactory();
- }
-
- private DrillSchemaFactory(){
- this.registry = null;
- }
-
- public DrillSchemaFactory(StorageEngines engines, DrillConfig config) throws SetupException {
- super();
- this.registry = new SchemaProviderRegistry(config);
-
- for (Map.Entry<String, StorageEngineConfig> entry : engines) {
- SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
- preEntries.put(entry.getKey(), new StorageEngineEntry(entry.getValue(), provider));
- }
-
- }
-
- public Schema apply(SchemaPlus root) {
- List<String> schemaNames = Lists.newArrayList();
- Schema defaultSchema = null;
- for(Entry<String, StorageEngineEntry> e : preEntries.entrySet()){
- FileSystemSchema schema = new FileSystemSchema(e.getValue().getConfig(), e.getValue().getProvider(), root, e.getKey());
- if(defaultSchema == null) defaultSchema = schema;
- root.add(schema);
- schemaNames.add(e.getKey());
- }
- logger.debug("Registered schemas for {}", schemaNames);
- return defaultSchema;
- }
-
-
- private class StorageEngineEntry{
- StorageEngineConfig config;
- SchemaProvider provider;
-
-
- public StorageEngineEntry(StorageEngineConfig config, SchemaProvider provider) {
- super();
- this.config = config;
- this.provider = provider;
- }
-
- public StorageEngineConfig getConfig() {
- return config;
- }
- public SchemaProvider getProvider() {
- return provider;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index 818e892..0fdff3c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -24,7 +24,6 @@ import net.hydromatic.optiq.tools.RelConversionException;
import net.hydromatic.optiq.tools.RuleSet;
import net.hydromatic.optiq.tools.ValidationException;
-import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.FunctionRegistry;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
@@ -34,7 +33,7 @@ import org.apache.drill.exec.planner.logical.DrillRel;
import org.apache.drill.exec.planner.logical.DrillRuleSets;
import org.apache.drill.exec.planner.logical.DrillScreenRel;
import org.apache.drill.exec.planner.logical.DrillStoreRel;
-import org.apache.drill.exec.planner.logical.StorageEngines;
+import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory;
import org.eigenbase.rel.RelNode;
import org.eigenbase.sql.SqlExplain;
import org.eigenbase.sql.SqlKind;
@@ -43,9 +42,6 @@ import org.eigenbase.sql.SqlNode;
import org.eigenbase.sql.fun.SqlStdOperatorTable;
import org.eigenbase.sql.parser.SqlParseException;
-import com.google.common.base.Charsets;
-import com.google.common.io.Resources;
-
public class DrillSqlWorker {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSqlWorker.class);
@@ -55,9 +51,7 @@ public class DrillSqlWorker {
public DrillSqlWorker(DrillSchemaFactory schemaFactory, FunctionRegistry functionRegistry) throws Exception {
this.registry = functionRegistry;
this.planner = Frameworks.getPlanner(ConnectionConfig.Lex.MYSQL, schemaFactory, SqlStdOperatorTable.instance(), new RuleSet[]{DrillRuleSets.DRILL_BASIC_RULES});
-
}
-
public LogicalPlan getPlan(String sql) throws SqlParseException, ValidationException, RelConversionException{
SqlNode sqlNode = planner.parse(sql);
@@ -77,8 +71,6 @@ public class DrillSqlWorker {
break;
default:
}
-
-
}
SqlNode validatedNode = planner.validate(sqlNode);
@@ -96,28 +88,6 @@ public class DrillSqlWorker {
return implementor.getPlan();
}
- private void x() throws Exception {
- String sqlAgg = "select a, count(1) from parquet.`/Users/jnadeau/region.parquet` group by a";
- String sql = "select * from parquet.`/Users/jnadeau/region.parquet`";
-
-
- System.out.println(sql);
- System.out.println(getPlan(sql).toJsonString(DrillConfig.create()));
- System.out.println("///////////");
- System.out.println(sqlAgg);
- System.out.println(getPlan(sqlAgg).toJsonString(DrillConfig.create()));
- }
-
- public static void main(String[] args) throws Exception {
- DrillConfig config = DrillConfig.create();
-
- String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
- StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class);
- FunctionRegistry fr = new FunctionRegistry(config);
- DrillSchemaFactory schemaFactory = new DrillSchemaFactory(engines, config);
- DrillSqlWorker worker = new DrillSqlWorker(schemaFactory, fr);
- worker.x();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
deleted file mode 100644
index ac29fc3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner.sql;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
-
-import net.hydromatic.linq4j.expressions.DefaultExpression;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.SchemaPlus;
-import net.hydromatic.optiq.TableFunction;
-
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.planner.logical.DrillTable;
-import org.apache.drill.exec.store.SchemaProvider;
-
-public class FileSystemSchema implements Schema, ExpandingConcurrentMap.MapValueFactory<String, DrillTable>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchema.class);
-
- private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<String, DrillTable>(this);
-
- private final SchemaPlus parentSchema;
- private final String name;
- private final Expression expression = new DefaultExpression(Object.class);
- private final SchemaProvider schemaProvider;
- private final StorageEngineConfig config;
-
- public FileSystemSchema(StorageEngineConfig config, SchemaProvider schemaProvider, SchemaPlus parentSchema, String name) {
- super();
- this.parentSchema = parentSchema;
- this.name = name;
- this.schemaProvider = schemaProvider;
- this.config = config;
- }
-
- @Override
- public Schema getSubSchema(String name) {
- return null;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Expression getExpression() {
- return expression;
- }
-
- @Override
- public Collection<TableFunction> getTableFunctions(String name) {
- return Collections.emptyList();
- }
-
- @Override
- public SchemaPlus getParentSchema() {
- return parentSchema;
- }
-
- @Override
- public Set<String> getTableNames() {
- return tables.keySet();
- }
-
- @Override
- public Set<String> getTableFunctionNames() {
- return Collections.emptySet();
- }
-
- @Override
- public Set<String> getSubSchemaNames() {
- return Collections.emptySet();
- }
-
- @Override
- public boolean isMutable() {
- return true;
- }
-
- @Override
- public DrillTable getTable(String name) {
- return tables.get(name);
- }
-
- @Override
- public DrillTable create(String key) {
- Object selection = schemaProvider.getSelectionBaseOnName(key);
- if(selection == null) return null;
-
- return new DrillTable(name, this.name, selection, config);
- }
-
- @Override
- public void destroy(DrillTable value) {
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 60a7d5c..0e6b7bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -24,26 +24,22 @@ import java.util.Collection;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FunctionRegistry;
-import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.planner.logical.StorageEngines;
-import org.apache.drill.exec.planner.sql.DrillSchemaFactory;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.rpc.control.Controller;
import org.apache.drill.exec.rpc.control.WorkEventBus;
import org.apache.drill.exec.rpc.data.DataConnectionCreator;
-import org.apache.drill.exec.store.StorageEngine;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory;
+import org.apache.drill.exec.store.StoragePlugin;
import com.codahale.metrics.MetricRegistry;
-import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
-import com.google.common.io.Resources;
public class DrillbitContext {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillbitContext.class);
@@ -55,13 +51,12 @@ public class DrillbitContext {
private final DataConnectionCreator connectionsPool;
private final DistributedCache cache;
private final DrillbitEndpoint endpoint;
- private final StorageEngineRegistry storageEngineRegistry;
+ private final StoragePluginRegistry storagePlugins;
private final OperatorCreatorRegistry operatorCreatorRegistry;
private final Controller controller;
private final WorkEventBus workBus;
private final FunctionImplementationRegistry functionRegistry;
private final FunctionRegistry functionRegistryX;
- private final DrillSchemaFactory factory;
public DrillbitContext(DrillbitEndpoint endpoint, BootStrapContext context, ClusterCoordinator coord, Controller controller, DataConnectionCreator connectionsPool, DistributedCache cache, WorkEventBus workBus) {
super();
@@ -76,27 +71,11 @@ public class DrillbitContext {
this.connectionsPool = connectionsPool;
this.cache = cache;
this.endpoint = endpoint;
- this.storageEngineRegistry = new StorageEngineRegistry(this);
- this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storageEngineRegistry);
+ this.storagePlugins = new StoragePluginRegistry(this);
+ this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storagePlugins);
this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig());
this.functionRegistry = new FunctionImplementationRegistry(context.getConfig());
-
- DrillSchemaFactory factory = null;
- try{
- String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
- StorageEngines engines = context.getConfig().getMapper().readValue(enginesData, StorageEngines.class);
- factory = new DrillSchemaFactory(engines, context.getConfig());
- }catch(Exception e){
- logger.error("Failure reading storage engines data. Creating empty list of schemas.", e);
- factory = DrillSchemaFactory.createEmpty();
- }
- this.factory = factory;
this.functionRegistryX = new FunctionRegistry(context.getConfig());
-
- }
-
- public DrillSchemaFactory getSchemaFactory(){
- return factory;
}
public FunctionRegistry getFunctionRegistry(){
@@ -131,8 +110,8 @@ public class DrillbitContext {
return operatorCreatorRegistry;
}
- public StorageEngine getStorageEngine(StorageEngineConfig config) throws ExecutionSetupException {
- return storageEngineRegistry.getEngine(config);
+ public StoragePluginRegistry getStorage(){
+ return this.storagePlugins;
}
public NioEventLoopGroup getBitLoopGroup(){
@@ -160,5 +139,8 @@ public class DrillbitContext {
return reader;
}
+ public DrillSchemaFactory getSchemaFactory(){
+ return storagePlugins.getSchemaFactory();
+ }
}