You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/08/16 03:44:46 UTC
[08/27] Initial Parquet commit. Suports INT, LONG, FLOAT, DOUBLE,
distributed scheduling.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
new file mode 100644
index 0000000..75a52c5
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java
@@ -0,0 +1,594 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store;
+
+import com.beust.jcommander.internal.Lists;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.server.BootStrapContext;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.RemoteServiceSet;
+
+import org.apache.drill.exec.store.parquet.ParquetStorageEngine;
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+import parquet.bytes.BytesInput;
+import parquet.column.ColumnDescriptor;
+import parquet.column.page.Page;
+import parquet.column.page.PageReadStore;
+import parquet.column.page.PageReader;
+import parquet.hadoop.Footer;
+
+import parquet.hadoop.ParquetFileWriter;
+import parquet.hadoop.metadata.CompressionCodecName;
+import parquet.hadoop.metadata.ParquetMetadata;
+import parquet.schema.MessageType;
+import parquet.schema.MessageTypeParser;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static parquet.column.Encoding.PLAIN;
+
+public class ParquetRecordReaderTest {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StorageEngineRegistry.class);
+
+ private boolean VERBOSE_DEBUG = false;
+
+ @Test
+ public void testMultipleRowGroupsAndReads() throws Exception {
+ testParquetFullEngine(true, "/parquet_scan_screen.json", "/tmp/testParquetFile_many_types_3", 2, numberRowGroups, recordsPerRowGroup);
+ }
+
+ @Test
+ public void testMultipleRowGroupsAndReadsEvent() throws Exception {
+ testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/testParquetFile_many_types_3", 2, numberRowGroups, recordsPerRowGroup);
+ }
+
+ int numberRowGroups = 20;
+ static int recordsPerRowGroup = 3000000;
+
+ // 10 mb per page
+ static int bytesPerPage = 1024 * 1024 * 10;
+ // { 00000001, 00000010, 00000100, 00001000, 00010000, ... }
+ byte[] bitFields = {1, 2, 4, 8, 16, 32, 64, -128};
+ static final byte allBitsTrue = -1;
+ static final byte allBitsFalse = 0;
+ static final byte[] varLen1 = {50, 51, 52, 53, 54, 55, 56};
+ static final byte[] varLen2 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
+ static final byte[] varLen3 = {100, 99, 98};
+
+ private static class FieldInfo {
+
+ String parquetType;
+ String name;
+ int bitLength;
+ int numberOfPages;
+ Object[] values;
+ TypeProtos.MinorType type;
+
+ FieldInfo(String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type){
+ this.parquetType = parquetType;
+ this.name = name;
+ this.bitLength = bitLength;
+ this.numberOfPages = Math.max(1, (int) Math.ceil(recordsPerRowGroup * bitLength / 8.0 / bytesPerPage));
+ this.values = values;
+ // generator is designed to use 3 values
+ assert values.length == 3;
+ this.type = type;
+ }
+ }
+
+
+ private static HashMap<String, FieldInfo> fields = new HashMap<>();
+ static {
+ Object[] intVals = {-200, 100, Integer.MAX_VALUE };
+ Object[] longVals = { -5000l, 5000l, Long.MAX_VALUE};
+ Object[] floatVals = { 1.74f, Float.MAX_VALUE, Float.MIN_VALUE};
+ Object[] doubleVals = {100.45d, Double.MAX_VALUE, Double.MIN_VALUE,};
+ Object[] boolVals = {false, false, true};
+ Object[] binVals = { varLen1, varLen2, varLen3};
+ Object[] bin2Vals = { varLen3, varLen2, varLen1};
+ fields.put("integer/", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT));
+ fields.put("bigInt/", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT));
+ fields.put("f/", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4));
+ fields.put("d/", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8));
+// fields.put("b/", new FieldInfo("binary", "b", 1, boolVals, TypeProtos.MinorType.BIT));
+ fields.put("bin/", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY));
+ fields.put("bin2/", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY));
+ }
+
+
+ private String getResource(String resourceName) {
+ return "resource:" + resourceName;
+ }
+
+ public void generateParquetFile(String filename, int numberRowGroups, int recordsPerRowGroup) throws Exception {
+
+ int currentBooleanByte = 0;
+ WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7);
+
+ Configuration configuration = new Configuration();
+ configuration.set(ParquetStorageEngine.HADOOP_DEFAULT_NAME, "file:///");
+ //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
+
+ FileSystem fs = FileSystem.get(configuration);
+ Path path = new Path(filename);
+ if (fs.exists(path)) fs.delete(path, false);
+
+
+ String messageSchema = "message m {";
+ for (FieldInfo fieldInfo : fields.values()) {
+ messageSchema += " required " + fieldInfo.parquetType + " " + fieldInfo.name + ";";
+ }
+ // remove the last semicolon, java really needs a join method for strings...
+ // TODO - nvm apparently it requires a semicolon after every field decl, might want to file a bug
+ //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
+ messageSchema += "}";
+
+ MessageType schema = MessageTypeParser.parseMessageType(messageSchema);
+
+ CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
+ ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
+ w.start();
+ HashMap<String, Integer> columnValuesWritten = new HashMap();
+ int valsWritten;
+ for (int k = 0; k < numberRowGroups; k++){
+ w.startBlock(1);
+
+ for (FieldInfo fieldInfo : fields.values()) {
+
+ if ( ! columnValuesWritten.containsKey(fieldInfo.name)){
+ columnValuesWritten.put((String) fieldInfo.name, 0);
+ valsWritten = 0;
+ } else {
+ valsWritten = columnValuesWritten.get(fieldInfo.name);
+ }
+
+ String[] path1 = {(String) fieldInfo.name};
+ ColumnDescriptor c1 = schema.getColumnDescription(path1);
+
+ w.startColumn(c1, recordsPerRowGroup, codec);
+ int valsPerPage = (int) Math.ceil(recordsPerRowGroup / (float) ((int) fieldInfo.numberOfPages));
+ byte[] bytes;
+ // for variable length binary fields
+ int bytesNeededToEncodeLength = 4;
+ if ((int) fieldInfo.bitLength > 0) {
+ bytes = new byte[(int) Math.ceil(valsPerPage * (int) fieldInfo.bitLength / 8.0)];
+ } else {
+ // the twelve at the end is to account for storing a 4 byte length with each value
+ int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength;
+ // used for the case where there is a number of values in this row group that is not divisible by 3
+ int leftOverBytes = 0;
+ if ( valsPerPage % 3 > 0 ) leftOverBytes += ((byte[])fieldInfo.values[1]).length + 4;
+ if ( valsPerPage % 3 > 1 ) leftOverBytes += ((byte[])fieldInfo.values[2]).length + 4;
+ bytes = new byte[valsPerPage / 3 * totalValLength + leftOverBytes];
+ }
+ int bytesPerPage = (int) (valsPerPage * ((int) fieldInfo.bitLength / 8.0));
+ int bytesWritten = 0;
+ for (int z = 0; z < (int) fieldInfo.numberOfPages; z++, bytesWritten = 0) {
+ for (int i = 0; i < valsPerPage; i++) {
+ //System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : ""));
+ if (fieldInfo.values[0] instanceof Boolean) {
+
+ bytes[currentBooleanByte] |= bitFields[booleanBitCounter.val] & ((boolean) fieldInfo.values[valsWritten % 3]
+ ? allBitsTrue : allBitsFalse);
+ booleanBitCounter.increment();
+ if (booleanBitCounter.val == 0) {
+ currentBooleanByte++;
+ }
+ valsWritten++;
+ if (currentBooleanByte > bytesPerPage) break;
+ } else {
+ if (fieldInfo.values[valsWritten % 3] instanceof byte[]){
+ System.arraycopy(ByteArrayUtil.toByta(((byte[])fieldInfo.values[valsWritten % 3]).length),
+ 0, bytes, bytesWritten, bytesNeededToEncodeLength);
+ System.arraycopy(fieldInfo.values[valsWritten % 3],
+ 0, bytes, bytesWritten + bytesNeededToEncodeLength, ((byte[])fieldInfo.values[valsWritten % 3]).length);
+ bytesWritten += ((byte[])fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength;
+ }
+ else{
+ System.arraycopy( ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]),
+ 0, bytes, i * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8);
+ }
+ valsWritten++;
+ }
+
+ }
+ w.writeDataPage((int)(recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length, BytesInput.from(bytes), PLAIN, PLAIN, PLAIN);
+ currentBooleanByte = 0;
+ }
+ w.endColumn();
+ columnValuesWritten.remove((String) fieldInfo.name);
+ columnValuesWritten.put((String) fieldInfo.name, valsWritten);
+ }
+
+ w.endBlock();
+ }
+ w.end(new HashMap<String, String>());
+ logger.debug("Finished generating parquet file.");
+ }
+
+ private class ParquetResultListener implements UserResultsListener {
+ private Vector<QueryResultBatch> results = new Vector<QueryResultBatch>();
+ private SettableFuture<List<QueryResultBatch>> future = SettableFuture.create();
+ int count = 0;
+ RecordBatchLoader batchLoader;
+ byte[] bytes;
+
+ int batchCounter = 1;
+ int columnValCounter = 0;
+ int i = 0;
+ FieldInfo currentField;
+ HashMap<String, Integer> valuesChecked = new HashMap();
+
+ ParquetResultListener(RecordBatchLoader batchLoader){
+ this.batchLoader = batchLoader;
+ }
+
+ @Override
+ public void submissionFailed(RpcException ex) {
+ logger.debug("Submission failed.", ex);
+ future.setException(ex);
+ }
+
+ @Override
+ public void resultArrived(QueryResultBatch result) {
+ logger.debug("result arrived in test batch listener.");
+ int columnValCounter = 0;
+ int i = 0;
+ FieldInfo currentField;
+ count += result.getHeader().getRowCount();
+ boolean schemaChanged = false;
+ try {
+ schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData());
+ } catch (SchemaChangeException e) {
+ e.printStackTrace();
+ }
+
+ int recordCount = 0;
+ // print headers.
+ if (schemaChanged) {
+ } // do not believe any change is needed for when the schema changes, with the current mock scan use case
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector vv = vw.getValueVector();
+ currentField = fields.get(vv.getField().getName());
+ if (VERBOSE_DEBUG){
+ System.out.println("\n" + (String) currentField.name);
+ }
+ if ( ! valuesChecked.containsKey(vv.getField().getName())){
+ valuesChecked.put(vv.getField().getName(), 0);
+ columnValCounter = 0;
+ } else {
+ columnValCounter = valuesChecked.get(vv.getField().getName());
+ }
+ for (int j = 0; j < ((BaseDataValueVector)vv).getAccessor().getValueCount(); j++) {
+ if (VERBOSE_DEBUG){
+ System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
+ }
+ assertField(vv, j, (TypeProtos.MinorType) currentField.type,
+ currentField.values[columnValCounter % 3], (String) currentField.name + "/");
+ columnValCounter++;
+ }
+ if (VERBOSE_DEBUG){
+ System.out.println("\n" + ((BaseDataValueVector)vv).getAccessor().getValueCount());
+ }
+ valuesChecked.remove(vv.getField().getName());
+ valuesChecked.put(vv.getField().getName(), columnValCounter);
+ }
+
+ if (VERBOSE_DEBUG){
+ for (i = 0; i < batchLoader.getRecordCount(); i++) {
+ recordCount++;
+ if (i % 50 == 0){
+ System.out.println();
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(pad(v.getField().getName(), 20) + " ");
+
+ }
+ System.out.println();
+ System.out.println();
+ }
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(pad(v.getAccessor().getObject(i).toString(), 20) + " ");
+ }
+ System.out.println(
+
+ );
+ }
+ }
+ batchCounter++;
+ if(result.getHeader().getIsLastChunk()){
+ future.set(results);
+ }
+ }
+
+ public List<QueryResultBatch> getResults() throws RpcException{
+ try{
+ return future.get();
+ }catch(Throwable t){
+ throw RpcException.mapException(t);
+ }
+ }
+ }
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngineEventBased(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, int numberRowGroups, int recordsPerRowGroup) throws Exception{
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ if (generateNew) generateParquetFile(filename, numberRowGroups, recordsPerRowGroup);
+
+ DrillConfig config = DrillConfig.create();
+
+ try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+ bit1.run();
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8));
+ int count = 0;
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
+ ParquetResultListener resultListener = new ParquetResultListener(batchLoader);
+ client.runQuery(UserProtos.QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8), resultListener);
+ }
+ }
+
+
+ // specific tests should call this method, but it is not marked as a test itself intentionally
+ public void testParquetFullEngine(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, int numberRowGroups, int recordsPerRowGroup) throws Exception{
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ if (generateNew) generateParquetFile(filename, numberRowGroups, recordsPerRowGroup);
+
+ DrillConfig config = DrillConfig.create();
+
+ try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
+ long A = System.nanoTime();
+ bit1.run();
+ long B = System.nanoTime();
+ client.connect();
+ long C = System.nanoTime();
+ System.out.println( new SimpleDateFormat("mm:ss S").format(new Date()) + " :Start query");
+ List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_screen.json"), Charsets.UTF_8));
+// List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
+ long D = System.nanoTime();
+ System.out.println(String.format("Took %f s to start drillbit", (float)(B-A) / 1E9));
+ System.out.println(String.format("Took %f s to connect", (float)(C-B) / 1E9));
+ System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
+ //List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
+ int count = 0;
+// RecordBatchLoader batchLoader = new RecordBatchLoader(new BootStrapContext(config).getAllocator());
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
+ byte[] bytes;
+
+ int batchCounter = 1;
+ int columnValCounter = 0;
+ int i = 0;
+ FieldInfo currentField;
+ HashMap<String, Integer> valuesChecked = new HashMap();
+ for(QueryResultBatch b : results){
+ count += b.getHeader().getRowCount();
+ boolean schemaChanged = batchLoader.load(b.getHeader().getDef(), b.getData());
+
+ int recordCount = 0;
+ // print headers.
+ if (schemaChanged) {
+ } // do not believe any change is needed for when the schema changes, with the current mock scan use case
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector vv = vw.getValueVector();
+ currentField = fields.get(vv.getField().getName());
+ if (VERBOSE_DEBUG){
+ System.out.println("\n" + (String) currentField.name);
+ }
+ if ( ! valuesChecked.containsKey(vv.getField().getName())){
+ valuesChecked.put(vv.getField().getName(), 0);
+ columnValCounter = 0;
+ } else {
+ columnValCounter = valuesChecked.get(vv.getField().getName());
+ }
+ for (int j = 0; j < ((BaseDataValueVector)vv).getAccessor().getValueCount(); j++) {
+ if (VERBOSE_DEBUG){
+ System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
+ }
+ assertField(vv, j, (TypeProtos.MinorType) currentField.type,
+ currentField.values[columnValCounter % 3], (String) currentField.name + "/");
+ columnValCounter++;
+ }
+ if (VERBOSE_DEBUG){
+ System.out.println("\n" + ((BaseDataValueVector)vv).getAccessor().getValueCount());
+ }
+ valuesChecked.remove(vv.getField().getName());
+ valuesChecked.put(vv.getField().getName(), columnValCounter);
+ }
+
+ if (VERBOSE_DEBUG){
+ for (i = 1; i < batchLoader.getRecordCount(); i++) {
+ recordCount++;
+ if (i % 50 == 0){
+ System.out.println();
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(pad(v.getField().getName(), 20) + " ");
+
+ }
+ System.out.println();
+ System.out.println();
+ }
+
+ for (VectorWrapper vw : batchLoader) {
+ ValueVector v = vw.getValueVector();
+ System.out.print(pad(v.getAccessor().getObject(i).toString(), 20) + " ");
+ }
+ System.out.println(
+
+ );
+ }
+ }
+ batchCounter++;
+ }
+ for (String s : valuesChecked.keySet()) {
+ assertEquals("Record count incorrect for column: " + s, recordsPerRowGroup * numberRowGroups * numberOfTimesRead, (long) valuesChecked.get(s));
+ }
+ assert valuesChecked.keySet().size() > 0;
+ }
+ }
+
+ public String pad(String value, int length) {
+ return pad(value, length, " ");
+ }
+
+ public String pad(String value, int length, String with) {
+ StringBuilder result = new StringBuilder(length);
+ result.append(value);
+
+ while (result.length() < length) {
+ result.insert(0, with);
+ }
+
+ return result.toString();
+ }
+
+ class MockOutputMutator implements OutputMutator {
+ List<MaterializedField> removedFields = Lists.newArrayList();
+ List<ValueVector> addFields = Lists.newArrayList();
+
+ @Override
+ public void removeField(MaterializedField field) throws SchemaChangeException {
+ removedFields.add(field);
+ }
+
+ @Override
+ public void addField(ValueVector vector) throws SchemaChangeException {
+ addFields.add(vector);
+ }
+
+ @Override
+ public void removeAllFields() {
+ addFields.clear();
+ }
+
+ @Override
+ public void setNewSchema() throws SchemaChangeException {
+ }
+
+ List<MaterializedField> getRemovedFields() {
+ return removedFields;
+ }
+
+ List<ValueVector> getAddFields() {
+ return addFields;
+ }
+ }
+
+ private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) {
+ assertField(valueVector, index, expectedMinorType, value, name, 0);
+ }
+
+ private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
+// UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
+// SchemaDefProtos.FieldDef def = metadata.getDef();
+// assertEquals(expectedMinorType, def.getMajorType().getMinorType());
+// assertEquals(name, def.getNameList().get(0).getName());
+// assertEquals(parentFieldId, def.getParentId());
+
+ if (expectedMinorType == TypeProtos.MinorType.MAP) {
+ return;
+ }
+
+ T val = (T) valueVector.getAccessor().getObject(index);
+ if (val instanceof byte[]) {
+ assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
+ } else {
+ assertEquals(value, val);
+ }
+ }
+
+ private class WrapAroundCounter {
+
+ int maxVal;
+ int val;
+
+ public WrapAroundCounter(int maxVal) {
+ this.maxVal = maxVal;
+ }
+
+ public int increment() {
+ val++;
+ if (val > maxVal) {
+ val = 0;
+ }
+ return val;
+ }
+
+ public void reset() {
+ val = 0;
+ }
+
+ }
+
+ private void validateFooters(final List<Footer> metadata) {
+ logger.debug(metadata.toString());
+ assertEquals(3, metadata.size());
+ for (Footer footer : metadata) {
+ final File file = new File(footer.getFile().toUri());
+ assertTrue(file.getName(), file.getName().startsWith("part"));
+ assertTrue(file.getPath(), file.exists());
+ final ParquetMetadata parquetMetadata = footer.getParquetMetadata();
+ assertEquals(2, parquetMetadata.getBlocks().size());
+ final Map<String, String> keyValueMetaData = parquetMetadata.getFileMetaData().getKeyValueMetaData();
+ assertEquals("bar", keyValueMetaData.get("foo"));
+ assertEquals(footer.getFile().getName(), keyValueMetaData.get(footer.getFile().getName()));
+ }
+ }
+
+ private void validateContains(MessageType schema, PageReadStore pages, String[] path, int values, BytesInput bytes)
+ throws IOException {
+ PageReader pageReader = pages.getPageReader(schema.getColumnDescription(path));
+ Page page = pageReader.readPage();
+ assertEquals(values, page.getValueCount());
+ assertArrayEquals(bytes.toByteArray(), page.getBytes().toByteArray());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
new file mode 100644
index 0000000..cff7c86
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
@@ -0,0 +1,229 @@
+package org.apache.drill.exec.store;
+
+import com.google.common.collect.ImmutableRangeMap;
+import com.google.common.collect.Range;
+import junit.framework.Assert;
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+import org.apache.drill.exec.physical.EndpointAffinity;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.parquet.ParquetGroupScan;
+import org.apache.drill.exec.store.parquet.ParquetRowGroupScan;
+import org.apache.drill.exec.store.parquet.ParquetStorageEngine;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertNull;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+
+public class TestAffinityCalculator {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAffinityCalculator.class);
+
+ String port = "1234";
+ final String path = "path";
+
+ public BlockLocation[] buildBlockLocations(String[] hosts, long blockSize) {
+ String[] names = new String[hosts.length];
+
+ for (int i = 0; i < hosts.length; i++) {
+ hosts[i] = "host" + i;
+ names[i] = "host:" + port;
+ }
+
+ BlockLocation[] blockLocations = new BlockLocation[3];
+ blockLocations[0] = new BlockLocation(new String[]{names[0], names[1], names[2]}, new String[]{hosts[0], hosts[1], hosts[2]}, 0, blockSize);
+ blockLocations[1] = new BlockLocation(new String[]{names[0], names[2], names[3]}, new String[]{hosts[0], hosts[2], hosts[3]}, blockSize, blockSize);
+ blockLocations[2] = new BlockLocation(new String[]{names[0], names[1], names[3]}, new String[]{hosts[0], hosts[1], hosts[3]}, blockSize*2, blockSize);
+
+ return blockLocations;
+ }
+
+ public BlockLocation[] buildBlockLocations2(String[] hosts, long blockSize) {
+ String[] names = new String[hosts.length];
+
+ for (int i = 0; i < hosts.length; i++) {
+ hosts[i] = "host" + i;
+ names[i] = "host:" + port;
+ }
+
+ BlockLocation[] blockLocations = new BlockLocation[4];
+ blockLocations[0] = new BlockLocation(new String[]{names[0]}, new String[]{hosts[0]}, 0, blockSize);
+ blockLocations[1] = new BlockLocation(new String[]{names[1]}, new String[]{hosts[1]}, blockSize, blockSize);
+ blockLocations[3] = new BlockLocation(new String[]{names[3]}, new String[]{hosts[3]}, blockSize*2, blockSize);
+ blockLocations[2] = new BlockLocation(new String[]{names[2]}, new String[]{hosts[2]}, blockSize*3, blockSize);
+
+ return blockLocations;
+ }
+ public void buildRowGroups(LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups, int numberOfBlocks, long blockSize, int numberOfRowGroups) {
+ long rowGroupSize = numberOfBlocks * blockSize / numberOfRowGroups;
+
+ rowGroups.clear();
+
+ for (int i = 0; i < numberOfRowGroups; i++) {
+ rowGroups.add(new ParquetGroupScan.RowGroupInfo(path, (long)i*rowGroupSize, (long)rowGroupSize, i));
+ }
+ }
+
+ public LinkedList<CoordinationProtos.DrillbitEndpoint> buildEndpoints(int numberOfEndpoints) {
+ LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = new LinkedList();
+
+ for (int i = 0; i < numberOfEndpoints; i++) {
+ endPoints.add(CoordinationProtos.DrillbitEndpoint.newBuilder().setAddress("host" + i).build());
+ }
+ return endPoints;
+ }
+
+ @Test
+ public void testSetEndpointBytes(@Injectable final FileSystem fs, @Injectable final FileStatus file) throws Throwable{
+ final long blockSize = 256*1024*1024;
+ LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups = new LinkedList<>();
+ int numberOfHosts = 4;
+ int numberOfBlocks = 3;
+ String port = "1234";
+ String[] hosts = new String[numberOfHosts];
+
+ final BlockLocation[] blockLocations = buildBlockLocations(hosts, blockSize);
+ final LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = buildEndpoints(numberOfHosts);
+ buildRowGroups(rowGroups, numberOfBlocks, blockSize, 3);
+
+ new NonStrictExpectations() {{
+ fs.getFileBlockLocations(file, 0, 3*blockSize); result = blockLocations;
+ fs.getFileStatus(new Path(path)); result = file;
+ file.getLen(); result = 3*blockSize;
+ }};
+
+
+ AffinityCalculator ac = new AffinityCalculator(path, fs, endPoints);
+ for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
+ ac.setEndpointBytes(rowGroup);
+ }
+ ParquetGroupScan.RowGroupInfo rg = rowGroups.get(0);
+ Long b = rg.getEndpointBytes().get(endPoints.get(0));
+ assertEquals(blockSize,b.longValue());
+ b = rg.getEndpointBytes().get(endPoints.get(3));
+ assertNull(b);
+
+ buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2);
+
+ ac = new AffinityCalculator(path, fs, endPoints);
+ for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
+ ac.setEndpointBytes(rowGroup);
+ }
+ rg = rowGroups.get(0);
+ b = rg.getEndpointBytes().get(endPoints.get(0));
+ assertEquals(blockSize*3/2,b.longValue());
+ b = rg.getEndpointBytes().get(endPoints.get(3));
+ assertEquals(blockSize / 2, b.longValue());
+
+ buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6);
+
+ ac = new AffinityCalculator(path, fs, endPoints);
+ for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
+ ac.setEndpointBytes(rowGroup);
+ }
+ rg = rowGroups.get(0);
+ b = rg.getEndpointBytes().get(endPoints.get(0));
+ assertEquals(blockSize/2,b.longValue());
+ b = rg.getEndpointBytes().get(endPoints.get(3));
+ assertNull(b);
+ }
+
+ @Test
+ public void testBuildRangeMap() {
+ BlockLocation[] blocks = buildBlockLocations(new String[4], 256*1024*1024);
+ long tA = System.nanoTime();
+ ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long,BlockLocation>();
+ for (BlockLocation block : blocks) {
+ long start = block.getOffset();
+ long end = start + block.getLength();
+ Range<Long> range = Range.closedOpen(start, end);
+ blockMapBuilder = blockMapBuilder.put(range, block);
+ }
+ ImmutableRangeMap<Long,BlockLocation> map = blockMapBuilder.build();
+ long tB = System.nanoTime();
+ System.out.println(String.format("Took %f ms to build range map", (float)(tB - tA) / 1e6));
+ }
+ /*
+ @Test
+ public void testApplyAssignments(@Injectable final DrillbitContext context, @Injectable final ParquetStorageEngine engine,
+ @Injectable final FileSystem fs, @Injectable final FileStatus file) throws IOException {
+
+ final long blockSize = 256*1024*1024;
+ LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups = new LinkedList<>();
+ int numberOfHosts = 4;
+ int numberOfBlocks = 4;
+ String port = "1234";
+ String[] hosts = new String[numberOfHosts];
+
+ final BlockLocation[] blockLocations = buildBlockLocations2(hosts, blockSize);
+ final LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = buildEndpoints(numberOfHosts);
+
+ new NonStrictExpectations() {{
+ engine.getFileSystem(); result = fs;
+ engine.getContext(); result = context;
+ context.getBits(); result = endPoints;
+ fs.getFileBlockLocations(file, 0, 3*blockSize); result = blockLocations;
+ fs.getFileStatus(new Path(path)); result = file;
+ file.getLen(); result = 3*blockSize;
+ }};
+
+ buildRowGroups(rowGroups, numberOfBlocks, blockSize, 4);
+ ParquetGroupScan scan = new ParquetGroupScan(rowGroups, engine);
+
+ List<EndpointAffinity> affinities = scan.getOperatorAffinity();
+
+ for (EndpointAffinity affinity : affinities) {
+ CoordinationProtos.DrillbitEndpoint db = affinity.getEndpoint();
+ assertEquals((float)0.25, affinity.getAffinity(), .01);
+ }
+
+ scan.applyAssignments(endPoints);
+
+ for (int i = 0; i < endPoints.size(); i++) {
+ List<ParquetRowGroupScan.RowGroupReadEntry> rowGroupReadEntries = scan.getSpecificScan(i).getRowGroupReadEntries();
+ assertEquals(1, rowGroupReadEntries.size());
+ switch(i) {
+ case 0: assertEquals(0,rowGroupReadEntries.get(0).getRowGroupIndex());
+ break;
+ case 1: assertEquals(1,rowGroupReadEntries.get(0).getRowGroupIndex());
+ break;
+ case 2: assertEquals(3,rowGroupReadEntries.get(0).getRowGroupIndex());
+ break;
+ case 3: assertEquals(2,rowGroupReadEntries.get(0).getRowGroupIndex());
+ break;
+ }
+ }
+
+ scan.applyAssignments(endPoints.subList(2,4));
+
+ List<ParquetRowGroupScan.RowGroupReadEntry> rowGroupReadEntries = scan.getSpecificScan(0).getRowGroupReadEntries();
+ assertEquals(2, rowGroupReadEntries.size());
+ assertEquals(3,rowGroupReadEntries.get(0).getRowGroupIndex());
+
+ rowGroupReadEntries = scan.getSpecificScan(1).getRowGroupReadEntries();
+ assertEquals(2, rowGroupReadEntries.size());
+ assertEquals(2,rowGroupReadEntries.get(0).getRowGroupIndex());
+
+ LinkedList<CoordinationProtos.DrillbitEndpoint> dupList = new LinkedList<>();
+ dupList.add(endPoints.get(0));
+ dupList.add(endPoints.get(0));
+ scan.applyAssignments(dupList);
+
+ rowGroupReadEntries = scan.getSpecificScan(0).getRowGroupReadEntries();
+ assertEquals(2, rowGroupReadEntries.size());
+ rowGroupReadEntries = scan.getSpecificScan(1).getRowGroupReadEntries();
+ assertEquals(2, rowGroupReadEntries.size());
+ }
+ */
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
new file mode 100644
index 0000000..e2a00f1
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
@@ -0,0 +1,56 @@
+package org.apache.drill.exec.store;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.google.common.io.Resources;
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.store.parquet.ParquetGroupScan;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.LinkedList;
+import java.util.List;
+
+import static junit.framework.Assert.assertNull;
+import static org.junit.Assert.assertEquals;
+
+public class TestParquetPhysicalPlan {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetPhysicalPlan.class);
+
+ //public String fileName = "/physical_test2.json";
+ public String fileName = "parquet_scan_union_screen_physical.json";
+
+ @Test
+ @Ignore
+ public void testParseParquetPhysicalPlan() throws Exception {
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ DrillConfig config = DrillConfig.create();
+
+ try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+ bit1.run();
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
+ System.out.println(String.format("Got %d results", results.size()));
+ client.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf b/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf
index 771a2fd..4829d34 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/drill-module.conf
@@ -20,10 +20,10 @@ drill.exec: {
context: "drillbit"
},
zk: {
- connect: "localhost:2181",
+ connect: "10.10.30.52:5181",
root: "/drill",
refresh: 500,
- timeout: 1000,
+ timeout: 5000,
retry: {
count: 7200,
delay: 500
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/filter/test1.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/filter/test1.json b/sandbox/prototype/exec/java-exec/src/test/resources/filter/test1.json
index c9b367f..4f82145 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/filter/test1.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/filter/test1.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4Equal.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4Equal.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4Equal.json
index 612b2b4..7ebe3dd 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4Equal.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4Equal.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThan.json
index dcbad4c..de300d7 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThanEqual.json
index b5c0d54..51f0a13 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4GreaterThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThan.json
index 06aafd9..95ef169 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThanEqual.json
index 92a27ac..b58d3b6 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4LessThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4NotEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4NotEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4NotEqual.json
index 0b97545..ba001d4 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4NotEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float4NotEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8Equal.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8Equal.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8Equal.json
index 1aa93ea..9edfd51 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8Equal.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8Equal.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThan.json
index ddccef1..08bb328 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThanEqual.json
index 665759f..67afb47 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8GreaterThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThan.json
index b86b118..af485b2 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThanEqual.json
index 7f622fd..6f5a21e 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8LessThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8NotEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8NotEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8NotEqual.json
index fdff828..cf3d053 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8NotEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/float8NotEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/intEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intEqual.json
index 88fb2eb..a1d96ec 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThan.json
index 015bf36..9e21f4b 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThanEqual.json
index 167d4ba..9692f82 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intGreaterThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThan.json
index 473c08f..d2ec94a 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThanEqual.json
index 418e5f0..834cf7e 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intLessThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/intNotEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intNotEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intNotEqual.json
index f0e8578..ceb2913 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/intNotEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/intNotEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/longEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longEqual.json
index 4b0e2b8..0a7827b 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThan.json
index 830c126..6d2a415 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThanEqual.json
index 56c0dd7..7dffbe8 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longGreaterThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThan.json
index 267a6c0..ace6eb0 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThanEqual.json
index e06d865..5900f73 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longLessThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/longNotEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longNotEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longNotEqual.json
index 71269a6..4db69c8 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/longNotEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/longNotEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json
index eca6d15..212c73a 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThan.json
index 7218449..c1d42fa 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThanEqual.json
index ed2456a..c560d0b 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntGreaterThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThan.json
index ebfee49..2cf2869 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThanEqual.json
index 98e6fcf..16b19d3 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntLessThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json
index a501f50..2e9778a 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json
index 60a00fb..ba9cba8 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThan.json
index c6da740..6a3fc33 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThanEqual.json
index a3f373c..daa34f9 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntGreaterThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThan.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThan.json
index 46eee38..0186e17 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThan.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThan.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThanEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThanEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThanEqual.json
index 77db318..2ee40eb 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThanEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntLessThanEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json
index 2cefc5c..494f938 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/testByteSubstring.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testByteSubstring.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testByteSubstring.json
index 3c246c5..299bb51 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testByteSubstring.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testByteSubstring.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNotNull.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNotNull.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNotNull.json
index f24d2c4..713d58f 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNotNull.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNotNull.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNull.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNull.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNull.json
index 850bdc5..e173adc 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNull.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testIsNull.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json
index 08a8ece..d0fe2b9 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstring.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstringNegative.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstringNegative.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstringNegative.json
index 4951913..12ff0f2 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstringNegative.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/testSubstringNegative.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/mock-scan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/mock-scan.json b/sandbox/prototype/exec/java-exec/src/test/resources/mock-scan.json
new file mode 100644
index 0000000..3660480
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/mock-scan.json
@@ -0,0 +1,31 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100000000, types: [
+ {name: "blue", type: "INT", mode: "REQUIRED"},
+ {name: "green", type: "INT", mode: "REQUIRED"}
+ ]},
+ {records: 100000000, types: [
+ {name: "blue", type: "INT", mode: "REQUIRED"},
+ {name: "green", type: "INT", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id: 4,
+ child: 1,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/parquet_scan_screen.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/parquet_scan_screen.json b/sandbox/prototype/exec/java-exec/src/test/resources/parquet_scan_screen.json
new file mode 100644
index 0000000..15d3936
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/parquet_scan_screen.json
@@ -0,0 +1,44 @@
+{
+ head:{
+ type:"APACHE_DRILL_LOGICAL",
+ version:"1",
+ generator:{
+ type:"manual",
+ info:"na"
+ }
+ },
+ storage:{
+ "parquet" :
+ {
+ "type":"parquet",
+ "dfsName" : "file:///"
+ }
+ },
+ query:[
+ {
+ @id:"1",
+ op:"scan",
+ memo:"initial_scan",
+ storageengine:"parquet",
+ selection: [
+ {
+ path: "/tmp/testParquetFile_many_types_3"
+ },
+ {
+ path: "/tmp/testParquetFile_many_types_3"
+ }
+ ]
+ },
+ {
+ @id:"2",
+ input: 1,
+ op: "store",
+ memo: "output sink",
+ target: {
+ file: "console:///stdout"
+ }
+
+ }
+
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json b/sandbox/prototype/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
new file mode 100644
index 0000000..954082c
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
@@ -0,0 +1,35 @@
+{
+ head : {
+ type : "APACHE_DRILL_PHYSICAL",
+ version : 1,
+ generator : {
+ type : "manual"
+ }
+ },
+ graph : [ {
+ pop : "parquet-scan",
+ @id : 1,
+ entries : [
+ {
+ path : "/tmp/testParquetFile_many_types_3"
+ },
+ {
+ path : "/tmp/testParquetFile_many_types_3"
+ }
+ ],
+ storageengine:{
+ "type":"parquet",
+ "dfsName" : "maprfs:///"
+ }
+ },
+ {
+ "@id": 2,
+ "child": 1,
+ "pop": "union-exchange"
+ },
+ {
+ pop : "screen",
+ @id : 3,
+ child : 2
+ } ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/physical_repeated_1.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/physical_repeated_1.json b/sandbox/prototype/exec/java-exec/src/test/resources/physical_repeated_1.json
index c26be01..71eff1d 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/physical_repeated_1.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/physical_repeated_1.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json b/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json
index 70a5115..2a7c935 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/project/test1.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/remover/test1.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/remover/test1.json b/sandbox/prototype/exec/java-exec/src/test/resources/remover/test1.json
index 7ef8fd6..3abe476 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/remover/test1.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/remover/test1.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/sort/one_key_sort.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/sort/one_key_sort.json b/sandbox/prototype/exec/java-exec/src/test/resources/sort/one_key_sort.json
index 3bd0b71..baabcb3 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/sort/one_key_sort.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/sort/one_key_sort.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 1000000, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/resources/sort/two_key_sort.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/sort/two_key_sort.json b/sandbox/prototype/exec/java-exec/src/test/resources/sort/two_key_sort.json
index 2394626..ab14002 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/sort/two_key_sort.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/sort/two_key_sort.json
@@ -9,7 +9,7 @@
graph:[
{
@id:1,
- pop:"mock-scan",
+ pop:"mock-sub-scan",
url: "http://apache.org",
entries:[
{records: 100, types: [
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/java-exec/src/test/sh/runbit
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/sh/runbit b/sandbox/prototype/exec/java-exec/src/test/sh/runbit
index 2885f7f..31d0729 100755
--- a/sandbox/prototype/exec/java-exec/src/test/sh/runbit
+++ b/sandbox/prototype/exec/java-exec/src/test/sh/runbit
@@ -5,5 +5,5 @@ PROJECT_ROOT=../../../
mvn dependency:build-classpath -f=$PROJECT_ROOT/pom.xml -Dmdep.outputFile=target/sh/cp.txt
CP=`cat $PROJECT_ROOT/target/sh/cp.txt`
CP=$CP:$PROJECT_ROOT/target/classes:$PROJECT_ROOT/target/test-classes
-java -javaagent:/src/jrebel/jrebel.jar -cp $CP org.apache.drill.exec.server.Drillbit
+java -XX:MaxDirectMemorySize=8192M -cp $CP org.apache.drill.exec.server.Drillbit
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
index 90f3374..384af2d 100644
--- a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
+++ b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
@@ -35,7 +35,6 @@ import org.apache.drill.exec.ref.rops.ROP;
import org.apache.drill.exec.ref.rops.ScanROP;
import org.apache.drill.exec.ref.rops.StoreROP;
import org.apache.drill.exec.ref.rops.UnionROP;
-import org.apache.drill.exec.ref.rops.UnionROP;
import org.apache.drill.exec.ref.rse.RSERegistry;
import org.apache.drill.exec.ref.rse.ReferenceStorageEngine;
import org.apache.drill.exec.ref.rse.ReferenceStorageEngine.ReadEntry;
@@ -106,7 +105,7 @@ class ROPConverter {
}
private ReferenceStorageEngine getEngine(String name){
- StorageEngineConfig config = plan.getStorageEngine(name);
+ StorageEngineConfig config = plan.getStorageEngineConfig(name);
if(config == null) throw new SetupException(String.format("Unable to find define logical plan of name [%s].", name));
ReferenceStorageEngine engine = engineRegistry.getEngine(config);
return engine;
@@ -118,7 +117,7 @@ class ROPConverter {
}
public void convertSpecific(Scan scan) throws SetupException {
- StorageEngineConfig engineConfig = plan.getStorageEngine(scan.getStorageEngine());
+ StorageEngineConfig engineConfig = plan.getStorageEngineConfig(scan.getStorageEngine());
ReferenceStorageEngine engine = engineRegistry.getEngine(engineConfig);
Collection<ReadEntry> readEntries;
try {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java
index aa8186d..6515a3a 100644
--- a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java
+++ b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ClasspathRSE.java
@@ -48,6 +48,11 @@ public class ClasspathRSE extends RSEBase {
@JsonTypeName("classpath")
public static class ClasspathRSEConfig extends StorageEngineConfigBase {
+ @Override
+ public boolean equals(Object o) {
+ // if fields are added to this class this method should be changed
+ return true;
+ }
}
public static class ClasspathInputConfig implements ReadEntry{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java
index 1570ea9..a914fe3 100644
--- a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java
+++ b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/ConsoleRSE.java
@@ -44,7 +44,13 @@ public class ConsoleRSE extends RSEBase {
public ConverterType type = ConverterType.JSON;
}
- @JsonTypeName("console") public static class ConsoleRSEConfig extends StorageEngineConfigBase {}
+ @JsonTypeName("console") public static class ConsoleRSEConfig extends StorageEngineConfigBase {
+ @Override
+ public boolean equals(Object o) {
+ // if fields are added to this class this method needs to be updated
+ return true;
+ }
+ }
public boolean supportsWrite() {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0a2f997f/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java
index 522191b..eece57d 100644
--- a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java
+++ b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rse/FileSystemRSE.java
@@ -72,6 +72,23 @@ public class FileSystemRSE extends RSEBase {
public FileSystemRSEConfig(@JsonProperty("root") String root) {
this.root = root;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FileSystemRSEConfig that = (FileSystemRSEConfig) o;
+
+ if (root != null ? !root.equals(that.root) : that.root != null) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return root != null ? root.hashCode() : 0;
+ }
}
public static class FileSystemInputConfig {