You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ti...@apache.org on 2018/07/30 21:46:10 UTC
[drill] branch master updated (471a866 -> 5b36adc)
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 471a866 [maven-release-plugin] prepare for next development iteration
new 24a1d55 DRILL-6641: Fix columnValueCounts in ParquetGroupScanStatistics when ParquetGroupScan has RowGroupInfo without column statistics
new 20ecab0 DRILL-6639: Exception happens while displaying operator profiles for some queries
new 5b36adc DRILL-6638: Fix TestE2EUnnestAndLateral tests introduced with DRILL-6546
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../drill/exec/store/mapr/db/MapRDBSubScan.java | 3 +-
.../native/client/src/protobuf/UserBitShared.pb.cc | 24 ++++--
.../native/client/src/protobuf/UserBitShared.pb.h | 12 ++-
.../apache/drill/exec/store/kudu/KuduWriter.java | 3 +-
.../drill/exec/store/mongo/MongoSubScan.java | 3 +-
.../drill/exec/store/openTSDB/OpenTSDBSubScan.java | 3 +-
.../exec/physical/config/IteratorValidator.java | 1 +
.../apache/drill/exec/physical/config/Values.java | 1 +
.../exec/server/rest/profile/OperatorWrapper.java | 4 +
.../drill/exec/store/dfs/easy/EasyWriter.java | 2 +-
.../exec/store/easy/json/JSONFormatPlugin.java | 2 +-
.../sequencefile/SequenceFileFormatPlugin.java | 3 +-
.../exec/store/httpd/HttpdLogFormatPlugin.java | 7 +-
.../drill/exec/store/image/ImageFormatPlugin.java | 3 +-
.../store/parquet/ParquetGroupScanStatistics.java | 26 +++----
.../drill/exec/store/pcap/PcapFormatPlugin.java | 2 +-
.../impl/lateraljoin/TestE2EUnnestAndLateral.java | 8 +-
.../org/apache/drill/exec/proto/UserBitShared.java | 88 ++++++++++++++++++++--
.../drill/exec/proto/beans/CoreOperatorType.java | 18 ++++-
protocol/src/main/protobuf/UserBitShared.proto | 8 ++
20 files changed, 176 insertions(+), 45 deletions(-)
[drill] 01/03: DRILL-6641: Fix columnValueCounts in
ParquetGroupScanStatistics when ParquetGroupScan has RowGroupInfo without
column statistics
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 24a1d55398daa7d40c329abe0e346c1c1226ddc7
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Fri Jul 27 13:49:09 2018 +0300
DRILL-6641: Fix columnValueCounts in ParquetGroupScanStatistics when ParquetGroupScan has RowGroupInfo without column statistics
closes #1406
---
.../store/parquet/ParquetGroupScanStatistics.java | 26 ++++++++++------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
index f7d5687..9381043 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.physical.base.GroupScan;
@@ -44,7 +45,7 @@ public class ParquetGroupScanStatistics {
// only for partition columns : value is unique for each partition
private Map<SchemaPath, TypeProtos.MajorType> partitionColTypeMap;
// total number of non-null value for each column in parquet files
- private Map<SchemaPath, Long> columnValueCounts;
+ private Map<SchemaPath, MutableLong> columnValueCounts;
// total number of rows (obtained from parquet footer)
private long rowCount;
@@ -61,7 +62,8 @@ public class ParquetGroupScanStatistics {
}
public long getColumnValueCount(SchemaPath column) {
- return columnValueCounts.containsKey(column) ? columnValueCounts.get(column) : 0;
+ MutableLong count = columnValueCounts.get(column);
+ return count != null ? count.getValue() : 0;
}
public List<SchemaPath> getPartitionColumns() {
@@ -87,19 +89,15 @@ public class ParquetGroupScanStatistics {
long rowCount = rowGroup.getRowCount();
for (ColumnMetadata column : rowGroup.getColumns()) {
SchemaPath schemaPath = SchemaPath.getCompoundPath(column.getName());
- Long previousCount = columnValueCounts.get(schemaPath);
- if (previousCount != null) {
- if (previousCount != GroupScan.NO_COLUMN_STATS && column.isNumNullsSet()) {
- Long newCount = rowCount - column.getNulls();
- columnValueCounts.put(schemaPath, columnValueCounts.get(schemaPath) + newCount);
- }
+ MutableLong emptyCount = new MutableLong();
+ MutableLong previousCount = columnValueCounts.putIfAbsent(schemaPath, emptyCount);
+ if (previousCount == null) {
+ previousCount = emptyCount;
+ }
+ if (previousCount.longValue() != GroupScan.NO_COLUMN_STATS && column.isNumNullsSet()) {
+ previousCount.add(rowCount - column.getNulls());
} else {
- if (column.isNumNullsSet()) {
- Long newCount = rowCount - column.getNulls();
- columnValueCounts.put(schemaPath, newCount);
- } else {
- columnValueCounts.put(schemaPath, GroupScan.NO_COLUMN_STATS);
- }
+ previousCount.setValue(GroupScan.NO_COLUMN_STATS);
}
boolean partitionColumn = checkForPartitionColumn(column, first, rowCount, parquetTableMetadata);
if (partitionColumn) {
[drill] 03/03: DRILL-6638: Fix TestE2EUnnestAndLateral tests
introduced with DRILL-6546
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 5b36adc5b55da48df65225790bb6045c927cf3d7
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Thu Jul 26 11:28:01 2018 +0300
DRILL-6638: Fix TestE2EUnnestAndLateral tests introduced with DRILL-6546
closes #1402
---
.../exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
index 88108a6..f8b58b9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
@@ -174,7 +174,7 @@ public class TestE2EUnnestAndLateral extends ClusterTest {
public void testUnnestWithItem() throws Exception {
String sql = "select u.item from\n" +
"cp.`lateraljoin/nested-customer.parquet` c," +
- "unnest(c.orders['items']) as u(item)\n" +
+ "unnest(c.orders[0]['items']) as u(item)\n" +
"limit 1";
testBuilder()
@@ -207,7 +207,7 @@ public class TestE2EUnnestAndLateral extends ClusterTest {
public void testUnnestWithMap() throws Exception {
String sql = "select u.item from\n" +
"cp.`lateraljoin/nested-customer.parquet` c," +
- "unnest(c.orders.items) as u(item)\n" +
+ "unnest(c.orders[0].items) as u(item)\n" +
"limit 1";
testBuilder()
@@ -225,8 +225,8 @@ public class TestE2EUnnestAndLateral extends ClusterTest {
public void testMultiUnnestWithMap() throws Exception {
String sql = "select u.item from\n" +
"cp.`lateraljoin/nested-customer.parquet` c," +
- "unnest(c.orders.items) as u(item)," +
- "unnest(c.orders.items) as u1(item1)\n" +
+ "unnest(c.orders[0].items) as u(item)," +
+ "unnest(c.orders[0].items) as u1(item1)\n" +
"limit 1";
testBuilder()
[drill] 02/03: DRILL-6639: Exception happens while displaying
operator profiles for some queries
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
timothyfarkas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 20ecab0637669ac5ca61b915966817546e535e48
Author: Vitalii Diravka <vi...@gmail.com>
AuthorDate: Fri Jul 27 04:15:59 2018 +0300
DRILL-6639: Exception happens while displaying operator profiles for some queries
closes #1404
---
.../drill/exec/store/mapr/db/MapRDBSubScan.java | 3 +-
.../native/client/src/protobuf/UserBitShared.pb.cc | 24 ++++--
.../native/client/src/protobuf/UserBitShared.pb.h | 12 ++-
.../apache/drill/exec/store/kudu/KuduWriter.java | 3 +-
.../drill/exec/store/mongo/MongoSubScan.java | 3 +-
.../drill/exec/store/openTSDB/OpenTSDBSubScan.java | 3 +-
.../exec/physical/config/IteratorValidator.java | 1 +
.../apache/drill/exec/physical/config/Values.java | 1 +
.../exec/server/rest/profile/OperatorWrapper.java | 4 +
.../drill/exec/store/dfs/easy/EasyWriter.java | 2 +-
.../exec/store/easy/json/JSONFormatPlugin.java | 2 +-
.../sequencefile/SequenceFileFormatPlugin.java | 3 +-
.../exec/store/httpd/HttpdLogFormatPlugin.java | 7 +-
.../drill/exec/store/image/ImageFormatPlugin.java | 3 +-
.../drill/exec/store/pcap/PcapFormatPlugin.java | 2 +-
.../org/apache/drill/exec/proto/UserBitShared.java | 88 ++++++++++++++++++++--
.../drill/exec/proto/beans/CoreOperatorType.java | 18 ++++-
protocol/src/main/protobuf/UserBitShared.proto | 8 ++
18 files changed, 160 insertions(+), 27 deletions(-)
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java
index 9547186..c6dfda4 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.physical.base.SubScan;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.store.StoragePluginRegistry;
import com.fasterxml.jackson.annotation.JacksonInject;
@@ -120,7 +121,7 @@ public class MapRDBSubScan extends AbstractBase implements SubScan {
@Override
public int getOperatorType() {
- return 1001;
+ return CoreOperatorType.MAPRDB_SUB_SCAN_VALUE;
}
@JsonIgnore
diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.cc b/contrib/native/client/src/protobuf/UserBitShared.pb.cc
index e6366a3..282f581 100644
--- a/contrib/native/client/src/protobuf/UserBitShared.pb.cc
+++ b/contrib/native/client/src/protobuf/UserBitShared.pb.cc
@@ -750,7 +750,7 @@ void protobuf_AddDesc_UserBitShared_2eproto() {
"TATEMENT\020\005*\207\001\n\rFragmentState\022\013\n\007SENDING\020"
"\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022"
"\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005"
- "\022\032\n\026CANCELLATION_REQUESTED\020\006*\223\007\n\020CoreOpe"
+ "\022\032\n\026CANCELLATION_REQUESTED\020\006*\271\010\n\020CoreOpe"
"ratorType\022\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAS"
"T_SENDER\020\001\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE"
"\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HAS"
@@ -773,11 +773,15 @@ void protobuf_AddDesc_UserBitShared_2eproto() {
"N\020&\022\021\n\rKUDU_SUB_SCAN\020\'\022\013\n\007FLATTEN\020(\022\020\n\014L"
"ATERAL_JOIN\020)\022\n\n\006UNNEST\020*\022,\n(HIVE_DRILL_"
"NATIVE_PARQUET_ROW_GROUP_SCAN\020+\022\r\n\tJDBC_"
- "SCAN\020,\022\022\n\016REGEX_SUB_SCAN\020-*g\n\nSaslStatus"
- "\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL_START\020\001\022\024\n\020SA"
- "SL_IN_PROGRESS\020\002\022\020\n\014SASL_SUCCESS\020\003\022\017\n\013SA"
- "SL_FAILED\020\004B.\n\033org.apache.drill.exec.pro"
- "toB\rUserBitSharedH\001", 5219);
+ "SCAN\020,\022\022\n\016REGEX_SUB_SCAN\020-\022\023\n\017MAPRDB_SUB"
+ "_SCAN\020.\022\022\n\016MONGO_SUB_SCAN\020/\022\017\n\013KUDU_WRIT"
+ "ER\0200\022\026\n\022OPEN_TSDB_SUB_SCAN\0201\022\017\n\013JSON_WRI"
+ "TER\0202\022\026\n\022HTPPD_LOG_SUB_SCAN\0203\022\022\n\016IMAGE_S"
+ "UB_SCAN\0204\022\025\n\021SEQUENCE_SUB_SCAN\0205*g\n\nSasl"
+ "Status\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL_START\020\001"
+ "\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014SASL_SUCCESS\020\003"
+ "\022\017\n\013SASL_FAILED\020\004B.\n\033org.apache.drill.ex"
+ "ec.protoB\rUserBitSharedH\001", 5385);
::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
"UserBitShared.proto", &protobuf_RegisterTypes);
UserCredentials::default_instance_ = new UserCredentials();
@@ -944,6 +948,14 @@ bool CoreOperatorType_IsValid(int value) {
case 43:
case 44:
case 45:
+ case 46:
+ case 47:
+ case 48:
+ case 49:
+ case 50:
+ case 51:
+ case 52:
+ case 53:
return true;
default:
return false;
diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.h b/contrib/native/client/src/protobuf/UserBitShared.pb.h
index 6efeffb..134dc2b 100644
--- a/contrib/native/client/src/protobuf/UserBitShared.pb.h
+++ b/contrib/native/client/src/protobuf/UserBitShared.pb.h
@@ -249,11 +249,19 @@ enum CoreOperatorType {
UNNEST = 42,
HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN = 43,
JDBC_SCAN = 44,
- REGEX_SUB_SCAN = 45
+ REGEX_SUB_SCAN = 45,
+ MAPRDB_SUB_SCAN = 46,
+ MONGO_SUB_SCAN = 47,
+ KUDU_WRITER = 48,
+ OPEN_TSDB_SUB_SCAN = 49,
+ JSON_WRITER = 50,
+ HTPPD_LOG_SUB_SCAN = 51,
+ IMAGE_SUB_SCAN = 52,
+ SEQUENCE_SUB_SCAN = 53
};
bool CoreOperatorType_IsValid(int value);
const CoreOperatorType CoreOperatorType_MIN = SINGLE_SENDER;
-const CoreOperatorType CoreOperatorType_MAX = REGEX_SUB_SCAN;
+const CoreOperatorType CoreOperatorType_MAX = SEQUENCE_SUB_SCAN;
const int CoreOperatorType_ARRAYSIZE = CoreOperatorType_MAX + 1;
const ::google::protobuf::EnumDescriptor* CoreOperatorType_descriptor();
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduWriter.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduWriter.java
index 88a6849..d0fa158 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduWriter.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduWriter.java
@@ -23,6 +23,7 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.physical.base.AbstractWriter;
import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.store.StoragePluginRegistry;
import com.fasterxml.jackson.annotation.JacksonInject;
@@ -56,7 +57,7 @@ public class KuduWriter extends AbstractWriter {
@Override
public int getOperatorType() {
- return 3001;
+ return CoreOperatorType.KUDU_WRITER_VALUE;
}
@Override
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoSubScan.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoSubScan.java
index ed9f555..cff3f57 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoSubScan.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoSubScan.java
@@ -29,6 +29,7 @@ import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.physical.base.SubScan;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.bson.Document;
import org.slf4j.Logger;
@@ -114,7 +115,7 @@ public class MongoSubScan extends AbstractBase implements SubScan {
@Override
public int getOperatorType() {
- return 1009;
+ return CoreOperatorType.MONGO_SUB_SCAN_VALUE;
}
@Override
diff --git a/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/OpenTSDBSubScan.java b/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/OpenTSDBSubScan.java
index 4e93804..240d661 100644
--- a/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/OpenTSDBSubScan.java
+++ b/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/OpenTSDBSubScan.java
@@ -29,6 +29,7 @@ import org.apache.drill.exec.physical.base.AbstractBase;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.PhysicalVisitor;
import org.apache.drill.exec.physical.base.SubScan;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,7 +74,7 @@ public class OpenTSDBSubScan extends AbstractBase implements SubScan {
@Override
public int getOperatorType() {
- return 0;
+ return CoreOperatorType.OPEN_TSDB_SUB_SCAN_VALUE;
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
index 6494f9f..9fbef97 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/IteratorValidator.java
@@ -41,6 +41,7 @@ public class IteratorValidator extends AbstractSingle{
@Override
public int getOperatorType() {
+ // TODO: DRILL-6643: this implementation should be revisited
return -1;
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Values.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Values.java
index f32ed3a..4458511 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Values.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Values.java
@@ -60,6 +60,7 @@ public class Values extends AbstractBase implements Leaf {
@Override
public int getOperatorType() {
+ // TODO: DRILL-6643: this implementation should be revisited
return -1;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index 0a4c4a5..85545e3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -253,6 +253,10 @@ public class OperatorWrapper {
* @return index of spill metric
*/
private int getSpillCycleMetricIndex(CoreOperatorType operatorType) {
+ // TODO: DRILL-6642, replace null values for ProtocolMessageEnum with UNRECOGNIZED NullValue to avoid null checks
+ if (operatorType == null) {
+ return NO_SPILL_METRIC_INDEX;
+ }
String metricName;
switch (operatorType) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java
index 52ce8b0..f0e7435 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyWriter.java
@@ -101,6 +101,6 @@ public class EasyWriter extends AbstractWriter {
@Override
public int getOperatorType() {
- return formatPlugin.getReaderOperatorType();
+ return formatPlugin.getWriterOperatorType();
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
index 5eec5cc..c42bbd3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
@@ -144,7 +144,7 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
@Override
public int getWriterOperatorType() {
- throw new UnsupportedOperationException();
+ return CoreOperatorType.JSON_WRITER_VALUE;
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/sequencefile/SequenceFileFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/sequencefile/SequenceFileFormatPlugin.java
index ec480ae..2da4475 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/sequencefile/SequenceFileFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/sequencefile/SequenceFileFormatPlugin.java
@@ -22,6 +22,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.RecordWriter;
@@ -76,7 +77,7 @@ public class SequenceFileFormatPlugin extends EasyFormatPlugin<SequenceFileForma
@Override
public int getReaderOperatorType() {
- return 4001;
+ return CoreOperatorType.SEQUENCE_SUB_SCAN_VALUE;
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
index cee9a89..23bd5d1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
@@ -34,6 +34,7 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.AbstractRecordReader;
import org.apache.drill.exec.store.RecordWriter;
@@ -266,11 +267,11 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
@Override
public int getReaderOperatorType() {
- return -1;
+ return CoreOperatorType.HTPPD_LOG_SUB_SCAN_VALUE;
}
@Override
public int getWriterOperatorType() {
- return -1;
+ throw new UnsupportedOperationException();
}
-}
\ No newline at end of file
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/image/ImageFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/image/ImageFormatPlugin.java
index 6b0b9b4..e26ae23 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/image/ImageFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/image/ImageFormatPlugin.java
@@ -26,6 +26,7 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.RecordWriter;
@@ -67,7 +68,7 @@ public class ImageFormatPlugin extends EasyFormatPlugin<ImageFormatConfig> {
@Override
public int getReaderOperatorType() {
- return 4002;
+ return CoreOperatorType.IMAGE_SUB_SCAN_VALUE;
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapFormatPlugin.java
index 0f24538..38cabbc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapFormatPlugin.java
@@ -81,7 +81,7 @@ public class PcapFormatPlugin extends EasyFormatPlugin<PcapFormatConfig> {
@Override
public int getWriterOperatorType() {
- return 0;
+ throw new UnsupportedOperationException();
}
@Override
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
index 0243ed5..a8574f5 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
@@ -549,6 +549,38 @@ public final class UserBitShared {
* <code>REGEX_SUB_SCAN = 45;</code>
*/
REGEX_SUB_SCAN(45, 45),
+ /**
+ * <code>MAPRDB_SUB_SCAN = 46;</code>
+ */
+ MAPRDB_SUB_SCAN(46, 46),
+ /**
+ * <code>MONGO_SUB_SCAN = 47;</code>
+ */
+ MONGO_SUB_SCAN(47, 47),
+ /**
+ * <code>KUDU_WRITER = 48;</code>
+ */
+ KUDU_WRITER(48, 48),
+ /**
+ * <code>OPEN_TSDB_SUB_SCAN = 49;</code>
+ */
+ OPEN_TSDB_SUB_SCAN(49, 49),
+ /**
+ * <code>JSON_WRITER = 50;</code>
+ */
+ JSON_WRITER(50, 50),
+ /**
+ * <code>HTPPD_LOG_SUB_SCAN = 51;</code>
+ */
+ HTPPD_LOG_SUB_SCAN(51, 51),
+ /**
+ * <code>IMAGE_SUB_SCAN = 52;</code>
+ */
+ IMAGE_SUB_SCAN(52, 52),
+ /**
+ * <code>SEQUENCE_SUB_SCAN = 53;</code>
+ */
+ SEQUENCE_SUB_SCAN(53, 53),
;
/**
@@ -735,6 +767,38 @@ public final class UserBitShared {
* <code>REGEX_SUB_SCAN = 45;</code>
*/
public static final int REGEX_SUB_SCAN_VALUE = 45;
+ /**
+ * <code>MAPRDB_SUB_SCAN = 46;</code>
+ */
+ public static final int MAPRDB_SUB_SCAN_VALUE = 46;
+ /**
+ * <code>MONGO_SUB_SCAN = 47;</code>
+ */
+ public static final int MONGO_SUB_SCAN_VALUE = 47;
+ /**
+ * <code>KUDU_WRITER = 48;</code>
+ */
+ public static final int KUDU_WRITER_VALUE = 48;
+ /**
+ * <code>OPEN_TSDB_SUB_SCAN = 49;</code>
+ */
+ public static final int OPEN_TSDB_SUB_SCAN_VALUE = 49;
+ /**
+ * <code>JSON_WRITER = 50;</code>
+ */
+ public static final int JSON_WRITER_VALUE = 50;
+ /**
+ * <code>HTPPD_LOG_SUB_SCAN = 51;</code>
+ */
+ public static final int HTPPD_LOG_SUB_SCAN_VALUE = 51;
+ /**
+ * <code>IMAGE_SUB_SCAN = 52;</code>
+ */
+ public static final int IMAGE_SUB_SCAN_VALUE = 52;
+ /**
+ * <code>SEQUENCE_SUB_SCAN = 53;</code>
+ */
+ public static final int SEQUENCE_SUB_SCAN_VALUE = 53;
public final int getNumber() { return value; }
@@ -787,6 +851,14 @@ public final class UserBitShared {
case 43: return HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN;
case 44: return JDBC_SCAN;
case 45: return REGEX_SUB_SCAN;
+ case 46: return MAPRDB_SUB_SCAN;
+ case 47: return MONGO_SUB_SCAN;
+ case 48: return KUDU_WRITER;
+ case 49: return OPEN_TSDB_SUB_SCAN;
+ case 50: return JSON_WRITER;
+ case 51: return HTPPD_LOG_SUB_SCAN;
+ case 52: return IMAGE_SUB_SCAN;
+ case 53: return SEQUENCE_SUB_SCAN;
default: return null;
}
}
@@ -24323,7 +24395,7 @@ public final class UserBitShared {
"TATEMENT\020\005*\207\001\n\rFragmentState\022\013\n\007SENDING\020" +
"\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022" +
"\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005" +
- "\022\032\n\026CANCELLATION_REQUESTED\020\006*\223\007\n\020CoreOpe" +
+ "\022\032\n\026CANCELLATION_REQUESTED\020\006*\271\010\n\020CoreOpe" +
"ratorType\022\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAS" +
"T_SENDER\020\001\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE" +
"\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HAS" +
@@ -24346,11 +24418,15 @@ public final class UserBitShared {
"N\020&\022\021\n\rKUDU_SUB_SCAN\020\'\022\013\n\007FLATTEN\020(\022\020\n\014L" +
"ATERAL_JOIN\020)\022\n\n\006UNNEST\020*\022,\n(HIVE_DRILL_" +
"NATIVE_PARQUET_ROW_GROUP_SCAN\020+\022\r\n\tJDBC_" +
- "SCAN\020,\022\022\n\016REGEX_SUB_SCAN\020-*g\n\nSaslStatus" +
- "\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL_START\020\001\022\024\n\020SA" +
- "SL_IN_PROGRESS\020\002\022\020\n\014SASL_SUCCESS\020\003\022\017\n\013SA" +
- "SL_FAILED\020\004B.\n\033org.apache.drill.exec.pro",
- "toB\rUserBitSharedH\001"
+ "SCAN\020,\022\022\n\016REGEX_SUB_SCAN\020-\022\023\n\017MAPRDB_SUB" +
+ "_SCAN\020.\022\022\n\016MONGO_SUB_SCAN\020/\022\017\n\013KUDU_WRIT" +
+ "ER\0200\022\026\n\022OPEN_TSDB_SUB_SCAN\0201\022\017\n\013JSON_WRI" +
+ "TER\0202\022\026\n\022HTPPD_LOG_SUB_SCAN\0203\022\022\n\016IMAGE_S",
+ "UB_SCAN\0204\022\025\n\021SEQUENCE_SUB_SCAN\0205*g\n\nSasl" +
+ "Status\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL_START\020\001" +
+ "\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014SASL_SUCCESS\020\003" +
+ "\022\017\n\013SASL_FAILED\020\004B.\n\033org.apache.drill.ex" +
+ "ec.protoB\rUserBitSharedH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
index 45b9cc11..53af571 100644
--- a/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
+++ b/protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
@@ -67,7 +67,15 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
UNNEST(42),
HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN(43),
JDBC_SCAN(44),
- REGEX_SUB_SCAN(45);
+ REGEX_SUB_SCAN(45),
+ MAPRDB_SUB_SCAN(46),
+ MONGO_SUB_SCAN(47),
+ KUDU_WRITER(48),
+ OPEN_TSDB_SUB_SCAN(49),
+ JSON_WRITER(50),
+ HTPPD_LOG_SUB_SCAN(51),
+ IMAGE_SUB_SCAN(52),
+ SEQUENCE_SUB_SCAN(53);
public final int number;
@@ -131,6 +139,14 @@ public enum CoreOperatorType implements com.dyuproject.protostuff.EnumLite<CoreO
case 43: return HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN;
case 44: return JDBC_SCAN;
case 45: return REGEX_SUB_SCAN;
+ case 46: return MAPRDB_SUB_SCAN;
+ case 47: return MONGO_SUB_SCAN;
+ case 48: return KUDU_WRITER;
+ case 49: return OPEN_TSDB_SUB_SCAN;
+ case 50: return JSON_WRITER;
+ case 51: return HTPPD_LOG_SUB_SCAN;
+ case 52: return IMAGE_SUB_SCAN;
+ case 53: return SEQUENCE_SUB_SCAN;
default: return null;
}
}
diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto
index 47715a4..4c4960e 100644
--- a/protocol/src/main/protobuf/UserBitShared.proto
+++ b/protocol/src/main/protobuf/UserBitShared.proto
@@ -334,6 +334,14 @@ enum CoreOperatorType {
HIVE_DRILL_NATIVE_PARQUET_ROW_GROUP_SCAN = 43;
JDBC_SCAN = 44;
REGEX_SUB_SCAN = 45;
+ MAPRDB_SUB_SCAN = 46;
+ MONGO_SUB_SCAN = 47;
+ KUDU_WRITER = 48;
+ OPEN_TSDB_SUB_SCAN = 49;
+ JSON_WRITER = 50;
+ HTPPD_LOG_SUB_SCAN = 51;
+ IMAGE_SUB_SCAN = 52;
+ SEQUENCE_SUB_SCAN = 53;
}
/* Registry that contains list of jars, each jar contains its name and list of function signatures.