You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ad...@apache.org on 2015/09/09 01:25:21 UTC
[1/5] drill git commit: DRILL-3555: Changing defaults for
planner.memory.max_query_memory_per_node causes queries with window function
to fail
Repository: drill
Updated Branches:
refs/heads/master 38e660e13 -> 2a1918471
DRILL-3555: Changing defaults for planner.memory.max_query_memory_per_node causes queries with window function to fail
this closes #137
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7a864a43
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7a864a43
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7a864a43
Branch: refs/heads/master
Commit: 7a864a43919a46b16a2f59964e7288e30d96eb2c
Parents: 38e660e
Author: adeneche <ad...@gmail.com>
Authored: Thu Aug 27 12:57:23 2015 -0700
Committer: adeneche <ad...@gmail.com>
Committed: Tue Sep 8 15:35:09 2015 -0700
----------------------------------------------------------------------
.../exec/physical/impl/TopN/TopNBatch.java | 3 +--
.../OrderedPartitionRecordBatch.java | 5 ++---
.../exec/physical/impl/sort/SortBatch.java | 6 ++----
.../impl/sort/SortRecordBatchBuilder.java | 20 ++++----------------
.../physical/impl/xsort/ExternalSortBatch.java | 3 +--
5 files changed, 10 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index 10f1d7f..3ef6bfe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -67,7 +67,6 @@ import com.sun.codemodel.JExpr;
public class TopNBatch extends AbstractRecordBatch<TopN> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopNBatch.class);
- private static final long MAX_SORT_BYTES = 1L * 1024 * 1024 * 1024;
private final int batchPurgeThreshold;
public final MappingSet MAIN_MAPPING = new MappingSet( (String) null, null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
@@ -285,7 +284,7 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
}
copier.setupRemover(context, batch, newBatch);
}
- SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);
+ SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator());
try {
do {
int count = selectionVector4.getCount();
http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index 1286fe1..0050b45 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -109,7 +109,6 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
public final MappingSet partitionMapping = new MappingSet("partitionIndex", null, "partitionVectors", null,
ClassGenerator.DEFAULT_CONSTANT_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
- private static long MAX_SORT_BYTES = 8l * 1024 * 1024 * 1024;
private final int recordsToSample; // How many records must be received before analyzing
private final int samplingFactor; // Will collect samplingFactor * number of partitions to send to distributed cache
private final float completionFactor; // What fraction of fragments must be completed before attempting to build
@@ -171,7 +170,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
// Start collecting batches until recordsToSample records have been collected
- SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);
+ SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator());
WritableBatch batch = null;
CachedVectorContainer sampleToSave = null;
VectorContainer containerToCache = new VectorContainer();
@@ -344,7 +343,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
// Get all samples from distributed map
- SortRecordBatchBuilder containerBuilder = new SortRecordBatchBuilder(context.getAllocator(), MAX_SORT_BYTES);
+ SortRecordBatchBuilder containerBuilder = new SortRecordBatchBuilder(context.getAllocator());
final VectorContainer allSamplesContainer = new VectorContainer();
final VectorContainer candidatePartitionTable = new VectorContainer();
CachedVectorContainer wrap = null;
http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
index dea6ba8..407f05d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
@@ -54,8 +54,6 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
public final MappingSet leftMapping = new MappingSet("leftIndex", null, ClassGenerator.DEFAULT_CONSTANT_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
public final MappingSet rightMapping = new MappingSet("rightIndex", null, ClassGenerator.DEFAULT_CONSTANT_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
- private static long MAX_SORT_BYTES = 8l * 1024 * 1024 * 1024;
-
private final RecordBatch incoming;
private final SortRecordBatchBuilder builder;
private Sorter sorter;
@@ -64,7 +62,7 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
public SortBatch(Sort popConfig, FragmentContext context, RecordBatch incoming) throws OutOfMemoryException {
super(popConfig, context);
this.incoming = incoming;
- this.builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);
+ this.builder = new SortRecordBatchBuilder(oContext.getAllocator());
}
@Override
@@ -122,7 +120,7 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
case OK:
if (!builder.add(incoming)) {
throw new UnsupportedOperationException("Sort doesn't currently support doing an external sort.");
- };
+ }
break;
default:
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
index 00f1992..edbb62f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
@@ -45,15 +45,12 @@ public class SortRecordBatchBuilder implements AutoCloseable {
private final ArrayListMultimap<BatchSchema, RecordBatchData> batches = ArrayListMultimap.create();
private int recordCount;
- private long runningBytes;
private long runningBatches;
- private final long maxBytes;
private SelectionVector4 sv4;
final PreAllocator svAllocator;
private boolean svAllocatorUsed = false;
- public SortRecordBatchBuilder(BufferAllocator a, long maxBytes) {
- this.maxBytes = maxBytes;
+ public SortRecordBatchBuilder(BufferAllocator a) {
this.svAllocator = a.getNewPreAllocator();
}
@@ -84,10 +81,7 @@ public class SortRecordBatchBuilder implements AutoCloseable {
if (batchBytes == 0 && batches.size() > 0) {
return true;
}
- if (batchBytes + runningBytes > maxBytes) {
- return false; // enough data memory.
- }
- if (runningBatches+1 > Character.MAX_VALUE) {
+ if (runningBatches >= Character.MAX_VALUE) {
return false; // allowed in batch.
}
if (!svAllocator.preAllocate(batch.getRecordCount()*4)) {
@@ -96,7 +90,7 @@ public class SortRecordBatchBuilder implements AutoCloseable {
RecordBatchData bd = new RecordBatchData(batch);
- runningBytes += batchBytes;
+ runningBatches++;
batches.put(batch.getSchema(), bd);
recordCount += bd.getRecordCount();
return true;
@@ -108,12 +102,6 @@ public class SortRecordBatchBuilder implements AutoCloseable {
return;
}
- if(batchBytes + runningBytes > maxBytes) {
- final String errMsg = String.format("Adding this batch causes the total size to exceed max allowed size. " +
- "Current runningBytes %d, Incoming batchBytes %d. maxBytes %d", runningBytes, batchBytes, maxBytes);
- logger.error(errMsg);
- throw new DrillRuntimeException(errMsg);
- }
if(runningBatches >= Character.MAX_VALUE) {
final String errMsg = String.format("Tried to add more than %d number of batches.", Character.MAX_VALUE);
logger.error(errMsg);
@@ -135,7 +123,7 @@ public class SortRecordBatchBuilder implements AutoCloseable {
}
return;
}
- runningBytes += batchBytes;
+ runningBatches++;
batches.put(rbd.getContainer().getSchema(), rbd);
recordCount += rbd.getRecordCount();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index d9866bb..ed32f43 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -83,7 +83,6 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExternalSortBatch.class);
private static final ControlsInjector injector = ControlsInjectorFactory.getInjector(ExternalSortBatch.class);
- private static final long MAX_SORT_BYTES = 1L * 1024 * 1024 * 1024;
private static final GeneratorMapping COPIER_MAPPING = new GeneratorMapping("doSetup", "doCopy", null, null);
private static final MappingSet MAIN_MAPPING = new MappingSet( (String) null, null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
private static final MappingSet LEFT_MAPPING = new MappingSet("leftIndex", null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
@@ -390,7 +389,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
builder.clear();
builder.close();
}
- builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);
+ builder = new SortRecordBatchBuilder(oContext.getAllocator());
for (BatchGroup group : batchGroups) {
RecordBatchData rbd = new RecordBatchData(group.getContainer());
[5/5] drill git commit: DRILL-3535: Add support for Drop Table
Posted by ad...@apache.org.
DRILL-3535: Add support for Drop Table
this closes #140
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2a191847
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2a191847
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2a191847
Branch: refs/heads/master
Commit: 2a191847154203871454b229d8ef322766aa9ee4
Parents: 41fc9ca
Author: Mehant Baid <me...@gmail.com>
Authored: Wed Aug 5 12:03:01 2015 -0700
Committer: adeneche <ad...@gmail.com>
Committed: Tue Sep 8 16:23:38 2015 -0700
----------------------------------------------------------------------
exec/java-exec/src/main/codegen/data/Parser.tdd | 3 +-
.../src/main/codegen/includes/parserImpls.ftl | 17 ++
.../planner/sql/handlers/DropTableHandler.java | 77 ++++++++
.../sql/parser/CompoundIdentifierConverter.java | 1 +
.../exec/planner/sql/parser/SqlDropTable.java | 95 ++++++++++
.../apache/drill/exec/store/AbstractSchema.java | 6 +
.../exec/store/dfs/BasicFormatMatcher.java | 11 +-
.../drill/exec/store/dfs/DrillFileSystem.java | 4 +
.../drill/exec/store/dfs/DrillPathFilter.java | 5 +-
.../drill/exec/store/dfs/FormatMatcher.java | 3 +
.../exec/store/dfs/WorkspaceSchemaFactory.java | 124 +++++++++++++
.../exec/store/parquet/ParquetFormatPlugin.java | 2 +-
.../java/org/apache/drill/TestDropTable.java | 174 +++++++++++++++++++
.../TestImpersonationMetadata.java | 44 ++++-
14 files changed, 558 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/codegen/data/Parser.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/Parser.tdd b/exec/java-exec/src/main/codegen/data/Parser.tdd
index 7c03ab0..6ff7fa4 100644
--- a/exec/java-exec/src/main/codegen/data/Parser.tdd
+++ b/exec/java-exec/src/main/codegen/data/Parser.tdd
@@ -46,7 +46,8 @@
"SqlCreateOrReplaceView()",
"SqlDropView()",
"SqlShowFiles()",
- "SqlCreateTable()"
+ "SqlCreateTable()",
+ "SqlDropTable()"
]
# List of methods for parsing custom literals.
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
index b1c22f2..c761d47 100644
--- a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
+++ b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
@@ -240,3 +240,20 @@ SqlNode SqlCreateTable() :
return new SqlCreateTable(pos, tblName, fieldList, partitionFieldList, query);
}
}
+
+/**
+ * Parses a drop table statement.
+ * DROP TABLE table_name;
+ */
+SqlNode SqlDropTable() :
+{
+ SqlParserPos pos;
+}
+{
+ <DROP> { pos = getPos(); }
+ <TABLE>
+ {
+ return new SqlDropTable(pos, CompoundIdentifier());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropTableHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropTableHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropTableHandler.java
new file mode 100644
index 0000000..211d256
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DropTableHandler.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql.handlers;
+
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.planner.sql.DirectPlan;
+import org.apache.drill.exec.planner.sql.SchemaUtilites;
+import org.apache.drill.exec.planner.sql.parser.SqlDropTable;
+import org.apache.drill.exec.store.AbstractSchema;
+
+import java.io.IOException;
+
+// SqlHandler for dropping a table.
+public class DropTableHandler extends DefaultSqlHandler {
+
+ private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DropTableHandler.class);
+
+ public DropTableHandler(SqlHandlerConfig config) {
+ super(config);
+ }
+
+ /**
+ * Function resolves the schema and invokes the drop method. Raises an exception if the schema is
+ * immutable.
+ * @param sqlNode - Table name identifier
+ * @return - Single row indicating drop succeeded, raise exception otherwise
+ * @throws ValidationException
+ * @throws RelConversionException
+ * @throws IOException
+ */
+ @Override
+ public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException {
+
+ SqlDropTable dropTableNode = ((SqlDropTable) sqlNode);
+ SqlIdentifier tableIdentifier = dropTableNode.getTableIdentifier();
+
+ SchemaPlus defaultSchema = context.getNewDefaultSchema();
+ AbstractSchema drillSchema = null;
+
+ if (tableIdentifier != null) {
+ drillSchema = SchemaUtilites.resolveToMutableDrillSchema(defaultSchema, dropTableNode.getSchema());
+ }
+
+ String tableName = ((SqlDropTable) sqlNode).getName();
+ if (drillSchema == null) {
+ throw UserException.validationError()
+ .message("Invalid table_name [%s]", tableName)
+ .build(logger);
+ }
+
+ drillSchema.dropTable(tableName);
+
+ return DirectPlan.createDirectPlan(context, true,
+ String.format("Table [%s] %s", tableName, "dropped"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
index f9032a4..ebe6d39 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
@@ -160,6 +160,7 @@ public class CompoundIdentifierConverter extends SqlShuttle {
rules.put(SqlUseSchema.class, R(D));
rules.put(SqlJoin.class, R(D, D, D, D, D, E));
rules.put(SqlOrderBy.class, R(D, E, D, D));
+ rules.put(SqlDropTable.class, R(D));
REWRITE_RULES = ImmutableMap.copyOf(rules);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropTable.java
new file mode 100644
index 0000000..de272a1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlDropTable.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.sql.parser;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler;
+import org.apache.drill.exec.planner.sql.handlers.DropTableHandler;
+import org.apache.drill.exec.planner.sql.handlers.SqlHandlerConfig;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlSpecialOperator;
+import org.apache.calcite.sql.SqlWriter;
+import org.apache.calcite.sql.parser.SqlParserPos;
+
+import com.google.common.collect.ImmutableList;
+
+public class SqlDropTable extends DrillSqlCall {
+ public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator("DROP_TABLE", SqlKind.OTHER) {
+ @Override
+ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) {
+ return new SqlDropTable(pos, (SqlIdentifier) operands[0]);
+ }
+ };
+
+ private SqlIdentifier tableName;
+
+ public SqlDropTable(SqlParserPos pos, SqlIdentifier tableName) {
+ super(pos);
+ this.tableName = tableName;
+ }
+
+ @Override
+ public SqlOperator getOperator() {
+ return OPERATOR;
+ }
+
+ @Override
+ public List<SqlNode> getOperandList() {
+ return Collections.singletonList((SqlNode) tableName);
+ }
+
+ @Override
+ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) {
+ writer.keyword("DROP");
+ writer.keyword("TABLE");
+ tableName.unparse(writer, leftPrec, rightPrec);
+ }
+
+ @Override
+ public AbstractSqlHandler getSqlHandler(SqlHandlerConfig config) {
+ return new DropTableHandler(config);
+ }
+
+ public List<String> getSchema() {
+ if (tableName.isSimple()) {
+ return ImmutableList.of();
+ }
+
+ return tableName.names.subList(0, tableName.names.size()-1);
+ }
+
+ public String getName() {
+ if (tableName.isSimple()) {
+ return tableName.getSimple();
+ }
+
+ return tableName.names.get(tableName.names.size() - 1);
+ }
+
+ public SqlIdentifier getTableIdentifier() {
+ return tableName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
index a952cc2..0db51ab1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java
@@ -190,4 +190,10 @@ public abstract class AbstractSchema implements Schema, SchemaPartitionExplorer,
public void close() throws Exception {
// no-op: default implementation for most implementations.
}
+
+ public void dropTable(String tableName) {
+ throw UserException.unsupportedError()
+ .message("Dropping tables is not supported in schema [%s]", getSchemaPath())
+ .build(logger);
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
index 1cb9f82..7f4a8d4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
@@ -72,7 +72,7 @@ public class BasicFormatMatcher extends FormatMatcher{
@Override
public FormatSelection isReadable(DrillFileSystem fs, FileSelection selection) throws IOException {
- if (isReadable(fs, selection.getFirstPath(fs))) {
+ if (isFileReadable(fs, selection.getFirstPath(fs))) {
if (plugin.getName() != null) {
NamedFormatPluginConfig namedConfig = new NamedFormatPluginConfig();
namedConfig.name = plugin.getName();
@@ -84,8 +84,12 @@ public class BasicFormatMatcher extends FormatMatcher{
return null;
}
- protected final boolean isReadable(DrillFileSystem fs, FileStatus status) throws IOException {
- CompressionCodec codec = null;
+ /*
+ * Function returns true if the file extension matches the pattern
+ */
+ @Override
+ public boolean isFileReadable(DrillFileSystem fs, FileStatus status) throws IOException {
+ CompressionCodec codec = null;
if (compressible) {
codec = codecFactory.getCodec(status.getPath());
}
@@ -111,7 +115,6 @@ public class BasicFormatMatcher extends FormatMatcher{
return false;
}
-
@Override
@JsonIgnore
public FormatPlugin getFormatPlugin() {
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFileSystem.java
index 25dd811..5e720a2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFileSystem.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillFileSystem.java
@@ -71,6 +71,10 @@ import com.google.common.collect.Maps;
public class DrillFileSystem extends FileSystem implements OpenFileTracker {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFileSystem.class);
private final static boolean TRACKING_ENABLED = AssertionUtil.isAssertionsEnabled();
+
+ public static final String HIDDEN_FILE_PREFIX = "_";
+ public static final String DOT_FILE_PREFIX = ".";
+
private final ConcurrentMap<DrillFSDataInputStream, DebugStackTrace> openedFiles = Maps.newConcurrentMap();
private final FileSystem underlyingFs;
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillPathFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillPathFilter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillPathFilter.java
index 81c8779..00f463d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillPathFilter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/DrillPathFilter.java
@@ -23,7 +23,10 @@ import org.apache.hadoop.mapred.Utils;
public class DrillPathFilter extends Utils.OutputFileUtils.OutputFilesFilter {
@Override
public boolean accept(Path path) {
- if (path.toString().contains("_metadata")) {
+ if (path.getName().startsWith(DrillFileSystem.HIDDEN_FILE_PREFIX)) {
+ return false;
+ }
+ if (path.getName().startsWith(DrillFileSystem.DOT_FILE_PREFIX)) {
return false;
}
return super.accept(path);
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
index 0b8c7a8..c36e17c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.store.dfs;
+import org.apache.hadoop.fs.FileStatus;
+
import java.io.IOException;
public abstract class FormatMatcher {
@@ -24,5 +26,6 @@ public abstract class FormatMatcher {
public abstract boolean supportDirectoryReads();
public abstract FormatSelection isReadable(DrillFileSystem fs, FileSelection selection) throws IOException;
+ public abstract boolean isFileReadable(DrillFileSystem fs, FileStatus status) throws IOException;
public abstract FormatPlugin getFormatPlugin();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
index a7e83f6..22d00a2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
@@ -20,8 +20,11 @@ package org.apache.drill.exec.store.dfs;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Queue;
import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Pattern;
import com.google.common.base.Strings;
@@ -62,6 +65,7 @@ public class WorkspaceSchemaFactory {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WorkspaceSchemaFactory.class);
private final List<FormatMatcher> fileMatchers;
+ private final List<FormatMatcher> dropFileMatchers;
private final List<FormatMatcher> dirMatchers;
private final WorkspaceConfig config;
@@ -104,6 +108,9 @@ public class WorkspaceSchemaFactory {
final FormatMatcher fallbackMatcher = new BasicFormatMatcher(formatPlugin,
ImmutableList.of(Pattern.compile(".*")), ImmutableList.<MagicString>of());
fileMatchers.add(fallbackMatcher);
+ dropFileMatchers = fileMatchers.subList(0, fileMatchers.size() - 1);
+ } else {
+ dropFileMatchers = fileMatchers.subList(0, fileMatchers.size());
}
}
@@ -321,8 +328,125 @@ public class WorkspaceSchemaFactory {
return null;
}
+ private FormatMatcher findMatcher(FileStatus file) {
+ FormatMatcher matcher = null;
+ try {
+ for (FormatMatcher m : dropFileMatchers) {
+ if (m.isFileReadable(fs, file)) {
+ return m;
+ }
+ }
+ } catch (IOException e) {
+ logger.debug("Failed to find format matcher for file: %s", file, e);
+ }
+ return matcher;
+ }
+
@Override
public void destroy(DrillTable value) {
}
+
+ /**
+ * Check if the table contains homogenenous files that can be read by Drill. Eg: parquet, json csv etc.
+ * However if it contains more than one of these formats or a totally different file format that Drill cannot
+ * understand then we will raise an exception.
+ * @param tableName - name of the table to be checked for homogeneous property
+ * @return
+ * @throws IOException
+ */
+ private boolean isHomogeneous(String tableName) throws IOException {
+ FileSelection fileSelection = FileSelection.create(fs, config.getLocation(), tableName);
+
+ if (fileSelection == null) {
+ throw UserException
+ .validationError()
+ .message(String.format("Table [%s] not found", tableName))
+ .build(logger);
+ }
+
+ FormatMatcher matcher = null;
+ Queue<FileStatus> listOfFiles = new LinkedList<>();
+ listOfFiles.addAll(fileSelection.getFileStatusList(fs));
+
+ while (!listOfFiles.isEmpty()) {
+ FileStatus currentFile = listOfFiles.poll();
+ if (currentFile.isDirectory()) {
+ listOfFiles.addAll(fs.list(true, currentFile.getPath()));
+ } else {
+ if (matcher != null) {
+ if (!matcher.isFileReadable(fs, currentFile)) {
+ return false;
+ }
+ } else {
+ matcher = findMatcher(currentFile);
+ // Did not match any of the file patterns, exit
+ if (matcher == null) {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * We check if the table contains homogeneous file formats that Drill can read. Once the checks are performed
+ * we rename the file to start with an "_". After the rename we issue a recursive delete of the directory.
+ * @param table - Path of table to be dropped
+ */
+ @Override
+ public void dropTable(String table) {
+ DrillFileSystem fs = getFS();
+ String defaultLocation = getDefaultLocation();
+ try {
+ if (!isHomogeneous(table)) {
+ throw UserException
+ .validationError()
+ .message("Table contains different file formats. \n" +
+ "Drop Table is only supported for directories that contain homogeneous file formats consumable by Drill")
+ .build(logger);
+ }
+
+ StringBuilder tableRenameBuilder = new StringBuilder();
+ int lastSlashIndex = table.lastIndexOf(Path.SEPARATOR);
+ if (lastSlashIndex != -1) {
+ tableRenameBuilder.append(table.substring(0, lastSlashIndex + 1));
+ }
+ // Generate unique identifier which will be added as a suffix to the table name
+ ThreadLocalRandom r = ThreadLocalRandom.current();
+ long time = (System.currentTimeMillis()/1000);
+ Long p1 = ((Integer.MAX_VALUE - time) << 32) + r.nextInt();
+ Long p2 = r.nextLong();
+ final String fileNameDelimiter = DrillFileSystem.HIDDEN_FILE_PREFIX;
+ String[] pathSplit = table.split(Path.SEPARATOR);
+ /*
+ * Builds the string for the renamed table
+ * Prefixes the table name with an underscore (intent for this to be treated as a hidden file)
+ * and suffixes the table name with unique identifiers (similar to how we generate query id's)
+ * separated by underscores
+ */
+ tableRenameBuilder
+ .append(DrillFileSystem.HIDDEN_FILE_PREFIX)
+ .append(pathSplit[pathSplit.length - 1])
+ .append(fileNameDelimiter)
+ .append(p1.toString())
+ .append(fileNameDelimiter)
+ .append(p2.toString());
+
+ String tableRename = tableRenameBuilder.toString();
+ fs.rename(new Path(defaultLocation, table), new Path(defaultLocation, tableRename));
+ fs.delete(new Path(defaultLocation, tableRename), true);
+ } catch (AccessControlException e) {
+ throw UserException
+ .permissionError()
+ .message("Unauthorized to drop table", e)
+ .build(logger);
+ } catch (IOException e) {
+ throw UserException
+ .dataWriteError()
+ .message("Failed to drop table", e)
+ .build(logger);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index 56a1f00..446e12a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -223,7 +223,7 @@ public class ParquetFormatPlugin implements FormatPlugin{
if (files.length == 0) {
return false;
}
- return super.isReadable(fs, files[0]);
+ return super.isFileReadable(fs, files[0]);
}
} catch (IOException e) {
logger.info("Failure while attempting to check for Parquet metadata file.", e);
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java b/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java
new file mode 100644
index 0000000..4f8fe1a
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill;
+
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+import org.junit.Assert;
+
+
+public class TestDropTable extends PlanTestBase {
+
+ private static final String CREATE_SIMPLE_TABLE = "create table %s as select 1 from cp.`employee.json`";
+ private static final String DROP_TABLE = "drop table %s";
+ private static final String BACK_TICK = "`";
+
+ @Test
+ public void testDropJsonTable() throws Exception {
+ test("use dfs_test.tmp");
+ test("alter session set `store.format` = 'json'");
+
+ final String tableName = "simple_json";
+ // create a json table
+ test(String.format(CREATE_SIMPLE_TABLE, tableName));
+
+ // drop the table
+ final String dropSql = String.format(DROP_TABLE, tableName);
+ testBuilder()
+ .sqlQuery(dropSql)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Table [%s] dropped", tableName))
+ .go();
+ }
+
+ @Test
+ public void testDropParquetTable() throws Exception {
+ test("use dfs_test.tmp");
+ final String tableName = "simple_json";
+
+ // create a parquet table
+ test(String.format(CREATE_SIMPLE_TABLE, tableName));
+
+ // drop the table
+ final String dropSql = String.format(DROP_TABLE, tableName);
+ testBuilder()
+ .sqlQuery(dropSql)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Table [%s] dropped", tableName))
+ .go();
+ }
+
+ @Test
+ public void testDropTextTable() throws Exception {
+ test("use dfs_test.tmp");
+
+ test("alter session set `store.format` = 'csv'");
+ final String csvTable = "simple_csv";
+
+ // create a csv table
+ test(String.format(CREATE_SIMPLE_TABLE, csvTable));
+
+ // drop the table
+ String dropSql = String.format(DROP_TABLE, csvTable);
+ testBuilder()
+ .sqlQuery(dropSql)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Table [%s] dropped", csvTable))
+ .go();
+
+ test("alter session set `store.format` = 'psv'");
+ final String psvTable = "simple_psv";
+
+ // create a psv table
+ test(String.format(CREATE_SIMPLE_TABLE, psvTable));
+
+ // drop the table
+ dropSql = String.format(DROP_TABLE, psvTable);
+ testBuilder()
+ .sqlQuery(dropSql)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Table [%s] dropped", psvTable))
+ .go();
+
+ test("alter session set `store.format` = 'tsv'");
+ final String tsvTable = "simple_tsv";
+
+ // create a tsv table
+ test(String.format(CREATE_SIMPLE_TABLE, tsvTable));
+
+ // drop the table
+ dropSql = String.format(DROP_TABLE, tsvTable);
+ testBuilder()
+ .sqlQuery(dropSql)
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Table [%s] dropped", tsvTable))
+ .go();
+ }
+
+ @Test
+ public void testNonHomogenousDrop() throws Exception {
+ test("use dfs_test.tmp");
+ final String tableName = "homogenous_table";
+
+ // create a parquet table
+ test(String.format(CREATE_SIMPLE_TABLE, tableName));
+
+ // create a json table within the same directory
+ test("alter session set `store.format` = 'json'");
+ final String nestedJsonTable = tableName + Path.SEPARATOR + "json_table";
+ test(String.format(CREATE_SIMPLE_TABLE, BACK_TICK + nestedJsonTable + BACK_TICK));
+
+ test("show files from " + tableName);
+
+ boolean dropFailed = false;
+ // this should fail, because the directory contains non-homogenous files
+ try {
+ test(String.format(DROP_TABLE, tableName));
+ } catch (UserException e) {
+ Assert.assertTrue(e.getMessage().contains("VALIDATION ERROR"));
+ dropFailed = true;
+ }
+
+ Assert.assertTrue("Dropping of non-homogeneous table should have failed", dropFailed);
+
+ // drop the individual json table
+ testBuilder()
+ .sqlQuery(String.format(DROP_TABLE, BACK_TICK + nestedJsonTable + BACK_TICK))
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Table [%s] dropped", nestedJsonTable))
+ .go();
+
+ // Now drop should succeed
+ testBuilder()
+ .sqlQuery(String.format(DROP_TABLE, tableName))
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Table [%s] dropped", tableName))
+ .go();
+ }
+
+ @Test
+ public void testDropOnImmutableSchema() throws Exception {
+ boolean dropFailed = false;
+ try {
+ test("drop table dfs.`/tmp`");
+ } catch (UserException e) {
+ Assert.assertTrue(e.getMessage().contains("PARSE ERROR"));
+ dropFailed = true;
+ }
+
+ Assert.assertTrue("Dropping table on immutable schema failed", dropFailed);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/2a191847/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
index ef3765e..998e35d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.impersonation;
import com.google.common.base.Joiner;
import com.google.common.collect.Maps;
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.store.dfs.WorkspaceConfig;
import org.apache.hadoop.fs.FileSystem;
@@ -26,6 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -36,7 +38,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
/**
- * Tests impersonation on metadata related queries as SHOW FILES, SHOW TABLES, CREATE VIEW and CREATE TABLE
+ * Tests impersonation on metadata related queries as SHOW FILES, SHOW TABLES, CREATE VIEW, CREATE TABLE and DROP TABLE
*/
public class TestImpersonationMetadata extends BaseTestImpersonation {
private static final String user1 = "drillTestUser1";
@@ -83,9 +85,49 @@ public class TestImpersonationMetadata extends BaseTestImpersonation {
// Create /drillTestGrp1_700 directory with permissions 700 (owned by user1)
createAndAddWorkspace("drillTestGrp1_700", "/drillTestGrp1_700", (short)0700, user1, group1, workspaces);
+ // create /user2_workspace1 with 775 permissions (owner by user1)
+ createAndAddWorkspace("user2_workspace1", "/user2_workspace1", (short)0775, user2, group1, workspaces);
+
+ // create /user2_workspace with 755 permissions (owner by user1)
+ createAndAddWorkspace("user2_workspace2", "/user2_workspace2", (short)0755, user2, group1, workspaces);
+
return workspaces;
}
+ @Test
+ public void testDropTable() throws Exception {
+
+ // create tables as user2
+ updateClient(user2);
+ test(String.format("use `%s.user2_workspace1`", MINIDFS_STORAGE_PLUGIN_NAME));
+ // create a table that can be dropped by another user in a different group
+ test("create table parquet_table_775 as select * from cp.`employee.json`");
+
+ // create a table that cannot be dropped by another user
+ test(String.format("use `%s.user2_workspace2`", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("create table parquet_table_700 as select * from cp.`employee.json`");
+
+ // Drop tables as user1
+ updateClient(user1);
+ test(String.format("use `%s.user2_workspace1`", MINIDFS_STORAGE_PLUGIN_NAME));
+ testBuilder()
+ .sqlQuery("drop table parquet_table_775")
+ .unOrdered()
+ .baselineColumns("ok", "summary")
+ .baselineValues(true, String.format("Table [%s] dropped", "parquet_table_775"))
+ .go();
+
+ test(String.format("use `%s.user2_workspace2`", MINIDFS_STORAGE_PLUGIN_NAME));
+ boolean dropFailed = false;
+ try {
+ test("drop table parquet_table_700");
+ } catch (UserException e) {
+ Assert.assertTrue(e.getMessage().contains("PERMISSION ERROR"));
+ dropFailed = true;
+ }
+ Assert.assertTrue("Permission checking failed during drop table", dropFailed);
+ }
+
@Test // DRILL-3037
public void testImpersonatingProcessUser() throws Exception {
updateClient(processUser);
[4/5] drill git commit: DRILL-3622: When user authentication is
enabled, enforce admin privileges to update SYSTEM options
Posted by ad...@apache.org.
DRILL-3622: When user authentication is enabled, enforce admin privileges to update SYSTEM options
+ define what user is considered an admin
+ remove a stray file in test module (exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/RootResource.java)
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/41fc9ca5
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/41fc9ca5
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/41fc9ca5
Branch: refs/heads/master
Commit: 41fc9ca52c6983e78b2a609b96b087814a3c7969
Parents: 1601a7c
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Sep 8 17:01:14 2015 +0000
Committer: adeneche <ad...@gmail.com>
Committed: Tue Sep 8 15:35:45 2015 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/ExecConstants.java | 16 +++
.../org/apache/drill/exec/ops/QueryContext.java | 4 +
.../planner/sql/handlers/SetOptionHandler.java | 21 +++-
.../user/security/UserAuthenticatorFactory.java | 4 +-
.../server/options/SystemOptionManager.java | 2 +
.../exec/server/options/TypeValidators.java | 16 +++
.../drill/exec/util/ImpersonationUtil.java | 40 +++++++
.../java/org/apache/drill/BaseTestQuery.java | 15 ++-
.../security/UserAuthenticatorTestImpl.java | 19 +++-
.../exec/server/TestOptionsAuthEnabled.java | 112 +++++++++++++++++++
.../drill/exec/server/rest/RootResource.java | 26 -----
11 files changed, 243 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 140e9a8..0f6a5bb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec;
import org.apache.drill.exec.physical.impl.common.HashTable;
import org.apache.drill.exec.server.options.OptionValidator;
+import org.apache.drill.exec.server.options.TypeValidators.AdminOptionValidator;
import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator;
import org.apache.drill.exec.server.options.TypeValidators.DoubleValidator;
import org.apache.drill.exec.server.options.TypeValidators.EnumeratedStringValidator;
@@ -29,6 +30,7 @@ import org.apache.drill.exec.server.options.TypeValidators.RangeLongValidator;
import org.apache.drill.exec.server.options.TypeValidators.RangeDoubleValidator;
import org.apache.drill.exec.server.options.TypeValidators.StringValidator;
import org.apache.drill.exec.testing.ExecutionControls;
+import org.apache.drill.exec.util.ImpersonationUtil;
public interface ExecConstants {
public static final String ZK_RETRY_TIMES = "drill.exec.zk.retry.count";
@@ -254,4 +256,18 @@ public interface ExecConstants {
public static final String CTAS_PARTITIONING_HASH_DISTRIBUTE = "store.partition.hash_distribute";
public static final BooleanValidator CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR = new BooleanValidator(CTAS_PARTITIONING_HASH_DISTRIBUTE, false);
+
+ /**
+ * Option whose value is a comma separated list of admin usernames. Admin users are users who have special privileges
+ * such as changing system options.
+ */
+ String ADMIN_USERS_KEY = "security.admin.users";
+ StringValidator ADMIN_USERS_VALIDATOR =
+ new AdminOptionValidator(ADMIN_USERS_KEY, ImpersonationUtil.getProcessUserName());
+
+ /**
+ * Option whose value is a comma separated list of admin usergroups.
+ */
+ String ADMIN_USER_GROUPS_KEY = "security.admin.user_groups";
+ StringValidator ADMIN_USER_GROUPS_VALIDATOR = new AdminOptionValidator(ADMIN_USER_GROUPS_KEY, "");
}
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index 1cd67ac..238907d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -217,6 +217,10 @@ public class QueryContext implements AutoCloseable, OptimizerRulesContext {
return getConfig().getBoolean(ExecConstants.IMPERSONATION_ENABLED);
}
+ public boolean isUserAuthenticationEnabled() {
+ return getConfig().getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED);
+ }
+
public DrillOperatorTable getDrillOperatorTable() {
return table;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java
index 2b1a230..85ab528 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java
@@ -26,17 +26,21 @@ import org.apache.calcite.tools.ValidationException;
import org.apache.calcite.util.NlsString;
import org.apache.drill.common.exceptions.ExpressionParsingException;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlSetOption;
public class SetOptionHandler extends AbstractSqlHandler {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SetOptionHandler.class);
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SetOptionHandler.class);
private final QueryContext context;
@@ -65,6 +69,21 @@ public class SetOptionHandler extends AbstractSqlHandler {
default:
throw new ValidationException("Invalid OPTION scope. Scope must be SESSION or SYSTEM.");
}
+
+ if (type == OptionType.SYSTEM) {
+ // If the user authentication is enabled, make sure the user who is trying to change the system option has
+ // administrative privileges.
+ if (context.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ context.getQueryUserName(),
+ context.getOptions().getOption(ExecConstants.ADMIN_USERS_KEY).string_val,
+ context.getOptions().getOption(ExecConstants.ADMIN_USER_GROUPS_KEY).string_val)) {
+ throw UserException.permissionError()
+ .message("Not authorized to change SYSTEM options.")
+ .build(logger);
+ }
+ }
+
final OptionValue optionValue = createOptionValue(name, type, (SqlLiteral) value);
context.getOptions().setOption(optionValue);
}else{
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorFactory.java
index 51a5979..0fe302d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorFactory.java
@@ -92,13 +92,13 @@ public class UserAuthenticatorFactory {
return authenticator;
} catch(IllegalArgumentException | IllegalAccessException | InstantiationException e) {
throw new DrillbitStartupException(
- String.format("Failed to create and initialize the UserAuthenticator class '{}'",
+ String.format("Failed to create and initialize the UserAuthenticator class '%s'",
clazz.getCanonicalName()), e);
}
}
}
- String errMsg = String.format("Failed to find the implementation of '{}' for type '{}'",
+ String errMsg = String.format("Failed to find the implementation of '%s' for type '%s'",
UserAuthenticator.class.getCanonicalName(), authImplConfigured);
logger.error(errMsg);
throw new DrillbitStartupException(errMsg);
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 1b9906d..118f7ad 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -111,6 +111,8 @@ public class SystemOptionManager extends BaseOptionManager {
ExecConstants.NEW_VIEW_DEFAULT_PERMS_VALIDATOR,
ExecConstants.USE_OLD_ASSIGNMENT_CREATOR_VALIDATOR,
ExecConstants.CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR,
+ ExecConstants.ADMIN_USERS_VALIDATOR,
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR,
QueryClassLoader.JAVA_COMPILER_VALIDATOR,
QueryClassLoader.JAVA_COMPILER_JANINO_MAXSIZE,
QueryClassLoader.JAVA_COMPILER_DEBUG,
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
index 73f067b..53cd4f3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
@@ -133,6 +133,22 @@ public class TypeValidators {
}
}
+ public static class AdminOptionValidator extends StringValidator {
+ public AdminOptionValidator(String name, String def) {
+ super(name, def);
+ }
+
+ @Override
+ public void validate(OptionValue v) {
+ if (v.type != OptionType.SYSTEM) {
+ throw UserException.validationError()
+ .message("Admin related settings can only be set at SYSTEM level scope. Given scope '%s'.", v.type)
+ .build(logger);
+ }
+ super.validate(v);
+ }
+ }
+
/**
* Validator that checks if the given value is included in a list of acceptable values. Case insensitive.
*/
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
index aa766be..37fdc40 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ImpersonationUtil.java
@@ -17,7 +17,9 @@
*/
package org.apache.drill.exec.util;
+import com.google.common.base.Splitter;
import com.google.common.base.Strings;
+import com.google.common.collect.Sets;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
@@ -26,6 +28,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.util.Set;
/**
* Utilities for impersonation purpose.
@@ -33,6 +36,8 @@ import java.security.PrivilegedExceptionAction;
public class ImpersonationUtil {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ImpersonationUtil.class);
+ private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+
/**
* Create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation} of operator owner if operator
* owner is valid. Otherwise create and return proxy user {@link org.apache.hadoop.security.UserGroupInformation} for
@@ -152,4 +157,39 @@ public class ImpersonationUtil {
return fs;
}
+
+ /**
+ * Given admin user/group list, finds whether the given username has admin privileges.
+ *
+ * @param userName User who is checked for administrative privileges.
+ * @param adminUsers Comma separated list of admin usernames,
+ * @param adminGroups Comma separated list of admin usergroups
+ * @return
+ */
+ public static boolean hasAdminPrivileges(final String userName, final String adminUsers, final String adminGroups) {
+ // Process user is by default an admin
+ if (getProcessUserName().equals(userName)) {
+ return true;
+ }
+
+ final Set<String> adminUsersSet = Sets.newHashSet(SPLITTER.split(adminUsers));
+ if (adminUsersSet.contains(userName)) {
+ return true;
+ }
+
+ final UserGroupInformation ugi = createProxyUgi(userName);
+ final String[] userGroups = ugi.getGroupNames();
+ if (userGroups == null || userGroups.length == 0) {
+ return false;
+ }
+
+ final Set<String> adminUserGroupsSet = Sets.newHashSet(SPLITTER.split(adminGroups));
+ for (String userGroup : userGroups) {
+ if (adminUserGroupsSet.contains(userGroup)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index eaf8765..1381949 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -44,6 +44,7 @@ import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.rpc.user.UserSession;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
@@ -211,8 +212,20 @@ public class BaseTestQuery extends ExecTest {
* @param user
*/
public static void updateClient(String user) throws Exception {
+ updateClient(user, null);
+ }
+
+ /*
+ * Close the current <i>client</i> and open a new client for the given user and password credentials. Tests
+ * executed after this method call use the new <i>client</i>.
+ * @param user
+ */
+ public static void updateClient(final String user, final String password) throws Exception {
final Properties props = new Properties();
- props.setProperty("user", user);
+ props.setProperty(UserSession.USER, user);
+ if (password != null) {
+ props.setProperty(UserSession.PASSWORD, password);
+ }
updateClient(props);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorTestImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorTestImpl.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorTestImpl.java
index c89471f..dc30797 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorTestImpl.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorTestImpl.java
@@ -17,9 +17,10 @@
*/
package org.apache.drill.exec.rpc.user.security;
-import com.google.common.base.Strings;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.exception.DrillbitStartupException;
+import org.apache.drill.exec.util.ImpersonationUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
@@ -33,8 +34,20 @@ public class UserAuthenticatorTestImpl implements UserAuthenticator {
public static final String TEST_USER_1 = "testUser1";
public static final String TEST_USER_2 = "testUser2";
+ public static final String ADMIN_USER = "admin";
+ public static final String PROCESS_USER = ImpersonationUtil.getProcessUserName();
public static final String TEST_USER_1_PASSWORD = "testUser1Password";
public static final String TEST_USER_2_PASSWORD = "testUser2Password";
+ public static final String ADMIN_USER_PASSWORD = "adminUserPw";
+ public static final String PROCESS_USER_PASSWORD = "processUserPw";
+
+ public static final String ADMIN_GROUP = "admingrp";
+
+ static {
+ UserGroupInformation.createUserForTesting("testUser1", new String[]{"g1", ADMIN_GROUP});
+ UserGroupInformation.createUserForTesting("testUser2", new String[]{ "g1" });
+ UserGroupInformation.createUserForTesting("admin", new String[]{ ADMIN_GROUP });
+ }
@Override
public void setup(DrillConfig drillConfig) throws DrillbitStartupException {
@@ -50,7 +63,9 @@ public class UserAuthenticatorTestImpl implements UserAuthenticator {
}
if (!(TEST_USER_1.equals(user) && TEST_USER_1_PASSWORD.equals(password)) &&
- !(TEST_USER_2.equals(user) && TEST_USER_2_PASSWORD.equals(password))) {
+ !(TEST_USER_2.equals(user) && TEST_USER_2_PASSWORD.equals(password)) &&
+ !(ADMIN_USER.equals(user) && ADMIN_USER_PASSWORD.equals(password)) &&
+ !(PROCESS_USER.equals(user) && PROCESS_USER_PASSWORD.equals(password))) {
throw new UserAuthenticationException();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java
new file mode 100644
index 0000000..b029caa
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.Properties;
+
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.ADMIN_GROUP;
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.ADMIN_USER;
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.ADMIN_USER_PASSWORD;
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.PROCESS_USER;
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.PROCESS_USER_PASSWORD;
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_1;
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_1_PASSWORD;
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_2;
+import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_2_PASSWORD;
+
+/**
+ * Test setting system scoped options with user authentication enabled. (DRILL-3622)
+ */
+public class TestOptionsAuthEnabled extends BaseTestQuery {
+ private static final String setSysOptionQuery =
+ String.format("ALTER SYSTEM SET `%s` = %d;", ExecConstants.SLICE_TARGET, 200);
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ // Create a new DrillConfig which has user authentication enabled and test authenticator set
+ final Properties props = cloneDefaultTestConfigProperties();
+ props.setProperty(ExecConstants.USER_AUTHENTICATION_ENABLED, "true");
+ props.setProperty(ExecConstants.USER_AUTHENTICATOR_IMPL, UserAuthenticatorTestImpl.TYPE);
+
+ updateTestCluster(1, DrillConfig.create(props));
+
+ updateClient(PROCESS_USER, PROCESS_USER_PASSWORD);
+
+ // Add user "admin" to admin username list
+ test(String.format("ALTER SYSTEM SET `%s`='%s,%s'", ExecConstants.ADMIN_USERS_KEY, ADMIN_USER, PROCESS_USER));
+
+ // Set "admingrp" to admin username list
+ test(String.format("ALTER SYSTEM SET `%s`='%s'", ExecConstants.ADMIN_USER_GROUPS_KEY, ADMIN_GROUP));
+ }
+
+ @Test
+ public void updateSysOptAsAdminUser() throws Exception {
+ updateClient(ADMIN_USER, ADMIN_USER_PASSWORD);
+ setOptHelper();
+ }
+
+ @Test
+ public void updateSysOptAsNonAdminUser() throws Exception {
+ updateClient(TEST_USER_2, TEST_USER_2_PASSWORD);
+ errorMsgTestHelper(setSysOptionQuery, "Not authorized to change SYSTEM options.");
+ }
+
+ @Test
+ public void updateSysOptAsUserInAdminGroup() throws Exception {
+ updateClient(TEST_USER_1, TEST_USER_1_PASSWORD);
+ setOptHelper();
+ }
+
+ @Test
+ public void trySettingAdminOptsAtSessionScopeAsAdmin() throws Exception {
+ updateClient(ADMIN_USER, ADMIN_USER_PASSWORD);
+ final String setOptionQuery =
+ String.format("ALTER SESSION SET `%s`='%s,%s'", ExecConstants.ADMIN_USERS_KEY, ADMIN_USER, PROCESS_USER);
+ errorMsgTestHelper(setOptionQuery, "Admin related settings can only be set at SYSTEM level scope");
+ }
+
+ @Test
+ public void trySettingAdminOptsAtSessionScopeAsNonAdmin() throws Exception {
+ updateClient(TEST_USER_2, TEST_USER_2_PASSWORD);
+ final String setOptionQuery =
+ String.format("ALTER SESSION SET `%s`='%s,%s'", ExecConstants.ADMIN_USERS_KEY, ADMIN_USER, PROCESS_USER);
+ errorMsgTestHelper(setOptionQuery, "Admin related settings can only be set at SYSTEM level scope");
+ }
+
+ private void setOptHelper() throws Exception {
+ try {
+ test(setSysOptionQuery);
+ testBuilder()
+ .sqlQuery(String.format("SELECT num_val FROM sys.options WHERE name = '%s' AND type = 'SYSTEM'",
+ ExecConstants.SLICE_TARGET))
+ .unOrdered()
+ .baselineColumns("num_val")
+ .baselineValues(200L)
+ .go();
+ } finally {
+ test(String.format("ALTER SYSTEM SET `%s` = %d;", ExecConstants.SLICE_TARGET, ExecConstants.SLICE_TARGET_DEFAULT));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/41fc9ca5/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/RootResource.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/RootResource.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/RootResource.java
deleted file mode 100644
index 59adad9..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/RootResource.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.server.rest;
-
-import javax.ws.rs.Path;
-
-@Path("/")
-public class RootResource {
- public int hi = 5;
- public String blue = "yo";
-}
[3/5] drill git commit: DRILL-3668: Incorrect results FIRST_VALUE
function
Posted by ad...@apache.org.
DRILL-3668: Incorrect results FIRST_VALUE function
added DefaultFrameTemplate.resetInternal() and generate code to set first value of internal batch to NULL at the end of each partition
added unit test to make sure bug has been fixed
this closes #146
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1601a7c5
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1601a7c5
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1601a7c5
Branch: refs/heads/master
Commit: 1601a7c5e9a2bb3772f12f0650e13935ab5ba563
Parents: 6db7c05
Author: adeneche <ad...@gmail.com>
Authored: Wed Aug 19 11:52:29 2015 -0700
Committer: adeneche <ad...@gmail.com>
Committed: Tue Sep 8 15:35:36 2015 -0700
----------------------------------------------------------------------
.../main/codegen/templates/FixedValueVectors.java | 8 ++++++++
.../codegen/templates/NullableValueVectors.java | 15 +++++++++++++++
.../codegen/templates/VariableLengthVectors.java | 9 +++++++++
.../impl/window/DefaultFrameTemplate.java | 7 +++++++
.../drill/exec/vector/BaseDataValueVector.java | 5 +++++
.../org/apache/drill/exec/vector/BitVector.java | 9 +++++++++
.../org/apache/drill/exec/vector/ValueVector.java | 17 +++++++++++++++++
.../physical/impl/window/TestWindowFrame.java | 10 ++++++++++
.../src/test/resources/window/3668.parquet | Bin 0 -> 395 bytes
exec/java-exec/src/test/resources/window/3668.sql | 6 ++++++
10 files changed, 86 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
index 9531987..f1eb756 100644
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -123,6 +123,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
allocateBytes(valueCount * ${type.width});
}
+ @Override
+ public void reset() {
+ allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+ allocationMonitor = 0;
+ zeroVector();
+ super.reset();
+ }
+
private void allocateBytes(final long size) {
if (size > MAX_ALLOCATION_SIZE) {
throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index 259005f..85a87f6 100644
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -168,6 +168,13 @@ public final class ${className} extends BaseDataValueVector implements <#if type
accessor.reset();
}
+ public void reset() {
+ bits.zeroVector();
+ mutator.reset();
+ accessor.reset();
+ super.reset();
+ }
+
@Override
public int getByteCapacity(){
return values.getByteCapacity();
@@ -193,6 +200,14 @@ public final class ${className} extends BaseDataValueVector implements <#if type
accessor.reset();
}
+ @Override
+ public void reset() {
+ bits.zeroVector();
+ mutator.reset();
+ accessor.reset();
+ super.reset();
+ }
+
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index c9dae77..92c3700 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -328,6 +328,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
offsetVector.zeroVector();
}
+ @Override
+ public void reset() {
+ allocationSizeInBytes = INITIAL_BYTE_COUNT;
+ allocationMonitor = 0;
+ data.readerIndex(0);
+ offsetVector.zeroVector();
+ super.reset();
+ }
+
public void reAlloc() {
final long newAllocationSize = allocationSizeInBytes*2L;
if (newAllocationSize > MAX_ALLOCATION_SIZE) {
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
index b56e421..9bde6a5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
@@ -24,6 +24,8 @@ import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.ValueVector;
import javax.inject.Named;
@@ -144,6 +146,11 @@ public abstract class DefaultFrameTemplate implements WindowFramer {
private void cleanPartition() {
partition = null;
resetValues();
+ for (VectorWrapper<?> vw : internal) {
+ if ((vw.getValueVector() instanceof BaseDataValueVector)) {
+ ((BaseDataValueVector) vw.getValueVector()).reset();
+ }
+ }
lagCopiedToInternal = false;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
index 42e0972..2fc741c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
@@ -82,4 +82,9 @@ public abstract class BaseDataValueVector extends BaseValueVector {
return data;
}
+ /**
+ * This method has a similar effect of allocateNew() without actually clearing and reallocating
+ * the value vector. The purpose is to move the value vector to a "mutate" state
+ */
+ public void reset() {}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 9d31ea8..624e737 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -104,6 +104,15 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
return true;
}
+ @Override
+ public void reset() {
+ valueCount = 0;
+ allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+ allocationMonitor = 0;
+ zeroVector();
+ super.reset();
+ }
+
/**
* Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
*
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 3948163..8ec9fb2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -37,6 +37,23 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
* A vector when instantiated, relies on a {@link org.apache.drill.exec.record.DeadBuf dead buffer}. It is important
* that vector is allocated before attempting to read or write.
*
+ * There are a few "rules" around vectors:
+ *
+ * <ul>
+ * <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
+ * <li>null vectors start with all values as null before writing anything</li>
+ * <li>for variable width types, the offset vector should be all zeros before writing</li>
+ * <li>you must call setValueCount before a vector can be read</li>
+ * <li>you should never write to a vector once it has been read.</li>
+ * </ul>
+ *
+ * Please note that the current implementation doesn't enfore those rules, hence we may find few places that
+ * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)
+ *
+ * This interface "should" strive to guarantee this order of operation:
+ * <blockquote>
+ * allocate > mutate > setvaluecount > access > clear (or allocate to start the process over).
+ * </blockquote>
*/
public interface ValueVector extends Closeable, Iterable<ValueVector> {
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
index 23e9b46..ba66fce 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
@@ -296,6 +296,16 @@ public class TestWindowFrame extends BaseTestQuery {
}
@Test
+ public void test3668Fix() throws Exception {
+ testBuilder()
+ .sqlQuery(getFile("window/3668.sql"), TEST_RES_PATH)
+ .ordered()
+ .baselineColumns("cnt").baselineValues(2L)
+ .build()
+ .run();
+ }
+
+ @Test
public void testPartitionNtile() {
Partition partition = new Partition(12);
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/test/resources/window/3668.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/3668.parquet b/exec/java-exec/src/test/resources/window/3668.parquet
new file mode 100644
index 0000000..4ff9075
Binary files /dev/null and b/exec/java-exec/src/test/resources/window/3668.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/1601a7c5/exec/java-exec/src/test/resources/window/3668.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/3668.sql b/exec/java-exec/src/test/resources/window/3668.sql
new file mode 100644
index 0000000..e7ea7ab
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/3668.sql
@@ -0,0 +1,6 @@
+select count(fv) as cnt
+from (
+ select first_value(c2) over(partition by c2 order by c1) as fv
+ from dfs_test.`%s/window/3668.parquet`
+)
+where fv = 'e'
\ No newline at end of file
[2/5] drill git commit: DRILL-3566: Fix:
PreparedStatement.executeQuery() got ClassCastException.
Posted by ad...@apache.org.
DRILL-3566: Fix: PreparedStatement.executeQuery() got ClassCastException.
Main:
Restored DrillResultSetImpl(...)'s statement parameter from overly
restrictive DrillStatementImpl to AvaticaStatement and removed caller
cast that was throwing. (Relatedly, adjusted getStatement() and moved
internal casting from statement to connection.)
Added basic test of querying via PreparedStatement. [PreparedStatementTest]
Added some case test of statement-creation methods. [ConnectionTest]
Removed now-unneeded DrillPrepareResult cast.
Eliminated redundant getConnection() and getClient calls, intermediate variables, etc.
Fixed comment error.
this closes #143
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6db7c05a
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6db7c05a
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6db7c05a
Branch: refs/heads/master
Commit: 6db7c05a95950ccb91dddb4e6097da8f72c4dd35
Parents: 7a864a4
Author: dbarclay <db...@maprtech.com>
Authored: Mon Jul 27 19:27:50 2015 -0700
Committer: adeneche <ad...@gmail.com>
Committed: Tue Sep 8 15:35:24 2015 -0700
----------------------------------------------------------------------
.../drill/jdbc/impl/DrillJdbc41Factory.java | 4 +-
.../jdbc/impl/DrillPreparedStatementImpl.java | 3 +-
.../drill/jdbc/impl/DrillResultSetImpl.java | 28 ++-
.../org/apache/drill/jdbc/ConnectionTest.java | 203 ++++++++++++++++++-
.../org/apache/drill/jdbc/JdbcTestBase.java | 3 +-
.../drill/jdbc/PreparedStatementTest.java | 34 +++-
6 files changed, 248 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/6db7c05a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillJdbc41Factory.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillJdbc41Factory.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillJdbc41Factory.java
index 11191ae..9723358 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillJdbc41Factory.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillJdbc41Factory.java
@@ -103,9 +103,7 @@ public class DrillJdbc41Factory extends DrillFactory {
TimeZone timeZone) {
final ResultSetMetaData metaData =
newResultSetMetaData(statement, prepareResult.getColumnList());
- return new DrillResultSetImpl( (DrillStatementImpl) statement,
- (DrillPrepareResult) prepareResult,
- metaData, timeZone);
+ return new DrillResultSetImpl(statement, prepareResult, metaData, timeZone);
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/6db7c05a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillPreparedStatementImpl.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillPreparedStatementImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillPreparedStatementImpl.java
index 86683cb..871298f 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillPreparedStatementImpl.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillPreparedStatementImpl.java
@@ -30,7 +30,8 @@ import net.hydromatic.avatica.AvaticaPreparedStatement;
* Implementation of {@link java.sql.PreparedStatement} for Drill.
*
* <p>
- * This class has sub-classes which implement JDBC 3.0 and JDBC 4.0 APIs; it is instantiated using
+ * This class has sub-classes which implement JDBC 3.0 and JDBC 4.0 APIs; it is
+ * instantiated using
* {@link net.hydromatic.avatica.AvaticaFactory#newPreparedStatement}.
* </p>
*/
http://git-wip-us.apache.org/repos/asf/drill/blob/6db7c05a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java
index 95fee00..039d133 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillResultSetImpl.java
@@ -43,6 +43,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import net.hydromatic.avatica.AvaticaPrepareResult;
import net.hydromatic.avatica.AvaticaResultSet;
+import net.hydromatic.avatica.AvaticaStatement;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
@@ -75,7 +76,7 @@ class DrillResultSetImpl extends AvaticaResultSet implements DrillResultSet {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(DrillResultSetImpl.class);
- private final DrillStatementImpl statement;
+ private final DrillConnectionImpl connection;
SchemaChangeListener changeListener;
final ResultsListener resultsListener;
@@ -87,18 +88,16 @@ class DrillResultSetImpl extends AvaticaResultSet implements DrillResultSet {
boolean hasPendingCancelationNotification;
- DrillResultSetImpl(DrillStatementImpl statement, AvaticaPrepareResult prepareResult,
+ DrillResultSetImpl(AvaticaStatement statement, AvaticaPrepareResult prepareResult,
ResultSetMetaData resultSetMetaData, TimeZone timeZone) {
super(statement, prepareResult, resultSetMetaData, timeZone);
- this.statement = statement;
+ connection = (DrillConnectionImpl) statement.getConnection();
+ client = connection.getClient();
final int batchQueueThrottlingThreshold =
- this.getStatement().getConnection().getClient().getConfig().getInt(
+ client.getConfig().getInt(
ExecConstants.JDBC_BATCH_QUEUE_THROTTLING_THRESHOLD );
- resultsListener = new ResultsListener( batchQueueThrottlingThreshold );
- DrillConnection c = (DrillConnection) statement.getConnection();
- DrillClient client = c.getClient();
+ resultsListener = new ResultsListener(batchQueueThrottlingThreshold);
batchLoader = new RecordBatchLoader(client.getAllocator());
- this.client = client;
cursor = new DrillCursor(this);
}
@@ -107,7 +106,7 @@ class DrillResultSetImpl extends AvaticaResultSet implements DrillResultSet {
* ResultSet is closed.
*
* @throws ExecutionCanceledSqlException if ResultSet is closed because of
- * cancelation and no QueryCanceledSqlException had been thrown yet
+ * cancelation and no QueryCanceledSqlException has been thrown yet
* for this ResultSet
* @throws AlreadyClosedSqlException if ResultSet is closed
* @throws SQLException if error in calling {@link #isClosed()}
@@ -850,9 +849,9 @@ class DrillResultSetImpl extends AvaticaResultSet implements DrillResultSet {
}
@Override
- public DrillStatementImpl getStatement() {
- // Note: No already-closed exception for close().
- return statement;
+ public AvaticaStatement getStatement() {
+ // Note: No already-closed exception for getStatement().
+ return super.getStatement();
}
@Override
@@ -1334,10 +1333,7 @@ class DrillResultSetImpl extends AvaticaResultSet implements DrillResultSet {
@Override
protected DrillResultSetImpl execute() throws SQLException{
- DrillConnectionImpl connection = (DrillConnectionImpl) statement.getConnection();
-
- connection.getClient().runQuery(QueryType.SQL, this.prepareResult.getSql(),
- resultsListener);
+ client.runQuery(QueryType.SQL, this.prepareResult.getSql(), resultsListener);
connection.getDriver().handler.onStatementExecute(statement, null);
super.execute();
http://git-wip-us.apache.org/repos/asf/drill/blob/6db7c05a/exec/jdbc/src/test/java/org/apache/drill/jdbc/ConnectionTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/ConnectionTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/ConnectionTest.java
index 8735146..198cf4c 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/ConnectionTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/ConnectionTest.java
@@ -21,15 +21,20 @@ import org.apache.drill.jdbc.Driver;
import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLFeatureNotSupportedException;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -44,6 +49,7 @@ public class ConnectionTest extends JdbcTestBase {
private static ExecutorService executor;
+
@BeforeClass
public static void setUpConnection() throws SQLException {
// (Note: Can't use JdbcTest's connect(...) because JdbcTest closes
@@ -60,6 +66,201 @@ public class ConnectionTest extends JdbcTestBase {
executor.shutdown();
}
+ private static void emitSupportExceptionWarning() {
+ System.err.println(
+ "Note: Still throwing older-Avatica UnsupportedOperationException"
+ + " instead of less-noncompliant SQLFeatureNotSupportedException" );
+ }
+
+
+ ////////////////////////////////////////
+ // Basic tests of statement creation methods (not necessarily executing
+ // statements):
+
+ //////////
+ // Simplest cases of createStatement, prepareStatement, prepareCall:
+
+ @Test
+ public void testCreateStatementBasicCaseWorks() throws SQLException {
+ Statement stmt = connection.createStatement();
+ ResultSet rs = stmt.executeQuery( "VALUES 1" );
+ assertTrue( rs.next() );
+ }
+
+ @Test
+ public void testPrepareStatementBasicCaseWorks() throws SQLException {
+ PreparedStatement stmt = connection.prepareStatement( "VALUES 1" );
+ ResultSet rs = stmt.executeQuery();
+ assertTrue( rs.next() );
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ public void testPrepareCallThrows() throws SQLException {
+ try {
+ connection.prepareCall( "VALUES 1" );
+ }
+ catch ( UnsupportedOperationException e) {
+ // TODO(DRILL-2769): Purge this mapping when right exception is thrown.
+ emitSupportExceptionWarning();
+ throw new SQLFeatureNotSupportedException(
+ "Note: Still throwing UnsupportedOperationException ", e );
+ }
+ }
+
+ //////////
+ // createStatement(int, int):
+
+ @Test
+ public void testCreateStatement_overload2_supportedCase_returns() throws SQLException {
+ connection.createStatement( ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_READ_ONLY );
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ @Ignore( "until unsupported characteristics are rejected" )
+ public void testCreateStatement_overload2_unsupportedType1_throws() throws SQLException {
+ connection.createStatement( ResultSet.TYPE_SCROLL_INSENSITIVE,
+ ResultSet.CONCUR_READ_ONLY );
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ @Ignore( "until unsupported characteristics are rejected" )
+ public void testCreateStatement_overload2_unsupportedType2_throws() throws SQLException {
+ connection.createStatement( ResultSet.TYPE_SCROLL_SENSITIVE,
+ ResultSet.CONCUR_READ_ONLY);
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ @Ignore( "until unsupported characteristics are rejected" )
+ public void testCreateStatement_overload2_unsupportedConcurrency_throws() throws SQLException {
+ connection.createStatement( ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_UPDATABLE );
+ }
+
+
+ //////////
+ // prepareStatement(String, int, int, int):
+
+ @Test
+ public void testPrepareStatement_overload2_supportedCase_returns() throws SQLException {
+ connection.prepareStatement( "VALUES 1",
+ ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_READ_ONLY );
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ @Ignore( "until unsupported characteristics are rejected" )
+ public void testPrepareStatement_overload2_unsupportedType1_throws() throws SQLException {
+ connection.prepareStatement( "VALUES 1",
+ ResultSet.TYPE_SCROLL_INSENSITIVE,
+ ResultSet.CONCUR_READ_ONLY );
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ @Ignore( "until unsupported characteristics are rejected" )
+ public void testPrepareStatement_overload2_unsupportedType2_throws() throws SQLException {
+ connection.prepareStatement( "VALUES 1",
+ ResultSet.TYPE_SCROLL_SENSITIVE,
+ ResultSet.CONCUR_READ_ONLY );
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ @Ignore( "until unsupported characteristics are rejected" )
+ public void testPrepareStatement_overload2_unsupportedConcurrency_throws() throws SQLException {
+ connection.prepareStatement( "VALUES 1",
+ ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_UPDATABLE );
+ }
+
+
+ //////////
+ // createStatement(int, int, int) (case not covered with
+ // createStatement(int, int)):
+
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ @Ignore( "until unsupported characteristics are rejected" )
+ public void testCreateStatement_overload3_unsupportedHoldability_throws() throws SQLException {
+ connection.createStatement( ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_READ_ONLY,
+ ResultSet.CLOSE_CURSORS_AT_COMMIT);
+ }
+
+
+ //////////
+ // prepareStatement(int, int, int) (case not covered with
+ // prepareStatement(int, int)):
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ @Ignore( "until unsupported characteristics are rejected" )
+ public void testPrepareStatement_overload3_unsupportedHoldability_throws() throws SQLException {
+ connection.prepareStatement( "VALUES 1",
+ ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_READ_ONLY,
+ ResultSet.CLOSE_CURSORS_AT_COMMIT );
+ }
+
+ //////////
+ // prepareCall(String, int, int, int):
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ public void testCreateCall_overload3_throws() throws SQLException {
+ try {
+ connection.prepareCall( "VALUES 1",
+ ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_READ_ONLY,
+ ResultSet.HOLD_CURSORS_OVER_COMMIT );
+ }
+ catch ( UnsupportedOperationException e) {
+ // TODO(DRILL-2769): Purge this mapping when right exception is thrown.
+ emitSupportExceptionWarning();
+ throw new SQLFeatureNotSupportedException(
+ "Note: Still throwing UnsupportedOperationException ", e );
+ }
+ }
+
+ //////////
+ // remaining prepareStatement(...):
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ public void testPrepareStatement_overload4_throws() throws SQLException {
+ try {
+ connection.prepareStatement( "VALUES 1", Statement.RETURN_GENERATED_KEYS );
+ }
+ catch ( UnsupportedOperationException e) {
+ // TODO(DRILL-2769): Purge this mapping when right exception is thrown.
+ emitSupportExceptionWarning();
+ throw new SQLFeatureNotSupportedException(
+ "Note: Still throwing UnsupportedOperationException ", e );
+ }
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ public void testPrepareStatement_overload5_throws() throws SQLException {
+ try {
+ connection.prepareStatement( "VALUES 1", new int[] { 1 } );
+ }
+ catch ( UnsupportedOperationException e) {
+ // TODO(DRILL-2769): Purge this mapping when right exception is thrown.
+ emitSupportExceptionWarning();
+ throw new SQLFeatureNotSupportedException(
+ "Note: Still throwing UnsupportedOperationException ", e );
+ }
+ }
+
+ @Test( expected = SQLFeatureNotSupportedException.class )
+ public void testPrepareStatement_overload6_throws() throws SQLException {
+ try {
+ connection.prepareStatement( "VALUES 1 AS colA", new String[] { "colA" } );
+ }
+ catch ( UnsupportedOperationException e) {
+ // TODO(DRILL-2769): Purge this mapping when right exception is thrown.
+ emitSupportExceptionWarning();
+ throw new SQLFeatureNotSupportedException(
+ "Note: Still throwing UnsupportedOperationException ", e );
+ }
+ }
+
////////////////////////////////////////
// Network timeout methods:
http://git-wip-us.apache.org/repos/asf/drill/blob/6db7c05a/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
index af49f7d..7fb601d 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
@@ -43,7 +43,8 @@ import org.junit.runner.Description;
// (e.g., the reusing of connections, the automatic interception of test
// failures and resetting of connections, etc.).
public class JdbcTestBase extends ExecTest {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JdbcTestBase.class);
+ @SuppressWarnings("unused")
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JdbcTestBase.class);
@Rule
public final TestRule watcher = new TestWatcher() {
http://git-wip-us.apache.org/repos/asf/drill/blob/6db7c05a/exec/jdbc/src/test/java/org/apache/drill/jdbc/PreparedStatementTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/PreparedStatementTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/PreparedStatementTest.java
index ad3538d..bea309f 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/PreparedStatementTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/PreparedStatementTest.java
@@ -17,7 +17,9 @@
*/
package org.apache.drill.jdbc;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
import static org.hamcrest.CoreMatchers.*;
import org.hamcrest.Matcher;
@@ -30,10 +32,14 @@ import java.sql.Clob;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
+/**
+ * Test for Drill's implementation of PreparedStatement's methods.
+ */
public class PreparedStatementTest extends JdbcTestBase {
/** Fuzzy matcher for parameters-not-supported message assertions. (Based on
@@ -57,12 +63,30 @@ public class PreparedStatementTest extends JdbcTestBase {
connection.close();
}
+
+ //////////
+ // Basic querying-works test:
+
+ /** Tests that basic executeQuery() (with query statement) works. */
+ @Test
+ public void testExecuteQueryBasicCaseWorks() throws SQLException {
+ PreparedStatement stmt = connection.prepareStatement( "VALUES 11" );
+ ResultSet rs = stmt.executeQuery();
+ assertThat( "Unexpected column count",
+ rs.getMetaData().getColumnCount(), equalTo( 1 ) );
+ assertTrue( "No expected first row", rs.next() );
+ assertThat( rs.getInt( 1 ), equalTo( 11 ) );
+ assertFalse( "Unexpected second row", rs.next() );
+ }
+
+
+ //////////
// Parameters-not-implemented tests:
/** Tests that basic case of trying to set parameter says not supported. */
@Test( expected = SQLFeatureNotSupportedException.class )
public void testParamSettingSaysUnsupported() throws SQLException {
- PreparedStatement prepStmt = connection.prepareStatement( "SELECT ?, ?" );
+ PreparedStatement prepStmt = connection.prepareStatement( "VALUES ?, ?" );
try {
prepStmt.setInt( 0, 123456789 );
}
@@ -77,7 +101,7 @@ public class PreparedStatementTest extends JdbcTestBase {
/** Tests that "not supported" has priority over "bad index" check. */
@Test( expected = SQLFeatureNotSupportedException.class )
public void testParamSettingWithImpossibleIndexSaysUnsupported() throws SQLException {
- PreparedStatement prepStmt = connection.prepareStatement( "SELECT ?, ?" );
+ PreparedStatement prepStmt = connection.prepareStatement( "VALUES ?, ?" );
try {
prepStmt.setString( -1, "some value" );
}
@@ -92,7 +116,7 @@ public class PreparedStatementTest extends JdbcTestBase {
/** Tests that "not supported" has priority over "bad index" check. */
@Test( expected = SQLFeatureNotSupportedException.class )
public void testParamSettingWithInconsistentIndexSaysUnsupported() throws SQLException {
- PreparedStatement prepStmt = connection.prepareStatement( "SELECT ?, ?" );
+ PreparedStatement prepStmt = connection.prepareStatement( "VALUES ?, ?" );
try {
prepStmt.setBytes( 4, null );
}
@@ -108,7 +132,7 @@ public class PreparedStatementTest extends JdbcTestBase {
* check. */
@Test( expected = SQLFeatureNotSupportedException.class )
public void testParamSettingWhenNoParametersIndexSaysUnsupported() throws SQLException {
- PreparedStatement prepStmt = connection.prepareStatement( "SELECT 1" );
+ PreparedStatement prepStmt = connection.prepareStatement( "VALUES 1" );
try {
prepStmt.setBytes( 4, null );
}
@@ -124,7 +148,7 @@ public class PreparedStatementTest extends JdbcTestBase {
* check. */
@Test( expected = SQLFeatureNotSupportedException.class )
public void testParamSettingWhenUnsupportedTypeSaysUnsupported() throws SQLException {
- PreparedStatement prepStmt = connection.prepareStatement( "SELECT 1" );
+ PreparedStatement prepStmt = connection.prepareStatement( "VALUES 1" );
try {
prepStmt.setClob( 2, (Clob) null );
}