You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2017/03/02 20:59:53 UTC
[26/27] drill git commit: DRILL-5290: Provide an option to build
operator table once for built-in static functions and reuse it across
queries.
DRILL-5290: Provide an option to build operator table once for built-in static functions and reuse it across queries.
close #757
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2b5a6f0b
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2b5a6f0b
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2b5a6f0b
Branch: refs/heads/master
Commit: 2b5a6f0b523859297f86298ef384cd146e425f72
Parents: 4a0fd56
Author: Padma Penumarthy <pp...@yahoo.com>
Authored: Wed Feb 22 10:31:01 2017 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Thu Mar 2 10:46:48 2017 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/ExecConstants.java | 4 ++++
.../org/apache/drill/exec/ops/QueryContext.java | 9 +++++++-
.../drill/exec/server/DrillbitContext.java | 23 ++++++++++++++++++++
.../server/options/SystemOptionManager.java | 3 ++-
4 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/2b5a6f0b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 60d6265..4f0f4d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -437,4 +437,8 @@ public interface ExecConstants {
String QUERY_PROFILE_DEBUG_OPTION = "exec.query_profile.debug_mode";
BooleanValidator QUERY_PROFILE_DEBUG_VALIDATOR = new BooleanValidator(
QUERY_PROFILE_DEBUG_OPTION, false, false);
+
+ String USE_DYNAMIC_UDFS_KEY = "exec.udf.use_dynamic";
+ BooleanValidator USE_DYNAMIC_UDFS = new BooleanValidator(USE_DYNAMIC_UDFS_KEY, true);
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2b5a6f0b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index 707815a..df3f4f4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -91,7 +91,14 @@ public class QueryContext implements AutoCloseable, OptimizerRulesContext, Schem
executionControls = new ExecutionControls(queryOptions, drillbitContext.getEndpoint());
plannerSettings = new PlannerSettings(queryOptions, getFunctionRegistry());
plannerSettings.setNumEndPoints(drillbitContext.getBits().size());
- table = new DrillOperatorTable(getFunctionRegistry(), drillbitContext.getOptionManager());
+
+ // If we do not need to support dynamic UDFs for this query, just use static operator table
+ // built at the startup. Else, build new operator table from latest version of function registry.
+ if (queryOptions.getOption(ExecConstants.USE_DYNAMIC_UDFS)) {
+ this.table = new DrillOperatorTable(drillbitContext.getFunctionImplementationRegistry(), drillbitContext.getOptionManager());
+ } else {
+ this.table = drillbitContext.getOperatorTable();
+ }
queryContextInfo = Utilities.createQueryContextInfo(session.getDefaultSchemaPath(), session.getSessionId());
contextInformation = new ContextInformation(session.getCredentials(), queryContextInfo);
http://git-wip-us.apache.org/repos/asf/drill/blob/2b5a6f0b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 23dc30c..6c68ab2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -33,6 +33,7 @@ import org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.planner.sql.DrillOperatorTable;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.rpc.control.Controller;
import org.apache.drill.exec.rpc.control.WorkEventBus;
@@ -62,6 +63,8 @@ public class DrillbitContext implements AutoCloseable {
private final CodeCompiler compiler;
private final ScanResult classpathScan;
private final LogicalPlanPersistence lpPersistence;
+ // operator table for standard SQL operators and functions, Drill built-in UDFs
+ private final DrillOperatorTable table;
public DrillbitContext(
@@ -91,6 +94,9 @@ public class DrillbitContext implements AutoCloseable {
this.systemOptions = new SystemOptionManager(lpPersistence, provider);
this.functionRegistry = new FunctionImplementationRegistry(context.getConfig(), classpathScan, systemOptions);
this.compiler = new CodeCompiler(context.getConfig(), systemOptions);
+
+ // This operator table is built once and used for all queries which do not need dynamic UDF support.
+ this.table = new DrillOperatorTable(functionRegistry, systemOptions);
}
public FunctionImplementationRegistry getFunctionImplementationRegistry() {
@@ -189,6 +195,23 @@ public class DrillbitContext implements AutoCloseable {
public RemoteFunctionRegistry getRemoteFunctionRegistry() { return functionRegistry.getRemoteFunctionRegistry(); }
+ /**
+ * Use the operator table built during startup when "exec.udf.use_dynamic" option
+ * is set to false.
+ * This operator table has standard SQL functions, operators and drill
+ * built-in user defined functions (UDFs).
+ * It does not include dynamic user defined functions (UDFs) that get added/removed
+ * at run time.
+ * This operator table is meant to be used for high throughput,
+ * low latency operational queries, for which cost of building operator table is
+ * high, both in terms of CPU and heap memory usage.
+ *
+ * @return - Operator table
+ */
+ public DrillOperatorTable getOperatorTable() {
+ return table;
+ }
+
@Override
public void close() throws Exception {
getOptionManager().close();
http://git-wip-us.apache.org/repos/asf/drill/blob/2b5a6f0b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 425131c..4a846c0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -165,7 +165,8 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR,
ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED_OPTION,
ExecConstants.ENABLE_QUERY_PROFILE_VALIDATOR,
- ExecConstants.QUERY_PROFILE_DEBUG_VALIDATOR
+ ExecConstants.QUERY_PROFILE_DEBUG_VALIDATOR,
+ ExecConstants.USE_DYNAMIC_UDFS
};
final Map<String, OptionValidator> tmp = new HashMap<>();
for (final OptionValidator validator : validators) {