You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ku...@apache.org on 2018/10/11 17:30:25 UTC
[drill] branch master updated: DRILL-3988: Expose Drill built-in
functions & UDFs in a system table (#1483)
This is an automated email from the ASF dual-hosted git repository.
kunal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 734c460 DRILL-3988: Expose Drill built-in functions & UDFs in a system table (#1483)
734c460 is described below
commit 734c46076b5df42be93aece1a363438d5c2a3e95
Author: Kunal Khatua <kk...@users.noreply.github.com>
AuthorDate: Thu Oct 11 10:30:15 2018 -0700
DRILL-3988: Expose Drill built-in functions & UDFs in a system table (#1483)
This commit exposes available SQL functions in Drill and also detects UDFs that have been dynamically loaded into Drill.
An example is shown below for 2 UDFs dynamically loaded into the cluster, along side the existing built-in functions that come with Drill.
```
0: jdbc:drill:schema=sys> select source, count(*) as functionCount from sys.functions group by source;
+-----------------------------------------+----------------+
| source | functionCount |
+-----------------------------------------+----------------+
| built-in | 2704 |
| simple-drill-function-1.0-SNAPSHOT.jar | 12 |
| drill-url-tools-1.0.jar | 1 |
+-----------------------------------------+----------------+
3 rows selected (0.209 seconds)
```
The system table exposes information as shown. The UDF is initialized, making the `returnType` available.
The `random(FLOAT8-REQUIRED,FLOAT8-REQUIRED)` function is an example of a UDF that has overloaded arguments (see `signature`).
The `url_parse(VARCHAR-REQUIRED)` function is another example of an initialized UDF.
Rest are built-in functions that meet the query's filter criteria.
```
0: jdbc:drill:schema=sys> select * from sys.functions where name like 'random' or name like '%url%';
+-------------+----------------------------------+-------------+-----------------------------------------+
| name | signature | returnType | source |
+-------------+----------------------------------+-------------+-----------------------------------------+
| parse_url | VARCHAR-REQUIRED | LATE | built-in |
| random | | FLOAT8 | built-in |
| random | FLOAT8-REQUIRED,FLOAT8-REQUIRED | FLOAT8 | simple-drill-function-1.0-SNAPSHOT.jar |
| url_decode | VARCHAR-REQUIRED | VARCHAR | built-in |
| url_encode | VARCHAR-REQUIRED | VARCHAR | built-in |
| url_parse | VARCHAR-REQUIRED | LATE | drill-url-tools-1.0.jar |
+-------------+----------------------------------+-------------+-----------------------------------------+
6 rows selected (0.619 seconds)
```
---
.../expr/fn/FunctionImplementationRegistry.java | 13 +++
.../expr/fn/registry/FunctionRegistryHolder.java | 34 ++++++
.../expr/fn/registry/LocalFunctionRegistry.java | 8 ++
.../apache/drill/exec/ops/FragmentContextImpl.java | 14 +++
.../apache/drill/exec/server/DrillbitContext.java | 4 +-
.../drill/exec/store/sys/FunctionsIterator.java | 128 +++++++++++++++++++++
.../apache/drill/exec/store/sys/SystemTable.java | 9 +-
.../fn/registry/FunctionRegistryHolderTest.java | 121 +++++++++++++++----
.../drill/exec/store/sys/TestSystemTable.java | 5 +
.../exec/work/metadata/TestMetadataProvider.java | 74 ++++++------
10 files changed, 342 insertions(+), 68 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index f4b8373..a6f9a7f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -28,6 +28,7 @@ import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -54,6 +55,7 @@ import org.apache.drill.exec.coord.store.TransientStoreListener;
import org.apache.drill.exec.exception.FunctionValidationException;
import org.apache.drill.exec.exception.JarValidationException;
import org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry;
+import org.apache.drill.exec.expr.fn.registry.FunctionHolder;
import org.apache.drill.exec.expr.fn.registry.JarScan;
import org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry;
import org.apache.drill.exec.planner.sql.DrillOperatorTable;
@@ -484,6 +486,17 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
}
/**
+ * Retrieve all functions, mapped by source jars (after syncing)
+ * @return Map of source jars and their functionHolders
+ */
+ public Map<String, List<FunctionHolder>> getAllJarsWithFunctionsHolders() {
+ if (useDynamicUdfs) {
+ syncWithRemoteRegistry(localFunctionRegistry.getVersion());
+ }
+ return localFunctionRegistry.getAllJarsWithFunctionsHolders();
+ }
+
+ /**
* Creates local udf directory, if it doesn't exist.
* Checks if local udf directory is a directory and if current application has write rights on it.
* Attempts to clean up local udf directory in case jars were left after previous drillbit run.
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
index d1d4fc9..d0383e9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolder.java
@@ -25,6 +25,8 @@ import org.apache.drill.common.concurrent.AutoCloseableLock;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
@@ -162,6 +164,38 @@ public class FunctionRegistryHolder {
}
/**
+ * Retrieves all functions (holders) associated with all the jars
+ * This is read operation, so several users can perform this operation at the same time.
+ * @return list of all functions, mapped by their sources
+ */
+ public Map<String, List<FunctionHolder>> getAllJarsWithFunctionHolders() {
+ Map<String, List<FunctionHolder>> allFunctionHoldersByJar = new HashMap<>();
+
+ try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
+ for (String jarName : jars.keySet()) {
+ //Capture functionHolders here
+ List<FunctionHolder> drillFuncHolderList = new LinkedList<>();
+
+ Map<String, Queue<String>> functionsInJar = jars.get(jarName);
+ for (Map.Entry<String, Queue<String>> functionEntry : functionsInJar.entrySet()) {
+ String fnName = functionEntry.getKey();
+ Queue<String> fnSignatureList = functionEntry.getValue();
+ //Get all FunctionHolders (irrespective of source)
+ Map<String, DrillFuncHolder> functionHolders = functions.get(fnName);
+ //Iterate for matching entries and populate new Map
+ for (Map.Entry<String, DrillFuncHolder> entry : functionHolders.entrySet()) {
+ if (fnSignatureList.contains(entry.getKey())) {
+ drillFuncHolderList.add(new FunctionHolder(fnName, entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+ allFunctionHoldersByJar.put(jarName, drillFuncHolderList);
+ }
+ }
+ return allFunctionHoldersByJar;
+ }
+
+ /**
* Retrieves all function names associated with the jar from {@link #jars}.
* Returns empty list if jar is not registered.
* This is read operation, so several users can perform this operation at the same time.
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/LocalFunctionRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/LocalFunctionRegistry.java
index cefbd8c..f96b1fb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/LocalFunctionRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/LocalFunctionRegistry.java
@@ -236,6 +236,14 @@ public class LocalFunctionRegistry {
}
/**
+ * Returns a map of all function holders mapped by source jars
+ * @return all functions organized by source jars
+ */
+ public Map<String, List<FunctionHolder>> getAllJarsWithFunctionsHolders() {
+ return registryHolder.getAllJarsWithFunctionHolders();
+ }
+
+ /**
* Registers all functions present in {@link DrillOperatorTable},
* also sets sync registry version used at the moment of function registration.
*
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
index fcfdc8c..6e40466 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
@@ -245,6 +245,8 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
Preconditions.checkArgument(this.buffers == null, "Can only set buffers once.");
this.buffers = buffers;
}
+
+ @Override
public QueryProfileStoreContext getProfileStoreContext() {
return context.getProfileStoreContext();
}
@@ -254,6 +256,7 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
return context.getUserConnections();
}
+ @Override
public void setExecutorState(final ExecutorState executorState) {
Preconditions.checkArgument(this.executorState == null, "ExecutorState can only be set once.");
this.executorState = executorState;
@@ -263,6 +266,7 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
executorState.fail(cause);
}
+ @Override
public SchemaPlus getFullRootSchema() {
if (queryContext == null) {
fail(new UnsupportedOperationException("Schema tree can only be created in root fragment. " +
@@ -284,6 +288,7 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
return queryContext.getFullRootSchema(schemaConfig);
}
+ @Override
public FragmentStats getStats() {
return stats;
}
@@ -332,10 +337,12 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
* The FragmentHandle for this Fragment
* @return FragmentHandle
*/
+ @Override
public FragmentHandle getHandle() {
return fragment.getHandle();
}
+ @Override
public String getFragIdString() {
final FragmentHandle handle = getHandle();
final String frag = handle != null ? handle.getMajorFragmentId() + ":" + handle.getMinorFragmentId() : "0:0";
@@ -367,6 +374,7 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
* Get this fragment's allocator.
* @return the allocator
*/
+ @Override
@Deprecated
public BufferAllocator getAllocator() {
if (allocator == null) {
@@ -417,10 +425,12 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
return tunnel;
}
+ @Override
public IncomingBuffers getBuffers() {
return buffers;
}
+ @Override
public OperatorContext newOperatorContext(PhysicalOperator popConfig, OperatorStats stats)
throws OutOfMemoryException {
OperatorContextImpl context = new OperatorContextImpl(popConfig, this, stats);
@@ -428,6 +438,7 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
return context;
}
+ @Override
public OperatorContext newOperatorContext(PhysicalOperator popConfig)
throws OutOfMemoryException {
OperatorContextImpl context = new OperatorContextImpl(popConfig, this);
@@ -450,10 +461,12 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
return executionControls;
}
+ @Override
public String getQueryUserName() {
return fragment.getCredentials().getUserName();
}
+ @Override
public boolean isImpersonationEnabled() {
// TODO(DRILL-2097): Until SimpleRootExec tests are removed, we need to consider impersonation disabled if there is
// no config
@@ -513,6 +526,7 @@ public class FragmentContextImpl extends BaseFragmentContext implements Executor
return valueHolder;
}
+ @Override
public ExecutorService getExecutor(){
return context.getExecutor();
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index b047f7d..b2802de 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -261,7 +261,9 @@ public class DrillbitContext implements AutoCloseable {
return classpathScan;
}
- public RemoteFunctionRegistry getRemoteFunctionRegistry() { return functionRegistry.getRemoteFunctionRegistry(); }
+ public RemoteFunctionRegistry getRemoteFunctionRegistry() {
+ return functionRegistry.getRemoteFunctionRegistry();
+ }
/**
* Use the operator table built during startup when "exec.udf.use_dynamic" option
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
new file mode 100644
index 0000000..86e8817
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/FunctionsIterator.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.exec.expr.fn.DrillFuncHolder;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.fn.FunctionLookupContext;
+import org.apache.drill.exec.expr.fn.registry.FunctionHolder;
+import org.apache.drill.exec.ops.ExecutorFragmentContext;
+import org.apache.drill.exec.store.pojo.NonNullable;
+
+/**
+ * List functions as a System Table
+ */
+public class FunctionsIterator implements Iterator<Object> {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionsIterator.class);
+
+ private final Iterator<FunctionInfo> sortedIterator;
+
+ public FunctionsIterator(ExecutorFragmentContext context) {
+ Map<String, FunctionInfo> functionMap = new HashMap<String, FunctionInfo>();
+ //Access Registry for function list
+ FunctionLookupContext functionLookupContext = context.getFunctionRegistry();
+ //Check true instance type
+ if (functionLookupContext instanceof FunctionImplementationRegistry) {
+ @SuppressWarnings("resource")
+ FunctionImplementationRegistry functionImplRegistry = (FunctionImplementationRegistry) functionLookupContext;
+ Map<String, List<FunctionHolder>> jarFunctionListMap = functionImplRegistry.getAllJarsWithFunctionsHolders();
+
+ for (String jarName : jarFunctionListMap.keySet()) {
+ for (FunctionHolder dfhEntry : jarFunctionListMap.get(jarName)) {
+ populateFunctionMap(functionMap, jarName, dfhEntry.getHolder());
+ }
+ }
+
+ List<FunctionInfo> functionList = new ArrayList<FunctionsIterator.FunctionInfo>(functionMap.values());
+ functionList.sort((FunctionInfo o1, FunctionInfo o2) -> {
+ int result = o1.name.compareTo(o2.name);
+ if (result == 0) {
+ result = o1.signature.compareTo(o2.signature);
+ }
+ if (result == 0) {
+ return o1.returnType.compareTo(o2.returnType);
+ }
+ return result;
+ });
+
+ sortedIterator = functionList.iterator();
+ } else {
+ //Logging error
+ logger.error("Function Registry was found to be of {} instead of {}. No functions could be loaded ",
+ FunctionImplementationRegistry.class, functionLookupContext.getClass());
+ sortedIterator = Collections.emptyIterator();
+ }
+
+ }
+
+ /**
+ * Populate the map of functionInfo based on the functionSignatureKey for a given Jar-DrillFunctionHolder
+ * @param functionMap map to populate
+ * @param jarName name of the source jar
+ * @param dfh functionHolder that carries all the registered names and the signature
+ */
+ private void populateFunctionMap(Map<String, FunctionInfo> functionMap, String jarName, DrillFuncHolder dfh) {
+ String registeredNames[] = dfh.getRegisteredNames();
+ String signature = dfh.getInputParameters();
+ String returnType = dfh.getReturnType().getMinorType().toString();
+ for (String name : registeredNames) {
+ //Generate a unique key for a function holder as 'functionName#functionSignature'
+ //Bumping capacity from default 16 to 64 (since key is expected to be bigger, and reduce probability of resizing)
+ String funcSignatureKey = new StringBuilder(64).append(name).append('#').append(signature).toString();
+ functionMap.put(funcSignatureKey, new FunctionInfo(name, signature, returnType, jarName));
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return sortedIterator.hasNext();
+ }
+
+ @Override
+ public FunctionInfo next() {
+ return sortedIterator.next();
+ }
+
+ /**
+ * Representation of an entry in the System table - Functions
+ */
+ public static class FunctionInfo {
+ @NonNullable
+ public final String name;
+ @NonNullable
+ public final String signature;
+ @NonNullable
+ public final String returnType;
+ @NonNullable
+ public final String source;
+
+ public FunctionInfo(String funcName, String funcSignature, String funcReturnType, String jarName) {
+ this.name = funcName;
+ this.signature = funcSignature;
+ this.returnType = funcReturnType;
+ this.source = jarName;
+ }
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
index 62e610d..fae4352 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
@@ -112,9 +112,16 @@ public enum SystemTable {
THREADS("threads", true, ThreadsIterator.ThreadsInfo.class) {
@Override
- public Iterator<Object> getIterator(final ExecutorFragmentContext context, final int maxRecords) {
+ public Iterator<Object> getIterator(final ExecutorFragmentContext context, final int maxRecords) {
return new ThreadsIterator(context);
}
+ },
+
+ FUNCTIONS("functions", false, FunctionsIterator.FunctionInfo.class) {
+ @Override
+ public Iterator<Object> getIterator(final ExecutorFragmentContext context, final int maxRecords) {
+ return new FunctionsIterator(context);
+ }
};
private final String tableName;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolderTest.java
index 9782315..3e8e6b8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/registry/FunctionRegistryHolderTest.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.expr.fn.registry;
import org.apache.drill.shaded.guava.com.google.common.collect.ArrayListMultimap;
import org.apache.drill.shaded.guava.com.google.common.collect.ListMultimap;
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.junit.Before;
@@ -28,10 +27,14 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -46,6 +49,8 @@ public class FunctionRegistryHolderTest {
private static final String built_in = "built-in";
private static final String udf_jar = "DrillUDF-1.0.jar";
+ private static final String LOWER_FUNC_NAME = "lower";
+ private static final String SHUFFLE_FUNC_NAME = "shuffle";
private static Map<String, List<FunctionHolder>> newJars;
private FunctionRegistryHolder registryHolder;
@@ -53,12 +58,14 @@ public class FunctionRegistryHolderTest {
@BeforeClass
public static void init() {
newJars = new HashMap<>();
- FunctionHolder lower = new FunctionHolder("lower", "lower(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
+ FunctionHolder lower = new FunctionHolder(LOWER_FUNC_NAME, "lower(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
FunctionHolder upper = new FunctionHolder("upper", "upper(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
- newJars.put(built_in, Lists.newArrayList(lower, upper));
+ FunctionHolder shuffle = new FunctionHolder(SHUFFLE_FUNC_NAME, "shuffle()", mock(DrillFuncHolder.class));
+ newJars.put(built_in, new ArrayList<>(Arrays.asList(lower, upper, shuffle)));
FunctionHolder custom_lower = new FunctionHolder("custom_lower", "lower(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
FunctionHolder custom_upper = new FunctionHolder("custom_upper", "custom_upper(VARCHAR-REQUIRED)", mock(DrillFuncHolder.class));
- newJars.put(udf_jar, Lists.newArrayList(custom_lower, custom_upper));
+ FunctionHolder overloaded_shuffle = new FunctionHolder(SHUFFLE_FUNC_NAME, "shuffle(FLOAT8-REQUIRED,FLOAT8-OPTIONAL)", mock(DrillFuncHolder.class));
+ newJars.put(udf_jar, new ArrayList<>(Arrays.asList(custom_lower, custom_upper, overloaded_shuffle)));
}
@Before
@@ -87,16 +94,16 @@ public class FunctionRegistryHolderTest {
@Test
public void testAddJars() {
resetRegistry();
- int functionsSize = 0;
List<String> jars = new ArrayList<>();
ListMultimap<String, DrillFuncHolder> functionsWithHolders = ArrayListMultimap.create();
ListMultimap<String, String> functionsWithSignatures = ArrayListMultimap.create();
+ Set<String> functionsSet = new HashSet<>();
for (Map.Entry<String, List<FunctionHolder>> jar : newJars.entrySet()) {
jars.add(jar.getKey());
for (FunctionHolder functionHolder : jar.getValue()) {
functionsWithHolders.put(functionHolder.getName(), functionHolder.getHolder());
functionsWithSignatures.put(functionHolder.getName(), functionHolder.getSignature());
- functionsSize++;
+ functionsSet.add(functionHolder.getName()); //Track unique function names
}
}
@@ -104,7 +111,7 @@ public class FunctionRegistryHolderTest {
registryHolder.addJars(newJars, ++expectedVersion);
assertEquals("Version number should match", expectedVersion, registryHolder.getVersion());
compareTwoLists(jars, registryHolder.getAllJarNames());
- assertEquals(functionsSize, registryHolder.functionsSize());
+ assertEquals(functionsSet.size(), registryHolder.functionsSize());
compareListMultimaps(functionsWithHolders, registryHolder.getAllFunctionsWithHolders());
compareListMultimaps(functionsWithSignatures, registryHolder.getAllFunctionsWithSignatures());
}
@@ -112,7 +119,7 @@ public class FunctionRegistryHolderTest {
@Test
public void testAddTheSameJars() {
resetRegistry();
- int functionsSize = 0;
+ Set<String> functionsSet = new HashSet<>();
List<String> jars = new ArrayList<>();
ListMultimap<String, DrillFuncHolder> functionsWithHolders = ArrayListMultimap.create();
ListMultimap<String, String> functionsWithSignatures = ArrayListMultimap.create();
@@ -121,14 +128,14 @@ public class FunctionRegistryHolderTest {
for (FunctionHolder functionHolder : jar.getValue()) {
functionsWithHolders.put(functionHolder.getName(), functionHolder.getHolder());
functionsWithSignatures.put(functionHolder.getName(), functionHolder.getSignature());
- functionsSize++;
+ functionsSet.add(functionHolder.getName()); //Track unique function names
}
}
int expectedVersion = 0;
registryHolder.addJars(newJars, ++expectedVersion);
assertEquals("Version number should match", expectedVersion, registryHolder.getVersion());
compareTwoLists(jars, registryHolder.getAllJarNames());
- assertEquals(functionsSize, registryHolder.functionsSize());
+ assertEquals(functionsSet.size(), registryHolder.functionsSize());
compareListMultimaps(functionsWithHolders, registryHolder.getAllFunctionsWithHolders());
compareListMultimaps(functionsWithSignatures, registryHolder.getAllFunctionsWithSignatures());
@@ -136,7 +143,7 @@ public class FunctionRegistryHolderTest {
registryHolder.addJars(newJars, ++expectedVersion);
assertEquals("Version number should match", expectedVersion, registryHolder.getVersion());
compareTwoLists(jars, registryHolder.getAllJarNames());
- assertEquals(functionsSize, registryHolder.functionsSize());
+ assertEquals(functionsSet.size(), registryHolder.functionsSize());
compareListMultimaps(functionsWithHolders, registryHolder.getAllFunctionsWithHolders());
compareListMultimaps(functionsWithSignatures, registryHolder.getAllFunctionsWithSignatures());
}
@@ -156,6 +163,47 @@ public class FunctionRegistryHolderTest {
}
@Test
+ public void testGetAllJarsWithFunctionHolders() {
+ Map<String, List<FunctionHolder>> fnHoldersInRegistry = registryHolder.getAllJarsWithFunctionHolders();
+ //Iterate and confirm lists are same
+ for (String jarName : newJars.keySet()) {
+ List<DrillFuncHolder> expectedHolderList = newJars.get(jarName).stream()
+ .map(FunctionHolder::getHolder) //Extract DrillFuncHolder
+ .collect(Collectors.toList());
+ List<DrillFuncHolder> testHolderList = fnHoldersInRegistry.get(jarName).stream()
+ .map(FunctionHolder::getHolder) //Extract DrillFuncHolder
+ .collect(Collectors.toList());
+
+ compareTwoLists(expectedHolderList, testHolderList);
+ }
+
+ Map<String, String> shuffleFunctionMap = new HashMap<>();
+ // Confirm that same function spans multiple jars with different signatures
+ //Init: Expected Map of items
+ for (String jarName : newJars.keySet()) {
+ for (FunctionHolder funcHolder : newJars.get(jarName)) {
+ if (SHUFFLE_FUNC_NAME.equals(funcHolder.getName())) {
+ shuffleFunctionMap.put(funcHolder.getSignature(), jarName);
+ }
+ }
+ }
+
+ //Test: Remove items from ExpectedMap based on match from testJar's functionHolder items
+ for (String testJar : registryHolder.getAllJarNames()) {
+ for (FunctionHolder funcHolder : fnHoldersInRegistry.get(testJar)) {
+ if (SHUFFLE_FUNC_NAME.equals(funcHolder.getName())) {
+ String testSignature = funcHolder.getSignature();
+ String expectedJar = shuffleFunctionMap.get(testSignature);
+ if (testJar.equals(expectedJar)) {
+ shuffleFunctionMap.remove(testSignature);
+ }
+ }
+ }
+ }
+ assertTrue(shuffleFunctionMap.isEmpty());
+ }
+
+ @Test
public void testGetFunctionNamesByJar() {
List<String> expectedResult = newJars.get(built_in).stream()
.map(FunctionHolder::getName)
@@ -202,26 +250,38 @@ public class FunctionRegistryHolderTest {
public void testGetHoldersByFunctionNameWithVersion() {
List<DrillFuncHolder> expectedResult = newJars.values().stream()
.flatMap(Collection::stream)
- .filter(f -> "lower".equals(f.getName()))
+ .filter(f -> LOWER_FUNC_NAME.equals(f.getName()))
.map(FunctionHolder::getHolder)
.collect(Collectors.toList());
assertFalse(expectedResult.isEmpty());
AtomicInteger version = new AtomicInteger();
- compareTwoLists(expectedResult, registryHolder.getHoldersByFunctionName("lower", version));
+ compareTwoLists(expectedResult, registryHolder.getHoldersByFunctionName(LOWER_FUNC_NAME, version));
assertEquals("Version number should match", version.get(), registryHolder.getVersion());
}
@Test
public void testGetHoldersByFunctionName() {
- List<DrillFuncHolder> expectedResult = newJars.values().stream()
- .flatMap(Collection::stream)
- .filter(f -> "lower".equals(f.getName()))
- .map(FunctionHolder::getHolder)
- .collect(Collectors.toList());
+ List<DrillFuncHolder> expectedUniqueResult = new ArrayList<>();
+ List<DrillFuncHolder> expectedMultipleResult = new ArrayList<>();
+ for (List<FunctionHolder> functionHolders : newJars.values()) {
+ for (FunctionHolder functionHolder : functionHolders) {
+ if (LOWER_FUNC_NAME.equals(functionHolder.getName())) {
+ expectedUniqueResult.add(functionHolder.getHolder());
+ } else
+ if (SHUFFLE_FUNC_NAME.equals(functionHolder.getName())) {
+ expectedMultipleResult.add(functionHolder.getHolder());
+ }
+ }
+ }
- assertFalse(expectedResult.isEmpty());
- compareTwoLists(expectedResult, registryHolder.getHoldersByFunctionName("lower"));
+ //Test for function with one signature
+ assertFalse(expectedUniqueResult.isEmpty());
+ compareTwoLists(expectedUniqueResult, registryHolder.getHoldersByFunctionName(LOWER_FUNC_NAME));
+
+ //Test for function with multiple signatures
+ assertFalse(expectedMultipleResult.isEmpty());
+ compareTwoLists(expectedMultipleResult, registryHolder.getHoldersByFunctionName(SHUFFLE_FUNC_NAME));
}
@Test
@@ -232,10 +292,23 @@ public class FunctionRegistryHolderTest {
@Test
public void testFunctionsSize() {
- int count = newJars.values().stream()
- .mapToInt(List::size)
- .sum();
- assertEquals("Functions size should match", count, registryHolder.functionsSize());
+ int fnCountInRegistryHolder = 0;
+ int fnCountInNewJars = 0;
+
+ Set<String> functionNameSet = new HashSet<>();
+ for (List<FunctionHolder> functionHolders : newJars.values()) {
+ for (FunctionHolder functionHolder : functionHolders) {
+ functionNameSet.add(functionHolder.getName()); //Track unique function names
+ fnCountInNewJars++; //Track all functions
+ }
+ }
+ assertEquals("Unique function name count should match", functionNameSet.size(), registryHolder.functionsSize());
+
+ for (String jarName : registryHolder.getAllJarNames()) {
+ fnCountInRegistryHolder += registryHolder.getFunctionNamesByJar(jarName).size();
+ }
+
+ assertEquals("Function count should match", fnCountInNewJars, fnCountInRegistryHolder);
}
@Test
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
index 4f47287..5c9990a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
@@ -77,6 +77,11 @@ public class TestSystemTable extends PlanTestBase {
}
@Test
+ public void functionsTable() throws Exception {
+ test("select * from sys.functions");
+ }
+
+ @Test
public void profilesTable() throws Exception {
test("select * from sys.profiles");
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
index de55f19..ce58cb0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
@@ -149,7 +149,7 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(18, tables.size());
+ assertEquals(19, tables.size());
verifyTable("information_schema", "CATALOGS", tables);
verifyTable("information_schema", "COLUMNS", tables);
@@ -157,15 +157,10 @@ public class TestMetadataProvider extends BaseTestQuery {
verifyTable("information_schema", "TABLES", tables);
verifyTable("information_schema", "VIEWS", tables);
verifyTable("information_schema", "FILES", tables);
- verifyTable("sys", "boot", tables);
- verifyTable("sys", "drillbits", tables);
- verifyTable("sys", "memory", tables);
- verifyTable("sys", SystemTable.OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.OPTIONS.getTableName(), tables);
- verifyTable("sys", "threads", tables);
- verifyTable("sys", "version", tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS.getTableName(), tables);
+ //Verify System Tables
+ for (SystemTable sysTbl : SystemTable.values()) {
+ verifyTable("sys", sysTbl.getTableName(), tables);
+ }
}
@Test
@@ -187,7 +182,7 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(18, tables.size());
+ assertEquals(19, tables.size());
verifyTable("information_schema", "CATALOGS", tables);
verifyTable("information_schema", "COLUMNS", tables);
@@ -195,15 +190,10 @@ public class TestMetadataProvider extends BaseTestQuery {
verifyTable("information_schema", "TABLES", tables);
verifyTable("information_schema", "VIEWS", tables);
verifyTable("information_schema", "FILES", tables);
- verifyTable("sys", "boot", tables);
- verifyTable("sys", "drillbits", tables);
- verifyTable("sys", "memory", tables);
- verifyTable("sys", SystemTable.OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.OPTIONS.getTableName(), tables);
- verifyTable("sys", "threads", tables);
- verifyTable("sys", "version", tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS.getTableName(), tables);
+ //Verify System Tables
+ for (SystemTable sysTbl : SystemTable.values()) {
+ verifyTable("sys", sysTbl.getTableName(), tables);
+ }
}
@Test
@@ -216,15 +206,15 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(10, tables.size());
-
- verifyTable("sys", "boot", tables);
- verifyTable("sys", "memory", tables);
- verifyTable("sys", SystemTable.OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.OPTIONS.getTableName(), tables);
- verifyTable("sys", "version", tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS_OLD.getTableName(), tables);
- verifyTable("sys", SystemTable.INTERNAL_OPTIONS.getTableName(), tables);
+ assertEquals(11, tables.size());
+
+ //Verify System Tables
+ for (SystemTable sysTbl : SystemTable.values()) {
+ String sysTblName = sysTbl.getTableName();
+ if (sysTblName.contains("o")) {
+ verifyTable("sys", sysTblName, tables);
+ }
+ }
}
@Test
@@ -250,7 +240,7 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<ColumnMetadata> columns = resp.getColumnsList();
- assertEquals(135, columns.size());
+ assertEquals(139, columns.size());
// too many records to verify the output.
}
@@ -265,11 +255,11 @@ public class TestMetadataProvider extends BaseTestQuery {
List<ColumnMetadata> columns = resp.getColumnsList();
assertEquals(6, columns.size());
- verifyColumn("sys", "drillbits", "user_port", columns);
- verifyColumn("sys", "drillbits", "control_port", columns);
- verifyColumn("sys", "drillbits", "data_port", columns);
- verifyColumn("sys", "memory", "user_port", columns);
- verifyColumn("sys", "threads", "user_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "user_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "control_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "data_port", columns);
+ verifyColumn("sys", SystemTable.MEMORY.getTableName(), "user_port", columns);
+ verifyColumn("sys", SystemTable.THREADS.getTableName(), "user_port", columns);
}
@Test
@@ -285,9 +275,9 @@ public class TestMetadataProvider extends BaseTestQuery {
List<ColumnMetadata> columns = resp.getColumnsList();
assertEquals(4, columns.size());
- verifyColumn("sys", "drillbits", "user_port", columns);
- verifyColumn("sys", "drillbits", "control_port", columns);
- verifyColumn("sys", "drillbits", "data_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "user_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "control_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "data_port", columns);
}
@Test
@@ -306,10 +296,10 @@ public class TestMetadataProvider extends BaseTestQuery {
List<ColumnMetadata> columns = resp.getColumnsList();
assertEquals(4, columns.size());
- verifyColumn("sys", "drillbits", "user_port", columns);
- verifyColumn("sys", "drillbits", "control_port", columns);
- verifyColumn("sys", "drillbits", "data_port", columns);
- verifyColumn("sys", "drillbits", "http_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "user_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "control_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "data_port", columns);
+ verifyColumn("sys", SystemTable.DRILLBITS.getTableName(), "http_port", columns);
}
/** Helper method to verify schema contents */