You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by su...@apache.org on 2016/12/13 00:40:04 UTC
[07/14] drill git commit: DRILL-5085: Update description for dynamic
UDFs directories in drill-env.sh and drill-module.conf
DRILL-5085: Update description for dynamic UDFs directories in drill-env.sh and drill-module.conf
closes #672
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3cd2f445
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3cd2f445
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3cd2f445
Branch: refs/heads/master
Commit: 3cd2f4453216436d3f742773c4a4c78a5984c363
Parents: a6a0f2f
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Wed Nov 30 15:30:35 2016 +0000
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Mon Dec 12 15:40:02 2016 -0800
----------------------------------------------------------------------
distribution/src/resources/drill-env.sh | 7 ++++-
.../src/resources/drill-override-example.conf | 16 ++++++++++-
.../org/apache/drill/exec/ExecConstants.java | 2 +-
.../expr/fn/FunctionImplementationRegistry.java | 15 ++++++-----
.../fn/registry/RemoteFunctionRegistry.java | 13 ++++-----
.../src/main/resources/drill-module.conf | 28 ++++++++++----------
.../org/apache/drill/TestDynamicUDFSupport.java | 1 +
7 files changed, 52 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/distribution/src/resources/drill-env.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/resources/drill-env.sh
index cd11b47..62dffe6 100644
--- a/distribution/src/resources/drill-env.sh
+++ b/distribution/src/resources/drill-env.sh
@@ -141,4 +141,9 @@
# Arguments passed to sqlline (the Drill shell) at all times: whether
# Drill is embedded in Sqlline or not.
-#export DRILL_SHELL_JAVA_OPTS="..."
\ No newline at end of file
+#export DRILL_SHELL_JAVA_OPTS="..."
+
+# Location Drill should use for temporary files, such as downloaded dynamic UDFs jars.
+# Set to "/tmp" by default.
+#
+# export DRILL_TMP_DIR="..."
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/distribution/src/resources/drill-override-example.conf
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-override-example.conf b/distribution/src/resources/drill-override-example.conf
index 4be4aa2..f9d27b3 100644
--- a/distribution/src/resources/drill-override-example.conf
+++ b/distribution/src/resources/drill-override-example.conf
@@ -170,7 +170,21 @@ drill.exec: {
threadpool_size: 8,
decode_threadpool_size: 1
},
- debug.error_on_leak: true
+ debug.error_on_leak: true,
+ # Settings for Dynamic UDFs.
+ # See https://gist.github.com/arina-ielchiieva/a1c4cfa3890145c5ecb1b70a39cbff55#file-dynamicudfssupport-md.
+ udf: {
+ # number of retry attempts to update remote function registry
+ # if registry version was changed during update
+ retry-attempts: 10,
+ directory: {
+ # Override this property if custom file system should be used to create remote directories
+ # instead of default taken from Hadoop configuration
+ fs: "hdfs:///",
+ # Set this property if custom absolute root should be used for remote directories
+ root: "/app/drill"
+ }
+ }
}
# Below SSL parameters need to be set for custom transport layer settings.
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index bb451c9..0eb9c58 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -116,9 +116,9 @@ public interface ExecConstants {
* Configuration properties connected with dynamic UDFs support
*/
String UDF_RETRY_ATTEMPTS = "drill.exec.udf.retry-attempts";
+ String UDF_DIRECTORY_LOCAL = "drill.exec.udf.directory.local";
String UDF_DIRECTORY_FS = "drill.exec.udf.directory.fs";
String UDF_DIRECTORY_ROOT = "drill.exec.udf.directory.root";
- String UDF_DIRECTORY_LOCAL = "drill.exec.udf.directory.local";
String UDF_DIRECTORY_STAGING = "drill.exec.udf.directory.staging";
String UDF_DIRECTORY_REGISTRY = "drill.exec.udf.directory.registry";
String UDF_DIRECTORY_TMP = "drill.exec.udf.directory.tmp";
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index d0aa33f..590a99a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -377,7 +377,6 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
* Creates local udf directory, if it doesn't exist.
* Checks if local udf directory is a directory and if current application has write rights on it.
* Attempts to clean up local udf directory in case jars were left after previous drillbit run.
- * Local udf directory path is concatenated from drill temporary directory and ${drill.exec.udf.directory.local}.
*
* @param config drill config
* @return path to local udf directory
@@ -395,22 +394,24 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
} catch (IOException e) {
throw new DrillRuntimeException("Error during local udf directory clean up", e);
}
+ logger.info("Created and validated local udf directory [{}]", udfPath);
return new Path(udfDir.toURI());
}
/**
- * First tries to get drill temporary directory value from environmental variable $DRILL_TMP_DIR,
- * then from config ${drill.tmp-dir}.
+ * First tries to get drill temporary directory value from from config ${drill.tmp-dir},
+ * then checks environmental variable $DRILL_TMP_DIR.
* If value is still missing, generates directory using {@link Files#createTempDir()}.
* If temporary directory was generated, sets {@link #deleteTmpDir} to true
* to delete directory on drillbit exit.
* @return drill temporary directory path
*/
private File getTmpDir(DrillConfig config) {
- String drillTempDir = System.getenv("DRILL_TMP_DIR");
-
- if (drillTempDir == null && config.hasPath(ExecConstants.DRILL_TMP_DIR)) {
+ String drillTempDir;
+ if (config.hasPath(ExecConstants.DRILL_TMP_DIR)) {
drillTempDir = config.getString(ExecConstants.DRILL_TMP_DIR);
+ } else {
+ drillTempDir = System.getenv("DRILL_TMP_DIR");
}
if (drillTempDir == null) {
@@ -477,7 +478,7 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
* Will unregister all functions associated with the jar name
* and delete binary and source associated with the jar from local udf directory
*/
- public class UnregistrationListener implements TransientStoreListener {
+ private class UnregistrationListener implements TransientStoreListener {
@Override
public void onChange(TransientStoreEvent event) {
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
index 4ce4a19..e5e62eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java
@@ -81,8 +81,7 @@ import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;
*/
public class RemoteFunctionRegistry implements AutoCloseable {
- public static final String REGISTRY = "registry";
-
+ private static final String registry_path = "registry";
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemoteFunctionRegistry.class);
private static final ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT);
@@ -108,15 +107,15 @@ public class RemoteFunctionRegistry implements AutoCloseable {
}
public Registry getRegistry() {
- return registry.get(REGISTRY, null);
+ return registry.get(registry_path, null);
}
public Registry getRegistry(DataChangeVersion version) {
- return registry.get(REGISTRY, version);
+ return registry.get(registry_path, version);
}
public void updateRegistry(Registry registryContent, DataChangeVersion version) throws VersionMismatchException {
- registry.put(REGISTRY, registryContent, version);
+ registry.put(registry_path, registryContent, version);
}
public void submitForUnregistration(String jar) {
@@ -168,7 +167,7 @@ public class RemoteFunctionRegistry implements AutoCloseable {
.persist()
.build();
registry = storeProvider.getOrCreateStore(registrationConfig);
- registry.putIfAbsent(REGISTRY, Registry.getDefaultInstance());
+ registry.putIfAbsent(registry_path, Registry.getDefaultInstance());
} catch (StoreException e) {
throw new DrillRuntimeException("Failure while loading remote registry.", e);
}
@@ -189,6 +188,7 @@ public class RemoteFunctionRegistry implements AutoCloseable {
* if not set, uses user home directory instead.
*/
private void prepareAreas(DrillConfig config) {
+ logger.info("Preparing three remote udf areas: staging, registry and tmp.");
Configuration conf = new Configuration();
if (config.hasPath(ExecConstants.UDF_DIRECTORY_FS)) {
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.getString(ExecConstants.UDF_DIRECTORY_FS));
@@ -245,6 +245,7 @@ public class RemoteFunctionRegistry implements AutoCloseable {
} catch (Exception e) {
DrillRuntimeException.format(e, "Error during udf area creation [%s] on file system [%s]", fullPath, fs.getUri());
}
+ logger.info("Created remote udf area [{}] on file system [{}]", fullPath, fs.getUri());
return path;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index deb31b3..ab73cec 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -45,11 +45,10 @@ drill.client: {
supports-complex-types: true
}
-// Directory is used as base for temporary storage of Dynamic UDF jars.
-// Set this property if you want to have custom temporary directory, instead of generated at runtime.
-// By default ${DRILL_TMP_DIR} is used if set.
-// drill.tmp-dir: "/tmp"
-// drill.tmp-dir: ${?DRILL_TMP_DIR}
+// Location Drill uses for temporary files, such as downloaded dynamic UDFs jars.
+// By default ${DRILL_TMP_DIR} is used if set or ${drill.tmp-dir} if it's been overridden.
+drill.tmp-dir: "/tmp"
+drill.tmp-dir: ${?DRILL_TMP_DIR}
drill.exec: {
cluster-id: "drillbits1"
@@ -202,24 +201,25 @@ drill.exec: {
udf: {
retry-attempts: 5,
directory: {
+ // Base directory for remote and local udf directories, unique among clusters.
+ base: ${drill.exec.zk.root}"/udf",
+
+ // Path to local udf directory, always created on local file system.
+ // Root for these directory is generated at runtime unless Drill temporary directory is set.
+ local: ${drill.exec.udf.directory.base}"/udf/local",
+
// Set this property if custom file system should be used to create remote directories, ex: fs: "file:///".
// fs: "",
// Set this property if custom absolute root should be used for remote directories, ex: root: "/app/drill".
// root: "",
- // Relative base directory for all udf directories (local and remote).
- base: ${drill.exec.zk.root}"/udf",
-
- // Relative path to local udf directory.
- // Generated temporary directory is used as root unless ${DRILL_TMP_DIR} or ${drill.tmp-dir} is set.
- local: ${drill.exec.udf.directory.base}"/local"
-
// Relative path to all remote udf directories.
- // Directories are created under default file system unless ${drill.exec.udf.directory.fs} is set.
+ // Directories are created under default file system taken from Hadoop configuration
+ // unless ${drill.exec.udf.directory.fs} is set.
// User home directory is used as root unless ${drill.exec.udf.directory.root} is set.
staging: ${drill.exec.udf.directory.base}"/staging",
registry: ${drill.exec.udf.directory.base}"/registry",
- tmp: ${drill.exec.udf.directory.base}"/tmp",
+ tmp: ${drill.exec.udf.directory.base}"/tmp"
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3cd2f445/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
index 4676e39..78cdbe2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
@@ -78,6 +78,7 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
public void setup() {
Properties overrideProps = new Properties();
overrideProps.setProperty("drill.exec.udf.directory.root", base.getRoot().getPath());
+ overrideProps.setProperty("drill.tmp-dir", base.getRoot().getPath());
updateTestCluster(1, DrillConfig.create(overrideProps));
}