You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pr...@apache.org on 2017/09/25 16:30:21 UTC
drill git commit: DRILL-5781: Fix unit test failures to use tests
config even if default config is available
Repository: drill
Updated Branches:
refs/heads/master 6cb626d78 -> 9620e3482
DRILL-5781: Fix unit test failures to use tests config even if default config is available
closes #942
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9620e348
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9620e348
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9620e348
Branch: refs/heads/master
Commit: 9620e34829a25d767f90b272b9b57ccc9e0e36ea
Parents: 6cb626d
Author: Volodymyr Vysotskyi <vv...@gmail.com>
Authored: Thu Sep 7 18:01:12 2017 +0000
Committer: Paul Rogers <pr...@maprtech.com>
Committed: Sun Sep 24 22:21:20 2017 -0700
----------------------------------------------------------------------
.../org/apache/drill/hbase/HBaseTestsSuite.java | 5 +-
.../src/test/resources/hbase-site.xml | 4 +-
.../apache/drill/exec/coord/zk/PathUtils.java | 17 ++++++-
.../org/apache/drill/exec/ZookeeperHelper.java | 5 +-
.../apache/drill/exec/ZookeeperTestUtil.java | 44 +++++++++++++++++
.../drill/exec/coord/zk/TestEphemeralStore.java | 5 +-
.../exec/coord/zk/TestZookeeperClient.java | 3 ++
.../exec/server/TestDrillbitResilience.java | 3 ++
exec/java-exec/src/test/resources/core-site.xml | 52 ++++++++++++++++++++
exec/java-exec/src/test/resources/login.conf | 25 ++++++++++
10 files changed, 155 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
index 3dd3608..1a176ca 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.drill.exec.ZookeeperTestUtil;
import org.apache.drill.exec.util.GuavaPatcher;
import org.apache.drill.hbase.test.Drill2130StorageHBaseHamcrestConfigurationTest;
import org.apache.hadoop.conf.Configuration;
@@ -95,6 +96,8 @@ public class HBaseTestsSuite {
@BeforeClass
public static void initCluster() throws Exception {
+ ZookeeperTestUtil.setJaasTestConfigFile();
+
if (initCount.get() == 0) {
synchronized (HBaseTestsSuite.class) {
if (initCount.get() == 0) {
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/contrib/storage-hbase/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/resources/hbase-site.xml b/contrib/storage-hbase/src/test/resources/hbase-site.xml
index 54425d4..f5c0fcb 100644
--- a/contrib/storage-hbase/src/test/resources/hbase-site.xml
+++ b/contrib/storage-hbase/src/test/resources/hbase-site.xml
@@ -66,15 +66,13 @@
Default is 10.
</description>
</property>
-<!--
- <property>
+ <property>
<name>hbase.master.info.port</name>
<value>-1</value>
<description>The port for the hbase master web UI
Set to -1 if you do not want the info server to run.
</description>
</property>
--->
<property>
<name>hbase.regionserver.info.port</name>
<value>-1</value>
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
index f01b989..bc452a9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,6 +20,8 @@ package org.apache.drill.exec.coord.zk;
import com.google.common.base.Preconditions;
import org.apache.parquet.Strings;
+import java.net.URL;
+
/**
* A convenience class used to expedite zookeeper paths manipulations.
*/
@@ -70,4 +72,17 @@ public final class PathUtils {
return builder.toString();
}
+ /**
+ * Creates and returns path with the protocol at the beginning from specified {@code url}.
+ *
+ * @param url the source of path and protocol
+ * @return string with protocol and path divided by colon
+ */
+ public static String getPathWithProtocol(URL url) {
+ if (url.getProtocol() != null) {
+ return url.getProtocol() + ":" + url.getPath();
+ } else {
+ return url.getPath();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java
index 8a07fd2..5c4f83f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -25,7 +25,6 @@ import java.lang.management.ManagementFactory;
import java.util.Properties;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.util.MiniZooKeeperCluster;
/**
@@ -94,6 +93,8 @@ public class ZookeeperHelper {
}
try {
+ ZookeeperTestUtil.setZookeeperSaslTestConfigProps();
+
zkCluster = new MiniZooKeeperCluster();
zkCluster.setDefaultClientPort(MiniZooKeeperCluster.DEFAULT_PORT);
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java
new file mode 100644
index 0000000..20d14de
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java
@@ -0,0 +1,44 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to you under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.drill.exec;
+
+import org.apache.drill.exec.coord.zk.PathUtils;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.server.ZooKeeperSaslServer;
+
+import static org.apache.zookeeper.Environment.JAAS_CONF_KEY;
+
+public class ZookeeperTestUtil {
+
+ private static final String LOGIN_CONF_RESOURCE_PATHNAME = "login.conf";
+
+ /**
+ * Sets zookeeper server and client SASL test config properties.
+ */
+ public static void setZookeeperSaslTestConfigProps() {
+ System.setProperty(ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY, "DrillTestServerForUnitTests");
+ System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, "DrillTestClientForUnitTests");
+ }
+
+ /**
+ * Specifies JAAS configuration file.
+ */
+ public static void setJaasTestConfigFile() {
+ String configPath = PathUtils.getPathWithProtocol(ClassLoader.getSystemResource(LOGIN_CONF_RESOURCE_PATHNAME));
+ System.setProperty(JAAS_CONF_KEY, configPath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
index 021a0b7..3e94589 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -27,6 +27,7 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.retry.RetryNTimes;
import org.apache.curator.test.TestingServer;
+import org.apache.drill.exec.ZookeeperTestUtil;
import org.apache.drill.exec.coord.store.TransientStoreConfig;
import org.apache.drill.exec.serialization.InstanceSerializer;
import org.junit.After;
@@ -61,6 +62,8 @@ public class TestEphemeralStore {
@Before
public void setUp() throws Exception {
+ ZookeeperTestUtil.setZookeeperSaslTestConfigProps();
+
server = new TestingServer();
final RetryPolicy policy = new RetryNTimes(2, 1000);
curator = CuratorFrameworkFactory.newClient(server.getConnectString(), policy);
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
index 88f1fcb..a43cee2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
@@ -31,6 +31,7 @@ import org.apache.curator.retry.RetryNTimes;
import org.apache.curator.test.TestingServer;
import org.apache.drill.common.collections.ImmutableEntry;
import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.ZookeeperTestUtil;
import org.apache.drill.exec.exception.VersionMismatchException;
import org.apache.drill.exec.store.sys.store.DataChangeVersion;
import org.apache.zookeeper.CreateMode;
@@ -71,6 +72,8 @@ public class TestZookeeperClient {
@Before
public void setUp() throws Exception {
+ ZookeeperTestUtil.setZookeeperSaslTestConfigProps();
+
server = new TestingServer();
final RetryPolicy policy = new RetryNTimes(1, 1000);
curator = CuratorFrameworkFactory.newClient(server.getConnectString(), policy);
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index 52f5fb9..a823820 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -42,6 +42,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.util.RepeatTestRule.Repeat;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ZookeeperHelper;
+import org.apache.drill.exec.ZookeeperTestUtil;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.exception.DrillbitStartupException;
import org.apache.drill.exec.exception.SchemaChangeException;
@@ -185,6 +186,8 @@ public class TestDrillbitResilience extends DrillTest {
// turn off the HTTP server to avoid port conflicts between the drill bits
System.setProperty(ExecConstants.HTTP_ENABLE, "false");
+ ZookeeperTestUtil.setJaasTestConfigFile();
+
// turn on error for failure in cancelled fragments
zkHelper = new ZookeeperHelper(true, true);
zkHelper.startZookeeper(1);
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/core-site.xml b/exec/java-exec/src/test/resources/core-site.xml
new file mode 100644
index 0000000..f56d618
--- /dev/null
+++ b/exec/java-exec/src/test/resources/core-site.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>fs.hdfs.impl</name>
+ <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
+ </property>
+ <property>
+ <name>fs.AbstractFileSystem.hdfs.impl</name>
+ <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
+ <description>The FileSystem for hdfs: uris.</description>
+ </property>
+ <property>
+ <name>dfs.namenode.edits.dir</name>
+ <value>${dfs.namenode.name.dir}</value>
+ <description>Determines where on the local filesystem the DFS name node
+ should store the transaction (edits) file. If this is a comma-delimited list
+ of directories then the transaction file is replicated in all of the
+ directories, for redundancy. Default value is same as dfs.namenode.name.dir
+ </description>
+ </property>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>file:///</value>
+ <description>The name of the default file system. A URI whose
+ scheme and authority determine the FileSystem implementation. The
+ uri's scheme determines the config property (fs.SCHEME.impl) naming
+ the FileSystem implementation class. The uri's authority is used to
+ determine the host, port, etc. for a filesystem.</description>
+ </property>
+ <property>
+ <name>datanucleus.autoCreateTables</name>
+ <value>true</value>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/resources/login.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/login.conf b/exec/java-exec/src/test/resources/login.conf
new file mode 100644
index 0000000..0916120
--- /dev/null
+++ b/exec/java-exec/src/test/resources/login.conf
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * simple login, just get OS creds
+ */
+hadoop_simple {
+ org.apache.hadoop.security.login.GenericOSLoginModule required;
+ org.apache.hadoop.security.login.HadoopLoginModule required;
+};
\ No newline at end of file