You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by dz...@apache.org on 2022/07/08 06:50:17 UTC
[drill] branch 1.20 updated (424d47de1f -> fb66c36668)
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a change to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
from 424d47de1f DRILL-8234: Register rules only from plugins used in the query (#2560)
new 7c77850ac1 DRILL-8237: Limit is not pushed down to scan for MSSQL (#2564)
new f4d37e2493 DRILL-8113: Support building with a JDK 8 target using newer JDKs (#2565)
new ffd3946867 DRILL-8245: Project pushdown depends on rules order and might not happen
new 5daf51a399 DRILL-8251: Upgrade Hadoop 2 Due To CVE (#2577)
new a5b66802f5 add log4j version in root pom
new fb66c36668 DRILL-8227: JConnect and jTDS JDBC drivers do not implement Connection::getSchema
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/drill/exec/store/httpd/HttpdParser.java | 43 +-
.../drill/exec/store/httpd/TestHTTPDLogReader.java | 6 +
.../store/httpd/TestHTTPDLogReaderUserAgent.java | 6 -
.../exec/store/jdbc/TestJdbcPluginWithMSSQL.java | 7 +-
.../apache/drill/exec/planner/PlannerPhase.java | 3 +-
.../logical/DrillPushFilterPastProjectRule.java | 11 +-
.../drill/exec/planner/logical/DrillScanRel.java | 8 +-
.../drill/exec/server/rest/auth/SpnegoConfig.java | 6 +-
.../exec/store/enumerable/plan/DrillJdbcSort.java | 3 +-
.../drill/exec/rpc/data/TestBitBitKerberos.java | 5 +-
.../rpc/user/security/TestUserBitKerberos.java | 280 ++++----
.../security/TestUserBitKerberosEncryption.java | 766 ++++++++-------------
.../rest/spnego/TestDrillSpnegoAuthenticator.java | 12 +-
.../rest/spnego/TestSpnegoAuthentication.java | 12 +-
.../exec/server/rest/spnego/TestSpnegoConfig.java | 12 +-
pom.xml | 4 +-
16 files changed, 504 insertions(+), 680 deletions(-)
[drill] 02/06: DRILL-8113: Support building with a JDK 8 target using newer JDKs (#2565)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit f4d37e24930e1f44f665c0cce8c977d6b632202c
Author: James Turton <91...@users.noreply.github.com>
AuthorDate: Wed Jun 1 06:50:56 2022 +0200
DRILL-8113: Support building with a JDK 8 target using newer JDKs (#2565)
Also upgrades and mostly enables the tests in TestUserBitKerberos*.java.
---
.../drill/exec/server/rest/auth/SpnegoConfig.java | 6 +-
.../drill/exec/rpc/data/TestBitBitKerberos.java | 5 +-
.../rpc/user/security/TestUserBitKerberos.java | 280 ++++----
.../security/TestUserBitKerberosEncryption.java | 766 ++++++++-------------
.../rest/spnego/TestDrillSpnegoAuthenticator.java | 12 +-
.../rest/spnego/TestSpnegoAuthentication.java | 12 +-
.../exec/server/rest/spnego/TestSpnegoConfig.java | 12 +-
pom.xml | 1 +
8 files changed, 456 insertions(+), 638 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/SpnegoConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/SpnegoConfig.java
index 4714689258..1570ed1776 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/SpnegoConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/SpnegoConfig.java
@@ -26,12 +26,12 @@ import org.apache.hadoop.security.UserGroupInformation;
public class SpnegoConfig {
- private UserGroupInformation loggedInUgi;
+ // Standard Object Identifier for the SPNEGO GSS-API mechanism.
+ public static final String GSS_SPNEGO_MECH_OID = "1.3.6.1.5.5.2";
+ private UserGroupInformation loggedInUgi;
private final String principal;
-
private final String keytab;
-
// Optional parameter
private final String clientNameMapping;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java
index dcd1cf9b1e..7571efb296 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java
@@ -117,7 +117,10 @@ public class TestBitBitKerberos extends BaseTestQuery {
// initialization which causes the tests to fail. So the following two changes are required.
// (1) Refresh Kerberos config.
- sun.security.krb5.Config.refresh();
+ // This disabled call to an unsupported internal API does not appear to be
+ // required and it prevents compiling with a target of JDK 8 on newer JDKs.
+ // sun.security.krb5.Config.refresh();
+
// (2) Reset the default realm.
final Field defaultRealm = KerberosName.class.getDeclaredField("defaultRealm");
defaultRealm.setAccessible(true);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
index a403d6f623..755ae06174 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
@@ -18,19 +18,19 @@
package org.apache.drill.exec.rpc.user.security;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import com.typesafe.config.ConfigValueFactory;
import org.apache.drill.categories.SecurityTest;
-import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.rpc.RpcMetrics;
import org.apache.drill.exec.rpc.control.ControlRpcMetrics;
import org.apache.drill.exec.rpc.data.DataRpcMetrics;
import org.apache.drill.exec.rpc.security.KerberosHelper;
import org.apache.drill.exec.rpc.user.UserRpcMetrics;
import org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl;
-import org.apache.drill.test.BaseTestQuery;
-import org.apache.hadoop.security.authentication.util.KerberosName;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.drill.test.ClientFixture;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.ClusterTest;
import org.apache.kerby.kerberos.kerb.client.JaasKrbUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -39,175 +39,171 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import javax.security.auth.Subject;
-import java.lang.reflect.Field;
import java.security.PrivilegedExceptionAction;
-import java.util.Properties;
-import static junit.framework.TestCase.assertTrue;
+import static junit.framework.TestCase.assertEquals;
-@Ignore("See DRILL-5387")
@Category(SecurityTest.class)
-public class TestUserBitKerberos extends BaseTestQuery {
- //private static final org.slf4j.Logger logger =org.slf4j.LoggerFactory.getLogger(TestUserBitKerberos.class);
+public class TestUserBitKerberos extends ClusterTest {
private static KerberosHelper krbHelper;
@BeforeClass
public static void setupTest() throws Exception {
-
krbHelper = new KerberosHelper(TestUserBitKerberos.class.getSimpleName(), null);
krbHelper.setupKdc(dirTestWatcher.getTmpDir());
+ cluster = defaultClusterConfig().build();
+ }
- // Create a new DrillConfig which has user authentication enabled and authenticator set to
- // UserAuthenticatorTestImpl.
- final DrillConfig newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))));
-
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.USER, "anonymous");
- connectionProps.setProperty(DrillProperties.PASSWORD, "anything works!");
-
- // Ignore the compile time warning caused by the code below.
-
- // Config is statically initialized at this point. But the above configuration results in a different
- // initialization which causes the tests to fail. So the following two changes are required.
-
- // (1) Refresh Kerberos config.
- sun.security.krb5.Config.refresh();
- // (2) Reset the default realm.
- final Field defaultRealm = KerberosName.class.getDeclaredField("defaultRealm");
- defaultRealm.setAccessible(true);
- defaultRealm.set(null, KerberosUtil.getDefaultRealm());
-
- updateTestCluster(1, newConfig, connectionProps);
+ private static ClusterFixtureBuilder defaultClusterConfig() {
+ return ClusterFixture.bareBuilder(dirTestWatcher)
+ .clusterSize(1)
+ .configProperty(ExecConstants.USER_AUTHENTICATION_ENABLED, true)
+ .configProperty(ExecConstants.USER_AUTHENTICATOR_IMPL, UserAuthenticatorTestImpl.TYPE)
+ .configProperty(ExecConstants.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .configProperty(ExecConstants.SERVICE_KEYTAB_LOCATION, krbHelper.serverKeytab.toString())
+ .configNonStringProperty(ExecConstants.AUTHENTICATION_MECHANISMS, Lists.newArrayList("plain", "kerberos"));
}
@Test
public void successKeytab() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
- updateClient(connectionProps);
-
- // Run few queries using the new client
- testBuilder()
+ try (
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
+
+ // Run few queries using the new client
+ client.testBuilder()
.sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
.unOrdered()
.baselineColumns("session_user")
.baselineValues(krbHelper.CLIENT_SHORT_NAME)
.go();
- test("SHOW SCHEMAS");
- test("USE INFORMATION_SCHEMA");
- test("SHOW TABLES");
- test("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
- test("SELECT * FROM cp.`region.json` LIMIT 5");
+
+ client.runSqlSilently("SHOW SCHEMAS");
+ client.runSqlSilently("USE INFORMATION_SCHEMA");
+ client.runSqlSilently("SHOW TABLES");
+ client.runSqlSilently("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
+ client.runSqlSilently("SELECT * FROM cp.`region.json` LIMIT 5");
+ }
}
@Test
public void successTicket() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KERBEROS_FROM_SUBJECT, "true");
- final Subject clientSubject = JaasKrbUtil.loginUsingKeytab(krbHelper.CLIENT_PRINCIPAL,
- krbHelper.clientKeytab.getAbsoluteFile());
-
- Subject.doAs(clientSubject, new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- updateClient(connectionProps);
- return null;
- }
- });
+ Subject clientSubject = JaasKrbUtil.loginUsingKeytab(
+ krbHelper.CLIENT_PRINCIPAL,
+ krbHelper.clientKeytab.getAbsoluteFile()
+ );
+
+ try (
+ ClientFixture client = Subject.doAs(
+ clientSubject,
+ (PrivilegedExceptionAction<ClientFixture>) () -> cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.KERBEROS_FROM_SUBJECT, "true")
+ .build()
+ )
+ ) {
// Run few queries using the new client
- testBuilder()
- .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
- .unOrdered()
- .baselineColumns("session_user")
- .baselineValues(krbHelper.CLIENT_SHORT_NAME)
- .go();
- test("SHOW SCHEMAS");
- test("USE INFORMATION_SCHEMA");
- test("SHOW TABLES");
- test("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
- test("SELECT * FROM cp.`region.json` LIMIT 5");
- }
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+
+ client.runSqlSilently("SHOW SCHEMAS");
+ client.runSqlSilently("USE INFORMATION_SCHEMA");
+ client.runSqlSilently("SHOW TABLES");
+ client.runSqlSilently("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
+ client.runSqlSilently("SELECT * FROM cp.`region.json` LIMIT 5");
+ }
+ }
@Test
- public void testUnecryptedConnectionCounter() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KERBEROS_FROM_SUBJECT, "true");
- final Subject clientSubject = JaasKrbUtil.loginUsingKeytab(krbHelper.CLIENT_PRINCIPAL,
- krbHelper.clientKeytab.getAbsoluteFile());
-
- Subject.doAs(clientSubject, new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- updateClient(connectionProps);
- return null;
- }
- });
-
- // Run few queries using the new client
- testBuilder()
- .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
- .unOrdered()
- .baselineColumns("session_user")
- .baselineValues(krbHelper.CLIENT_SHORT_NAME)
- .go();
-
- // Check encrypted counters value
- assertTrue(0 == UserRpcMetrics.getInstance().getEncryptedConnectionCount());
- assertTrue(0 == ControlRpcMetrics.getInstance().getEncryptedConnectionCount());
- assertTrue(0 == DataRpcMetrics.getInstance().getEncryptedConnectionCount());
-
- // Check unencrypted counters value
- assertTrue(1 == UserRpcMetrics.getInstance().getUnEncryptedConnectionCount());
- assertTrue(0 == ControlRpcMetrics.getInstance().getUnEncryptedConnectionCount());
- assertTrue(0 == DataRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ @Ignore("See DRILL-5387. This test works in isolation but not when sharing counters with other tests")
+ public void testUnencryptedConnectionCounter() throws Exception {
+ Subject clientSubject = JaasKrbUtil.loginUsingKeytab(
+ krbHelper.CLIENT_PRINCIPAL,
+ krbHelper.clientKeytab.getAbsoluteFile()
+ );
+
+ try (
+ // Use a dedicated cluster fixture so that the tested RPC counters have a clean start.
+ ClusterFixture cluster = defaultClusterConfig().build();
+ ClientFixture client = Subject.doAs(
+ clientSubject,
+ (PrivilegedExceptionAction<ClientFixture>) () -> cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.KERBEROS_FROM_SUBJECT, "true")
+ .build()
+ )
+ ) {
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+
+ RpcMetrics userMetrics = UserRpcMetrics.getInstance(),
+ ctrlMetrics = ControlRpcMetrics.getInstance(),
+ dataMetrics = DataRpcMetrics.getInstance();
+
+ // Check encrypted counters value
+ assertEquals(0, userMetrics.getEncryptedConnectionCount());
+ assertEquals(0, ctrlMetrics.getEncryptedConnectionCount());
+ assertEquals(0, dataMetrics.getEncryptedConnectionCount());
+
+ // Check unencrypted counters value
+ assertEquals(1, userMetrics.getUnEncryptedConnectionCount());
+ assertEquals(0, ctrlMetrics.getUnEncryptedConnectionCount());
+ assertEquals(0, dataMetrics.getUnEncryptedConnectionCount());
+ }
}
@Test
- public void testUnecryptedConnectionCounter_LocalControlMessage() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KERBEROS_FROM_SUBJECT, "true");
- final Subject clientSubject = JaasKrbUtil.loginUsingKeytab(krbHelper.CLIENT_PRINCIPAL,
- krbHelper.clientKeytab.getAbsoluteFile());
-
- Subject.doAs(clientSubject, new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- updateClient(connectionProps);
- return null;
- }
- });
-
- // Run query on memory system table this sends remote fragments to all Drillbit and Drillbits then send data
- // using data channel. In this test we have only 1 Drillbit so there should not be any control connection but a
- // local data connections
- testSql("SELECT * FROM sys.memory");
-
- // Check encrypted counters value
- assertTrue(0 == UserRpcMetrics.getInstance().getEncryptedConnectionCount());
- assertTrue(0 == ControlRpcMetrics.getInstance().getEncryptedConnectionCount());
- assertTrue(0 == DataRpcMetrics.getInstance().getEncryptedConnectionCount());
-
- // Check unencrypted counters value
- assertTrue(1 == UserRpcMetrics.getInstance().getUnEncryptedConnectionCount());
- assertTrue(0 == ControlRpcMetrics.getInstance().getUnEncryptedConnectionCount());
- assertTrue(2 == DataRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ @Ignore("See DRILL-5387. This test works in isolation but not when sharing counters with other tests")
+ public void testUnencryptedConnectionCounter_LocalControlMessage() throws Exception {
+ Subject clientSubject = JaasKrbUtil.loginUsingKeytab(
+ krbHelper.CLIENT_PRINCIPAL,
+ krbHelper.clientKeytab.getAbsoluteFile()
+ );
+
+ try (
+ // Use a dedicated cluster fixture so that the tested RPC counters have a clean start.
+ ClusterFixture cluster = defaultClusterConfig().build();
+ ClientFixture client = Subject.doAs(
+ clientSubject,
+ (PrivilegedExceptionAction<ClientFixture>) () -> cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.KERBEROS_FROM_SUBJECT, "true")
+ .build()
+ )
+ ) {
+ // Run query on memory system table this sends remote fragments to all Drillbit and Drillbits then send data
+ // using data channel. In this test we have only 1 Drillbit so there should not be any control connection but a
+ // local data connections
+ client.runSqlSilently("SELECT * FROM sys.memory");
+
+ RpcMetrics userMetrics = UserRpcMetrics.getInstance(),
+ ctrlMetrics = ControlRpcMetrics.getInstance(),
+ dataMetrics = DataRpcMetrics.getInstance();
+
+ // Check encrypted counters value
+ assertEquals(0, userMetrics.getEncryptedConnectionCount());
+ assertEquals(0, ctrlMetrics.getEncryptedConnectionCount());
+ assertEquals(0, dataMetrics.getEncryptedConnectionCount());
+
+ // Check unencrypted counters value
+ assertEquals(1, userMetrics.getUnEncryptedConnectionCount());
+ assertEquals(0, ctrlMetrics.getUnEncryptedConnectionCount());
+ assertEquals(2, dataMetrics.getUnEncryptedConnectionCount());
+ }
}
@AfterClass
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
index e609f59c3c..b0449d50be 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
@@ -18,21 +18,21 @@
package org.apache.drill.exec.rpc.user.security;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import com.typesafe.config.ConfigValueFactory;
-import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.ClientFixture;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.ClusterTest;
import org.apache.drill.categories.SecurityTest;
-import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.rpc.NonTransientRpcException;
import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.RpcMetrics;
import org.apache.drill.exec.rpc.control.ControlRpcMetrics;
import org.apache.drill.exec.rpc.data.DataRpcMetrics;
import org.apache.drill.exec.rpc.security.KerberosHelper;
import org.apache.drill.exec.rpc.user.UserRpcMetrics;
import org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl;
-import org.apache.hadoop.security.authentication.util.KerberosName;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.kerby.kerberos.kerb.client.JaasKrbUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -41,62 +41,34 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import javax.security.auth.Subject;
-import java.lang.reflect.Field;
import java.security.PrivilegedExceptionAction;
-import java.util.Properties;
-import static junit.framework.TestCase.assertTrue;
+import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.fail;
-@Ignore("See DRILL-5387")
@Category(SecurityTest.class)
-public class TestUserBitKerberosEncryption extends BaseTestQuery {
+public class TestUserBitKerberosEncryption extends ClusterTest {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(TestUserBitKerberosEncryption.class);
private static KerberosHelper krbHelper;
- private static DrillConfig newConfig;
@BeforeClass
public static void setupTest() throws Exception {
krbHelper = new KerberosHelper(TestUserBitKerberosEncryption.class.getSimpleName(), null);
krbHelper.setupKdc(dirTestWatcher.getTmpDir());
+ cluster = defaultClusterConfig().build();
+ }
- // Create a new DrillConfig which has user authentication enabled and authenticator set to
- // UserAuthenticatorTestImpl.
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- // Ignore the compile time warning caused by the code below.
-
- // Config is statically initialized at this point. But the above configuration results in a different
- // initialization which causes the tests to fail. So the following two changes are required.
-
- // (1) Refresh Kerberos config.
- sun.security.krb5.Config.refresh();
- // (2) Reset the default realm.
- final Field defaultRealm = KerberosName.class.getDeclaredField("defaultRealm");
- defaultRealm.setAccessible(true);
- defaultRealm.set(null, KerberosUtil.getDefaultRealm());
-
- // Start a secure cluster with client using Kerberos related parameters.
- updateTestCluster(1, newConfig, connectionProps);
+ private static ClusterFixtureBuilder defaultClusterConfig() {
+ return ClusterFixture.bareBuilder(dirTestWatcher)
+ .clusterSize(1)
+ .configProperty(ExecConstants.USER_AUTHENTICATION_ENABLED, true)
+ .configProperty(ExecConstants.USER_AUTHENTICATOR_IMPL, UserAuthenticatorTestImpl.TYPE)
+ .configProperty(ExecConstants.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .configProperty(ExecConstants.SERVICE_KEYTAB_LOCATION, krbHelper.serverKeytab.toString())
+ .configNonStringProperty(ExecConstants.AUTHENTICATION_MECHANISMS, Lists.newArrayList("plain", "kerberos"))
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, "true");
}
@AfterClass
@@ -106,39 +78,27 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
@Test
public void successKeytabWithoutChunking() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- updateTestCluster(1, newConfig, connectionProps);
-
- // Run few queries using the new client
- testBuilder()
- .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
- .unOrdered()
- .baselineColumns("session_user")
- .baselineValues(krbHelper.CLIENT_SHORT_NAME)
- .go();
- test("SHOW SCHEMAS");
- test("USE INFORMATION_SCHEMA");
- test("SHOW TABLES");
- test("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
- test("SELECT * FROM cp.`region.json`");
+ try (
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
+ // Run few queries using the new client
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+
+ client.runSqlSilently("SHOW SCHEMAS");
+ client.runSqlSilently("USE INFORMATION_SCHEMA");
+ client.runSqlSilently("SHOW TABLES");
+ client.runSqlSilently("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
+ client.runSqlSilently("SELECT * FROM cp.`region.json`");
+ }
}
/**
@@ -154,170 +114,121 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
* @throws Exception
*/
@Test
+ @Ignore("See DRILL-5387. This test works in isolation but not when sharing counters with other tests")
public void testConnectionCounters() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- updateTestCluster(1, newConfig, connectionProps);
-
- assertTrue(UserRpcMetrics.getInstance().getEncryptedConnectionCount() == 1);
- assertTrue(UserRpcMetrics.getInstance().getUnEncryptedConnectionCount() == 0);
-
- // Run few queries using the new client
- testBuilder()
+ try (
+ // Use a dedicated cluster fixture so that the tested RPC counters have a clean start.
+ ClusterFixture cluster = defaultClusterConfig().build();
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
+ client.testBuilder()
.sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
.unOrdered()
.baselineColumns("session_user")
.baselineValues(krbHelper.CLIENT_SHORT_NAME)
.go();
- // Check encrypted counters value
- assertTrue(1 == UserRpcMetrics.getInstance().getEncryptedConnectionCount());
- assertTrue(0 == ControlRpcMetrics.getInstance().getEncryptedConnectionCount());
- assertTrue(0 == DataRpcMetrics.getInstance().getEncryptedConnectionCount());
+ RpcMetrics userMetrics = UserRpcMetrics.getInstance(),
+ ctrlMetrics = ControlRpcMetrics.getInstance(),
+ dataMetrics = DataRpcMetrics.getInstance();
+
+ // Check encrypted counters value, only user-bit encryption is enabled
+ assertEquals(1, userMetrics.getEncryptedConnectionCount());
+ assertEquals(0, ctrlMetrics.getEncryptedConnectionCount());
+ assertEquals(0, dataMetrics.getEncryptedConnectionCount());
- // Check unencrypted counters value
- assertTrue(0 == UserRpcMetrics.getInstance().getUnEncryptedConnectionCount());
- assertTrue(0 == ControlRpcMetrics.getInstance().getUnEncryptedConnectionCount());
- assertTrue(0 == DataRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ // Check encrypted counters value, only user-bit encryption is enabled
+ assertEquals(0, userMetrics.getUnEncryptedConnectionCount());
+ assertEquals(0, ctrlMetrics.getUnEncryptedConnectionCount());
+ assertEquals(0, dataMetrics.getUnEncryptedConnectionCount());
+ }
}
@Test
public void successTicketWithoutChunking() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KERBEROS_FROM_SUBJECT, "true");
- final Subject clientSubject = JaasKrbUtil.loginUsingKeytab(krbHelper.CLIENT_PRINCIPAL,
- krbHelper.clientKeytab.getAbsoluteFile());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- Subject.doAs(clientSubject, new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- updateTestCluster(1, newConfig, connectionProps);
- return null;
- }
- });
-
- // Run few queries using the new client
- testBuilder()
+ Subject clientSubject = JaasKrbUtil.loginUsingKeytab(
+ krbHelper.CLIENT_PRINCIPAL,
+ krbHelper.clientKeytab.getAbsoluteFile()
+ );
+
+ try (
+ ClientFixture client = Subject.doAs(
+ clientSubject,
+ (PrivilegedExceptionAction<ClientFixture>) () -> cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.KERBEROS_FROM_SUBJECT, "true")
+ .build()
+ )
+ ) {
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+
+ client.runSqlSilently("SHOW SCHEMAS");
+ client.runSqlSilently("USE INFORMATION_SCHEMA");
+ client.runSqlSilently("SHOW TABLES");
+ client.runSqlSilently("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
+ client.runSqlSilently("SELECT * FROM cp.`region.json` LIMIT 5");
+ }
+ }
+
+ @Test
+ public void successKeytabWithChunking() throws Exception {
+ try (
+ ClusterFixture cluster = defaultClusterConfig()
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, 100)
+ .build();
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
+ client.testBuilder()
.sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
.unOrdered()
.baselineColumns("session_user")
.baselineValues(krbHelper.CLIENT_SHORT_NAME)
.go();
- test("SHOW SCHEMAS");
- test("USE INFORMATION_SCHEMA");
- test("SHOW TABLES");
- test("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
- test("SELECT * FROM cp.`region.json` LIMIT 5");
- }
- @Test
- public void successKeytabWithChunking() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE,
- ConfigValueFactory.fromAnyRef(100)));
-
- updateTestCluster(1, newConfig, connectionProps);
-
- // Run few queries using the new client
- testBuilder()
- .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
- .unOrdered()
- .baselineColumns("session_user")
- .baselineValues(krbHelper.CLIENT_SHORT_NAME)
- .go();
- test("SHOW SCHEMAS");
- test("USE INFORMATION_SCHEMA");
- test("SHOW TABLES");
- test("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
- test("SELECT * FROM cp.`region.json`");
+ client.runSqlSilently("SHOW SCHEMAS");
+ client.runSqlSilently("USE INFORMATION_SCHEMA");
+ client.runSqlSilently("SHOW TABLES");
+ client.runSqlSilently("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
+ client.runSqlSilently("SELECT * FROM cp.`region.json` LIMIT 5");
+ }
}
@Test
public void successKeytabWithChunkingDefaultChunkSize() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- updateTestCluster(1, newConfig, connectionProps);
-
- // Run few queries using the new client
- testBuilder()
- .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
- .unOrdered()
- .baselineColumns("session_user")
- .baselineValues(krbHelper.CLIENT_SHORT_NAME)
- .go();
- test("SHOW SCHEMAS");
- test("USE INFORMATION_SCHEMA");
- test("SHOW TABLES");
- test("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
- test("SELECT * FROM cp.`region.json` LIMIT 5");
- }
+ try (
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+ client.runSqlSilently("SHOW SCHEMAS");
+ client.runSqlSilently("USE INFORMATION_SCHEMA");
+ client.runSqlSilently("SHOW TABLES");
+ client.runSqlSilently("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
+ client.runSqlSilently("SELECT * FROM cp.`region.json` LIMIT 5");
+ }
+ }
/**
* This test will not cover the data channel since we are using only 1 Drillbit and the query doesn't involve
@@ -326,54 +237,36 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
*/
@Test
public void successEncryptionAllChannelChunkMode() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE,
- ConfigValueFactory.fromAnyRef(10000))
- .withValue(ExecConstants.BIT_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_AUTHENTICATION_MECHANISM,
- ConfigValueFactory.fromAnyRef("kerberos"))
- .withValue(ExecConstants.USE_LOGIN_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_ENCRYPTION_SASL_MAX_WRAPPED_SIZE,
- ConfigValueFactory.fromAnyRef(10000)));
-
- updateTestCluster(1, newConfig, connectionProps);
-
- // Run few queries using the new client
- testBuilder()
- .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
- .unOrdered()
- .baselineColumns("session_user")
- .baselineValues(krbHelper.CLIENT_SHORT_NAME)
- .go();
- test("SHOW SCHEMAS");
- test("USE INFORMATION_SCHEMA");
- test("SHOW TABLES");
- test("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
- test("SELECT * FROM cp.`region.json` LIMIT 5");
- }
-
+ try (
+ ClusterFixture cluster = defaultClusterConfig()
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, 100)
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, 10000)
+ .configProperty(ExecConstants.BIT_AUTHENTICATION_ENABLED, true)
+ .configProperty(ExecConstants.BIT_AUTHENTICATION_MECHANISM, "kerberos")
+ .configProperty(ExecConstants.USE_LOGIN_PRINCIPAL, true)
+ .configProperty(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED, true)
+ .configProperty(ExecConstants.BIT_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, 10000)
+ .build();
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+ client.runSqlSilently("SHOW SCHEMAS");
+ client.runSqlSilently("USE INFORMATION_SCHEMA");
+ client.runSqlSilently("SHOW TABLES");
+ client.runSqlSilently("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
+ client.runSqlSilently("SELECT * FROM cp.`region.json` LIMIT 5");
+ }
+ }
/**
* This test will not cover the data channel since we are using only 1 Drillbit and the query doesn't involve
@@ -382,48 +275,34 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
*/
@Test
public void successEncryptionAllChannel() throws Exception {
+ try (
+ ClusterFixture cluster = defaultClusterConfig()
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, 100)
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, 10000)
+ .configProperty(ExecConstants.BIT_AUTHENTICATION_ENABLED, true)
+ .configProperty(ExecConstants.BIT_AUTHENTICATION_MECHANISM, "kerberos")
+ .configProperty(ExecConstants.USE_LOGIN_PRINCIPAL, true)
+ .configProperty(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED, true)
+ .build();
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_AUTHENTICATION_MECHANISM,
- ConfigValueFactory.fromAnyRef("kerberos"))
- .withValue(ExecConstants.USE_LOGIN_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- updateTestCluster(1, newConfig, connectionProps);
-
- // Run few queries using the new client
- testBuilder()
- .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
- .unOrdered()
- .baselineColumns("session_user")
- .baselineValues(krbHelper.CLIENT_SHORT_NAME)
- .go();
- test("SHOW SCHEMAS");
- test("USE INFORMATION_SCHEMA");
- test("SHOW TABLES");
- test("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
- test("SELECT * FROM cp.`region.json` LIMIT 5");
+ client.runSqlSilently("SHOW SCHEMAS");
+ client.runSqlSilently("USE INFORMATION_SCHEMA");
+ client.runSqlSilently("SHOW TABLES");
+ client.runSqlSilently("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE 'COLUMNS'");
+ client.runSqlSilently("SELECT * FROM cp.`region.json` LIMIT 5");
+ }
}
/**
@@ -438,77 +317,55 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
*/
@Test
+ @Ignore("See DRILL-5387. This test works in isolation but not when sharing counters with other tests")
public void testEncryptedConnectionCountersAllChannel() throws Exception {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_AUTHENTICATION_MECHANISM,
- ConfigValueFactory.fromAnyRef("kerberos"))
- .withValue(ExecConstants.USE_LOGIN_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- updateTestCluster(1, newConfig, connectionProps);
-
- // Run few queries using the new client
- testBuilder()
+ try (
+ // Use a dedicated cluster fixture so that the tested RPC counters have a clean start.
+ ClusterFixture cluster = defaultClusterConfig()
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, 100)
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, 10000)
+ .configProperty(ExecConstants.BIT_AUTHENTICATION_ENABLED, true)
+ .configProperty(ExecConstants.BIT_AUTHENTICATION_MECHANISM, "kerberos")
+ .configProperty(ExecConstants.USE_LOGIN_PRINCIPAL, true)
+ .configProperty(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED, true)
+ .build();
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
+ client.testBuilder()
.sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
.unOrdered()
.baselineColumns("session_user")
.baselineValues(krbHelper.CLIENT_SHORT_NAME)
.go();
- // Check encrypted counters value
- assertTrue(1 == UserRpcMetrics.getInstance().getEncryptedConnectionCount());
- assertTrue(0 == ControlRpcMetrics.getInstance().getEncryptedConnectionCount());
- assertTrue(0 == DataRpcMetrics.getInstance().getEncryptedConnectionCount());
+ RpcMetrics userMetrics = UserRpcMetrics.getInstance(),
+ ctrlMetrics = ControlRpcMetrics.getInstance(),
+ dataMetrics = DataRpcMetrics.getInstance();
- // Check unencrypted counters value
- assertTrue(0 == UserRpcMetrics.getInstance().getUnEncryptedConnectionCount());
- assertTrue(0 == ControlRpcMetrics.getInstance().getUnEncryptedConnectionCount());
- assertTrue(0 == DataRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ // Check encrypted counters value
+ assertEquals(1, userMetrics.getEncryptedConnectionCount());
+ assertEquals(0, ctrlMetrics.getEncryptedConnectionCount());
+ assertEquals(0, dataMetrics.getEncryptedConnectionCount());
+
+ // Check unencrypted counters value
+ assertEquals(0, userMetrics.getUnEncryptedConnectionCount());
+ assertEquals(0, ctrlMetrics.getUnEncryptedConnectionCount());
+ assertEquals(0, dataMetrics.getUnEncryptedConnectionCount());
+ }
}
@Test
public void failurePlainMech() {
- try {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.USER, "anonymous");
- connectionProps.setProperty(DrillProperties.PASSWORD, "anything works!");
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- updateTestCluster(1, newConfig, connectionProps);
+ try (
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.USER, "anonymous")
+ .property(DrillProperties.PASSWORD, "anything works!")
+ .build()
+ ) {
fail();
} catch (Exception ex) {
assert (ex.getCause() instanceof NonTransientRpcException);
@@ -518,27 +375,16 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
@Test
public void encryptionEnabledWithOnlyPlainMech() {
- try {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
- updateTestCluster(1, newConfig, connectionProps);
-
+ try (
+ ClusterFixture cluster = defaultClusterConfig()
+ .configNonStringProperty(ExecConstants.AUTHENTICATION_MECHANISMS, Lists.newArrayList("plain"))
+ .build();
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .build()
+ ) {
fail();
} catch (Exception ex) {
assert (ex.getCause() instanceof NonTransientRpcException);
@@ -552,28 +398,14 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
*/
@Test
public void failureOldClientEncryptionEnabled() {
- try {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
- connectionProps.setProperty(DrillProperties.TEST_SASL_LEVEL, "1");
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
- updateTestCluster(1, newConfig, connectionProps);
-
+ try (
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .property(DrillProperties.TEST_SASL_LEVEL, "1")
+ .build()
+ ) {
fail();
} catch (Exception ex) {
assert (ex.getCause() instanceof RpcException);
@@ -587,26 +419,27 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
*/
@Test
public void successOldClientEncryptionDisabled() {
-
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
- connectionProps.setProperty(DrillProperties.TEST_SASL_LEVEL, "1");
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))));
-
- updateTestCluster(1, newConfig, connectionProps);
+ try (
+ ClusterFixture cluster = defaultClusterConfig()
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, false)
+ .build();
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .property(DrillProperties.TEST_SASL_LEVEL, "1")
+ .build()
+ ) {
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+ } catch (Exception ex) {
+ fail();
+ assert (ex.getCause() instanceof NonTransientRpcException);
+ }
}
/**
@@ -614,28 +447,18 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
* to server with encryption disabled.
*/
@Test
- public void clientNeedsEncryptionWithNoServerSupport() throws Exception {
- try {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
- connectionProps.setProperty(DrillProperties.SASL_ENCRYPT, "true");
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))));
-
- updateTestCluster(1, newConfig, connectionProps);
-
+ public void clientNeedsEncryptionWithNoServerSupport() {
+ try (
+ ClusterFixture cluster = defaultClusterConfig()
+ .configProperty(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, false)
+ .build();
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .property(DrillProperties.SASL_ENCRYPT, "true")
+ .build()
+ ) {
fail();
} catch (Exception ex) {
assert (ex.getCause() instanceof NonTransientRpcException);
@@ -647,32 +470,25 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
* to server with encryption enabled.
*/
@Test
- public void clientNeedsEncryptionWithServerSupport() throws Exception {
- try {
- final Properties connectionProps = new Properties();
- connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
- connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
- connectionProps.setProperty(DrillProperties.SASL_ENCRYPT, "true");
-
- newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
- .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
- ConfigValueFactory.fromAnyRef(true))
- .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
- ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
- .withValue(ExecConstants.SERVICE_PRINCIPAL,
- ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
- .withValue(ExecConstants.SERVICE_KEYTAB_LOCATION,
- ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
- .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
- ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
- .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
- ConfigValueFactory.fromAnyRef(true)));
-
- updateTestCluster(1, newConfig, connectionProps);
+ public void clientNeedsEncryptionWithServerSupport() {
+ try (
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
+ .property(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL)
+ .property(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath())
+ .property(DrillProperties.SASL_ENCRYPT, "true")
+ .build()
+ ) {
+ client.testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
} catch (Exception ex) {
fail();
assert (ex.getCause() instanceof NonTransientRpcException);
}
}
}
+
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java
index a23776e30c..914688e9b2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java
@@ -29,6 +29,7 @@ import org.apache.drill.exec.server.options.SystemOptionManager;
import org.apache.drill.exec.server.rest.WebServerConstants;
import org.apache.drill.exec.server.rest.auth.DrillSpnegoAuthenticator;
import org.apache.drill.exec.server.rest.auth.DrillSpnegoLoginService;
+import org.apache.drill.exec.server.rest.auth.SpnegoConfig;
import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.BaseTest;
import org.apache.hadoop.security.authentication.util.KerberosName;
@@ -50,7 +51,6 @@ import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
-import sun.security.jgss.GSSUtil;
import javax.security.auth.Subject;
import javax.servlet.http.HttpServletRequest;
@@ -67,7 +67,6 @@ import static org.mockito.Mockito.verify;
/**
* Test for validating {@link DrillSpnegoAuthenticator}
*/
-@Ignore("See DRILL-5387")
@Category(SecurityTest.class)
public class TestDrillSpnegoAuthenticator extends BaseTest {
@@ -84,8 +83,10 @@ public class TestDrillSpnegoAuthenticator extends BaseTest {
spnegoHelper = new KerberosHelper(TestSpnegoAuthentication.class.getSimpleName(), primaryName);
spnegoHelper.setupKdc(dirTestWatcher.getTmpDir());
-
- sun.security.krb5.Config.refresh();
+ // (1) Refresh Kerberos config.
+ // This disabled call to an unsupported internal API does not appear to be
+ // required and it prevents compiling with a target of JDK 8 on newer JDKs.
+ // sun.security.krb5.Config.refresh();
// (2) Reset the default realm.
final Field defaultRealm = KerberosName.class.getDeclaredField("defaultRealm");
@@ -203,6 +204,7 @@ public class TestDrillSpnegoAuthenticator extends BaseTest {
* {@link DrillSpnegoAuthenticator#validateRequest(javax.servlet.ServletRequest, javax.servlet.ServletResponse, boolean)}
*/
@Test
+ @Ignore("See DRILL-5387")
public void testAuthClientRequestForLogOut() throws Exception {
final HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
final HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
@@ -242,7 +244,7 @@ public class TestDrillSpnegoAuthenticator extends BaseTest {
final GSSManager gssManager = GSSManager.getInstance();
GSSContext gssContext = null;
try {
- final Oid oid = GSSUtil.GSS_SPNEGO_MECH_OID;
+ final Oid oid = new Oid(SpnegoConfig.GSS_SPNEGO_MECH_OID);
final GSSName serviceName = gssManager.createName(spnegoHelper.SERVER_PRINCIPAL, GSSName.NT_USER_NAME, oid);
gssContext = gssManager.createContext(serviceName, oid, null, GSSContext.DEFAULT_LIFETIME);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java
index 572f32c11f..fb79f173a2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java
@@ -34,6 +34,7 @@ import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.options.SystemOptionManager;
import org.apache.drill.exec.server.rest.auth.DrillHttpSecurityHandlerProvider;
import org.apache.drill.exec.server.rest.auth.DrillSpnegoLoginService;
+import org.apache.drill.exec.server.rest.auth.SpnegoConfig;
import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.BaseTest;
import org.apache.hadoop.security.authentication.util.KerberosName;
@@ -50,7 +51,6 @@ import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
-import sun.security.jgss.GSSUtil;
import javax.security.auth.Subject;
import java.lang.reflect.Field;
@@ -64,7 +64,6 @@ import static org.junit.Assert.assertTrue;
/**
* Test for validating {@link DrillSpnegoLoginService}
*/
-@Ignore("See DRILL-5387")
@Category(SecurityTest.class)
public class TestSpnegoAuthentication extends BaseTest {
@@ -79,8 +78,10 @@ public class TestSpnegoAuthentication extends BaseTest {
spnegoHelper = new KerberosHelper(TestSpnegoAuthentication.class.getSimpleName(), primaryName);
spnegoHelper.setupKdc(dirTestWatcher.getTmpDir());
-
- sun.security.krb5.Config.refresh();
+ // (1) Refresh Kerberos config.
+ // This disabled call to an unsupported internal API does not appear to be
+ // required and it prevents compiling with a target of JDK 8 on newer JDKs.
+ // sun.security.krb5.Config.refresh();
// (2) Reset the default realm.
final Field defaultRealm = KerberosName.class.getDeclaredField("defaultRealm");
@@ -246,6 +247,7 @@ public class TestSpnegoAuthentication extends BaseTest {
* when provided with client token for a configured service principal.
*/
@Test
+ @Ignore("See DRILL-5387")
public void testDrillSpnegoLoginService() throws Exception {
// Create client subject using it's principal and keytab
@@ -260,7 +262,7 @@ public class TestSpnegoAuthentication extends BaseTest {
final GSSManager gssManager = GSSManager.getInstance();
GSSContext gssContext = null;
try {
- final Oid oid = GSSUtil.GSS_SPNEGO_MECH_OID;
+ final Oid oid = new Oid(SpnegoConfig.GSS_SPNEGO_MECH_OID);
final GSSName serviceName = gssManager.createName(spnegoHelper.SERVER_PRINCIPAL, GSSName.NT_USER_NAME, oid);
gssContext = gssManager.createContext(serviceName, oid, null, GSSContext.DEFAULT_LIFETIME);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java
index 3831f50a2c..a91f802319 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -46,11 +45,8 @@ import static org.junit.Assert.assertTrue;
/**
* Test for validating {@link SpnegoConfig}
*/
-@Ignore("See DRILL-5387")
@Category(SecurityTest.class)
public class TestSpnegoConfig extends BaseTest {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSpnegoConfig.class);
-
private static KerberosHelper spnegoHelper;
private static final String primaryName = "HTTP";
@@ -62,8 +58,10 @@ public class TestSpnegoConfig extends BaseTest {
spnegoHelper = new KerberosHelper(TestSpnegoAuthentication.class.getSimpleName(), primaryName);
spnegoHelper.setupKdc(dirTestWatcher.getTmpDir());
-
- sun.security.krb5.Config.refresh();
+ // (1) Refresh Kerberos config.
+ // This disabled call to an unsupported internal API does not appear to be
+ // required and it prevents compiling with a target of JDK 8 on newer JDKs.
+ // sun.security.krb5.Config.refresh();
// (2) Reset the default realm.
final Field defaultRealm = KerberosName.class.getDeclaredField("defaultRealm");
@@ -163,4 +161,4 @@ public class TestSpnegoConfig extends BaseTest {
fail();
}
}
-}
\ No newline at end of file
+}
diff --git a/pom.xml b/pom.xml
index 98b920ba5c..25a54718ad 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3985,6 +3985,7 @@
<jdk>[9,)</jdk>
</activation>
<properties>
+ <maven.compiler.release>8</maven.compiler.release>
<junit.args>
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens java.base/java.net=ALL-UNNAMED
[drill] 01/06: DRILL-8237: Limit is not pushed down to scan for MSSQL (#2564)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7c77850ac12fbaa0d8c832efe61241891d343593
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Fri May 27 03:16:33 2022 +0300
DRILL-8237: Limit is not pushed down to scan for MSSQL (#2564)
---
.../org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java | 5 +----
.../org/apache/drill/exec/store/enumerable/plan/DrillJdbcSort.java | 3 ++-
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java
index 68c6871261..a1299f67a0 100644
--- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java
+++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java
@@ -31,7 +31,6 @@ import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.rowSet.RowSetUtilities;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.testcontainers.containers.MSSQLServerContainer;
@@ -48,7 +47,7 @@ import static org.junit.Assert.assertEquals;
@Category(JdbcStorageTest.class)
public class TestJdbcPluginWithMSSQL extends ClusterTest {
- private static MSSQLServerContainer jdbcContainer;
+ private static MSSQLServerContainer<?> jdbcContainer;
@BeforeClass
public static void initMSSQL() throws Exception {
@@ -313,8 +312,6 @@ public class TestJdbcPluginWithMSSQL extends ClusterTest {
}
@Test
- @Ignore
- // TODO: Enable once the push down logic has been clarified.
public void testLimitPushDownWithOffset() throws Exception {
String query = "select person_id, first_name from mssql.dbo.person limit 100 offset 10";
queryBuilder()
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/enumerable/plan/DrillJdbcSort.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/enumerable/plan/DrillJdbcSort.java
index 1f15ce3388..a1ab014f89 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/enumerable/plan/DrillJdbcSort.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/enumerable/plan/DrillJdbcSort.java
@@ -42,7 +42,8 @@ public class DrillJdbcSort extends JdbcRules.JdbcSort {
double numRows = mq.getRowCount(this);
double cpuCost = DrillCostBase.COMPARE_CPU_COST * numRows;
DrillCostBase.DrillCostFactory costFactory = (DrillCostBase.DrillCostFactory) planner.getCostFactory();
- return costFactory.makeCost(numRows, cpuCost, 0, 0);
+ // adjust cost to handle the case when the original limit was split
+ return costFactory.makeCost(numRows, cpuCost, 0, 0).multiplyBy(0.1);
}
return super.computeSelfCost(planner, mq);
}
[drill] 03/06: DRILL-8245: Project pushdown depends on rules order and might not happen
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ffd39468672fb56d46082d7bb8b0cdb4d0f9b76b
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Thu Jun 9 20:11:17 2022 +0300
DRILL-8245: Project pushdown depends on rules order and might not happen
---
.../apache/drill/exec/store/httpd/HttpdParser.java | 43 ++++++++++++----------
.../drill/exec/store/httpd/TestHTTPDLogReader.java | 6 +++
.../store/httpd/TestHTTPDLogReaderUserAgent.java | 6 ---
.../apache/drill/exec/planner/PlannerPhase.java | 3 +-
.../logical/DrillPushFilterPastProjectRule.java | 11 ++++--
.../drill/exec/planner/logical/DrillScanRel.java | 8 ++--
6 files changed, 43 insertions(+), 34 deletions(-)
diff --git a/contrib/format-httpd/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java b/contrib/format-httpd/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
index c31c5ad131..b7553051fb 100644
--- a/contrib/format-httpd/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
+++ b/contrib/format-httpd/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
@@ -33,10 +33,10 @@ import nl.basjes.parse.core.exceptions.DissectionFailure;
import nl.basjes.parse.core.exceptions.InvalidDissectorException;
import nl.basjes.parse.core.exceptions.MissingDissectorsException;
import nl.basjes.parse.httpdlog.HttpdLoglineParser;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -192,13 +192,7 @@ public class HttpdParser {
if (!isStarQuery() &&
!isMetadataQuery() &&
!isOnlyImplicitColumns()) {
- List<String> keysToRemove = new ArrayList<>();
- for (final String key : requestedPaths.keySet()) {
- if (!isRequested(key)) {
- keysToRemove.add(key);
- }
- }
- keysToRemove.forEach( key -> requestedPaths.remove(key));
+ requestedPaths = getRequestedColumnPaths();
}
EnumSet<Casts> allCasts;
@@ -256,6 +250,25 @@ public class HttpdParser {
return builder.build();
}
+ private Map<String, String> getRequestedColumnPaths() {
+ Map<String, String> requestedColumnPaths = new TreeMap<>();
+ for (SchemaPath requestedColumn : requestedColumns) {
+ String columnName = requestedColumn.getRootSegmentPath();
+ String parserPath = requestedPaths.get(columnName);
+ if (parserPath != null) {
+ requestedColumnPaths.put(columnName, parserPath);
+ } else {
+ requestedPaths.keySet()
+ .stream()
+ .filter(colName -> colName.endsWith(HttpdUtils.SAFE_WILDCARD)
+ && requestedColumn.rootName().startsWith(colName.substring(0, colName.length() - HttpdUtils.SAFE_WILDCARD.length())))
+ .findAny()
+ .ifPresent(colName -> requestedColumnPaths.put(colName, requestedPaths.get(colName)));
+ }
+ }
+ return requestedColumnPaths;
+ }
+
public void addFieldsToParser(RowSetLoader rowWriter) {
for (final Map.Entry<String, String> entry : requestedPaths.entrySet()) {
try {
@@ -268,24 +281,14 @@ public class HttpdParser {
}
public boolean isStarQuery() {
- return requestedColumns.size() == 1 && requestedColumns.get(0).isDynamicStar();
+ return requestedColumns.stream()
+ .anyMatch(SchemaPath::isDynamicStar);
}
public boolean isMetadataQuery() {
return requestedColumns.size() == 0;
}
- public boolean isRequested(String colName) {
- for (SchemaPath path : requestedColumns) {
- if (path.isDynamicStar()) {
- return true;
- } else if (path.nameEquals(colName)) {
- return true;
- }
- }
- return false;
- }
-
/*
This is for the edge case where a query only contains the implicit fields.
*/
diff --git a/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReader.java b/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReader.java
index 4b41562dcb..3950af3163 100644
--- a/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReader.java
+++ b/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReader.java
@@ -118,6 +118,12 @@ public class TestHTTPDLogReader extends ClusterTest {
"\\\"%{User-agent}i\\\"', " +
"flattenWildcards => true)) WHERE `request_firstline_original_uri_query_came__from` IS NOT NULL";
+ queryBuilder()
+ .sql(sql)
+ .planMatcher()
+ .include("columns=\\[`request_firstline_original_uri_query_came__from`\\]")
+ .match();
+
RowSet results = client.queryBuilder().sql(sql).rowSet();
TupleMetadata expectedSchema = new SchemaBuilder()
diff --git a/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReaderUserAgent.java b/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReaderUserAgent.java
index 8b23efbbfc..27041c45d2 100644
--- a/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReaderUserAgent.java
+++ b/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReaderUserAgent.java
@@ -199,8 +199,6 @@ public class TestHTTPDLogReaderUserAgent extends ClusterTest {
RowSet results = client.queryBuilder().sql(sql).rowSet();
- results.print();
-
TupleMetadata expectedSchema = new SchemaBuilder()
.addNullable("request_receive_time_epoch", MinorType.TIMESTAMP)
.addNullable("request_user-agent", MinorType.VARCHAR)
@@ -255,8 +253,4 @@ public class TestHTTPDLogReaderUserAgent extends ClusterTest {
RowSetUtilities.verify(expected, results);
}
-
-
}
-
-
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
index 97b34e1049..9a34b937b0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
@@ -322,7 +322,8 @@ public enum PlannerPhase {
/*
Filter push-down related rules
*/
- DrillPushFilterPastProjectRule.INSTANCE,
+ DrillPushFilterPastProjectRule.LOGICAL,
+ DrillPushFilterPastProjectRule.DRILL_INSTANCE,
// Due to infinite loop in planning (DRILL-3257/CALCITE-1271), temporarily use this rule in Hep planner
// RuleInstance.FILTER_SET_OP_TRANSPOSE_RULE,
DrillFilterAggregateTransposeRule.INSTANCE,
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
index 5ea1d58cda..146d0f209f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.logical;
+import org.apache.drill.exec.planner.common.DrillProjectRelBase;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
@@ -39,7 +40,10 @@ import java.util.List;
public class DrillPushFilterPastProjectRule extends RelOptRule {
- public final static RelOptRule INSTANCE = new DrillPushFilterPastProjectRule(DrillRelFactories.LOGICAL_BUILDER);
+ public final static RelOptRule LOGICAL = new DrillPushFilterPastProjectRule(
+ LogicalFilter.class, LogicalProject.class, DrillRelFactories.LOGICAL_BUILDER, "DrillPushFilterPastProjectRule:logical");
+ public final static RelOptRule DRILL_INSTANCE = new DrillPushFilterPastProjectRule(
+ DrillFilterRel.class, DrillProjectRelBase.class, DrillRelFactories.LOGICAL_BUILDER, "DrillPushFilterPastProjectRule:drill_logical");
private static final Collection<String> BANNED_OPERATORS;
@@ -49,8 +53,9 @@ public class DrillPushFilterPastProjectRule extends RelOptRule {
BANNED_OPERATORS.add("item");
}
- private DrillPushFilterPastProjectRule(RelBuilderFactory relBuilderFactory) {
- super(operand(LogicalFilter.class, operand(LogicalProject.class, any())), relBuilderFactory,null);
+ private DrillPushFilterPastProjectRule(Class<? extends Filter> filter,
+ Class<? extends Project> project, RelBuilderFactory relBuilderFactory, String description) {
+ super(operand(filter, operand(project, any())), relBuilderFactory,description);
}
//~ Methods ----------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
index bcd9792f60..61f06e712c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
@@ -60,9 +60,7 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel {
public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits,
final RelOptTable table, boolean partitionFilterPushdown) {
- // By default, scan does not support project pushdown.
- // Decision whether push projects into scan will be made solely in DrillPushProjIntoScanRule.
- this(cluster, traits, table, table.getRowType(), getProjectedColumns(table, true), partitionFilterPushdown);
+ this(cluster, traits, table, table.getRowType(), getProjectedColumns(table, false), partitionFilterPushdown);
this.settings = PrelUtil.getPlannerSettings(cluster.getPlanner());
}
@@ -139,7 +137,9 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel {
@Override
public RelOptCost computeSelfCost(final RelOptPlanner planner, RelMetadataQuery mq) {
final ScanStats stats = getGroupScan().getScanStats(settings);
- int columnCount = Utilities.isStarQuery(columns) ? STAR_COLUMN_COST : getRowType().getFieldCount();
+ double columnCount = Utilities.isStarQuery(columns)
+ ? STAR_COLUMN_COST
+ : Math.pow(getRowType().getFieldCount(), 2) / Math.max(columns.size(), 1);
// double rowCount = RelMetadataQuery.getRowCount(this);
double rowCount = Math.max(1, stats.getRecordCount());
[drill] 06/06: DRILL-8227: JConnect and jTDS JDBC drivers do not implement Connection::getSchema
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit fb66c3666812cbf7385ca44cf03adde8d14c0806
Author: kingswanwho <ji...@u.northwestern.edu>
AuthorDate: Tue Jul 5 17:06:03 2022 +0800
DRILL-8227: JConnect and jTDS JDBC drivers do not implement Connection::getSchema
Fix build errors in the original backport attempt (aa5341df6).
---
.../java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java
index a1299f67a0..f451d51c22 100644
--- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java
+++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.store.jdbc;
import org.apache.drill.categories.JdbcStorageTest;
import org.apache.drill.common.logical.security.PlainCredentialsProvider;
-import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.physical.rowSet.DirectRowSet;
import org.apache.drill.exec.physical.rowSet.RowSet;
@@ -85,7 +84,6 @@ public class TestJdbcPluginWithMSSQL extends ClusterTest {
false,
sourceParms,
credentialsProvider,
- AuthMode.SHARED_USER.name(),
100000
);
jdbcStorageConfig.setEnabled(true);
[drill] 04/06: DRILL-8251: Upgrade Hadoop 2 Due To CVE (#2577)
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 5daf51a399211c467fa22c0a82945d58cac157b6
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Mon Jun 20 14:13:50 2022 +0100
DRILL-8251: Upgrade Hadoop 2 Due To CVE (#2577)
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 25a54718ad..6d3d1f4cc1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4004,7 +4004,7 @@
</property>
</activation>
<properties>
- <hadoop.version>2.10.1</hadoop.version>
+ <hadoop.version>2.10.2</hadoop.version>
</properties>
<dependencyManagement>
<dependencies>
[drill] 05/06: add log4j version in root pom
Posted by dz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch 1.20
in repository https://gitbox.apache.org/repos/asf/drill.git
commit a5b66802f5c3e5a9df78bca40fee659fef8e26f9
Author: kingswanwho <ji...@u.northwestern.edu>
AuthorDate: Tue Jul 5 12:55:20 2022 +0800
add log4j version in root pom
---
pom.xml | 1 +
1 file changed, 1 insertion(+)
diff --git a/pom.xml b/pom.xml
index 6d3d1f4cc1..27385ae964 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,6 +140,7 @@
<univocity-parsers.version>2.8.3</univocity-parsers.version>
<mongo.version>4.3.3</mongo.version>
<junit.args />
+ <log4j.version>2.17.2</log4j.version>
</properties>
<scm>