You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2017/08/25 13:41:50 UTC
[1/3] drill git commit: DRILL-5725: Update Jackson version to 2.7.8
Repository: drill
Updated Branches:
refs/heads/master 0641dedcd -> a3442021b
DRILL-5725: Update Jackson version to 2.7.8
closes #908
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1c3c65fe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1c3c65fe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1c3c65fe
Branch: refs/heads/master
Commit: 1c3c65fe8a00899980414a7a410e08c70b6973f0
Parents: 0641ded
Author: Volodymyr Vysotskyi <vv...@gmail.com>
Authored: Wed Aug 16 14:16:55 2017 +0000
Committer: Arina Ielchiieva <ar...@gmail.com>
Committed: Fri Aug 25 14:52:41 2017 +0300
----------------------------------------------------------------------
exec/vector/pom.xml | 4 ++--
pom.xml | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/1c3c65fe/exec/vector/pom.xml
----------------------------------------------------------------------
diff --git a/exec/vector/pom.xml b/exec/vector/pom.xml
index 3a04a10..780d90c 100644
--- a/exec/vector/pom.xml
+++ b/exec/vector/pom.xml
@@ -44,12 +44,12 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>2.4.3</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.4.3</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/1c3c65fe/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 659c71d..6bd5b66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -38,7 +38,7 @@
<calcite.version>1.4.0-drill-r21</calcite.version>
<janino.version>2.7.6</janino.version>
<sqlline.version>1.1.9-drill-r7</sqlline.version>
- <jackson.version>2.7.1</jackson.version>
+ <jackson.version>2.7.8</jackson.version>
<mapr.release.version>5.2.1-mapr</mapr.release.version>
<ojai.version>1.1</ojai.version>
<kerby.version>1.0.0-RC2</kerby.version>
[2/3] drill git commit: DRILL-5701: Fix
drill.connections.rpc.. metric
behavior
Posted by ar...@apache.org.
DRILL-5701: Fix drill.connections.rpc.<user/control/data>.<encrypted/unencrypted> metric behavior
closes #894
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e91ae509
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e91ae509
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e91ae509
Branch: refs/heads/master
Commit: e91ae5099ade86e5d45157deacd2eae4f43a8c9c
Parents: 1c3c65f
Author: Sorabh Hamirwasia <sh...@maprtech.com>
Authored: Wed Aug 2 14:25:21 2017 -0700
Committer: Arina Ielchiieva <ar...@gmail.com>
Committed: Fri Aug 25 14:53:36 2017 +0300
----------------------------------------------------------------------
.../drill/exec/rpc/control/ControlClient.java | 8 +-
.../exec/rpc/control/ControlRpcMetrics.java | 12 +-
.../drill/exec/rpc/control/ControlServer.java | 11 +-
.../apache/drill/exec/rpc/data/DataClient.java | 13 +--
.../drill/exec/rpc/data/DataRpcMetrics.java | 12 +-
.../apache/drill/exec/rpc/data/DataServer.java | 11 +-
.../drill/exec/rpc/user/UserRpcMetrics.java | 12 +-
.../apache/drill/exec/rpc/user/UserServer.java | 9 +-
.../rpc/user/security/TestUserBitKerberos.java | 40 +++++++
.../security/TestUserBitKerberosEncryption.java | 110 +++++++++++++++++++
.../org/apache/drill/exec/rpc/RpcMetrics.java | 4 +
11 files changed, 217 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
index a46e968..1e0313a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
@@ -79,6 +79,11 @@ public class ControlClient extends BasicClient<RpcType, ControlConnection, BitCo
? config.getMessageHandler()
: new FailingRequestHandler<ControlConnection>(),
this);
+
+ // Increase the connection count here since at this point it means that we already have the TCP connection.
+ // Later when connection fails for any reason then we will decrease the counter based on Netty's connection close
+ // handler.
+ connection.incConnectionCounter();
return connection;
}
@@ -132,9 +137,6 @@ public class ControlClient extends BasicClient<RpcType, ControlConnection, BitCo
@Override
protected void finalizeConnection(BitControlHandshake handshake, ControlConnection connection) {
connection.setEndpoint(handshake.getEndpoint());
-
- // Increment the Control Connection counter.
- connection.incConnectionCounter();
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlRpcMetrics.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlRpcMetrics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlRpcMetrics.java
index ae9e7cc..344e807 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlRpcMetrics.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlRpcMetrics.java
@@ -26,7 +26,7 @@ import org.apache.drill.exec.rpc.RpcMetrics;
/**
* Holds metrics related to bit control rpc layer
*/
-class ControlRpcMetrics extends AbstractRpcMetrics {
+public class ControlRpcMetrics extends AbstractRpcMetrics {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ControlRpcMetrics.class);
// Total number of control connection's as client and server for a DrillBit.
@@ -78,6 +78,16 @@ class ControlRpcMetrics extends AbstractRpcMetrics {
}
}
+ @Override
+ public long getEncryptedConnectionCount() {
+ return encryptedConnections.getCount();
+ }
+
+ @Override
+ public long getUnEncryptedConnectionCount() {
+ return unencryptedConnection.getCount();
+ }
+
private void registerAllocatorMetrics(final BufferAllocator allocator) {
registerAllocatorMetrics(allocator, ALLOCATOR_METRICS_PREFIX + "bit.control.");
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java
index 09f6705..d8cc3fb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlServer.java
@@ -60,12 +60,18 @@ public class ControlServer extends BasicServer<RpcType, ControlConnection>{
@Override
protected ControlConnection initRemoteConnection(SocketChannel channel) {
super.initRemoteConnection(channel);
- return new ControlConnection(channel, "control server", config,
+ final ControlConnection controlConnection = new ControlConnection(channel, "control server", config,
config.getAuthMechanismToUse() == null
? config.getMessageHandler()
: new ServerAuthenticationHandler<>(config.getMessageHandler(),
RpcType.SASL_MESSAGE_VALUE, RpcType.SASL_MESSAGE),
this);
+
+ // Increase the connection count here since at this point it means that we already have the TCP connection.
+ // Later when connection fails for any reason then we will decrease the counter based on Netty's connection close
+ // handler.
+ controlConnection.incConnectionCounter();
+ return controlConnection;
}
@@ -104,9 +110,6 @@ public class ControlServer extends BasicServer<RpcType, ControlConnection>{
builder.addAllAuthenticationMechanisms(config.getAuthProvider().getAllFactoryNames());
}
- // Increase the Control Connection counter on server side
- connection.incConnectionCounter();
-
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
index 603168d..cba323e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
@@ -72,7 +72,12 @@ public class DataClient extends BasicClient<RpcType, DataClientConnection, BitCl
@Override
protected DataClientConnection initRemoteConnection(SocketChannel channel) {
super.initRemoteConnection(channel);
- this.connection = new DataClientConnection(channel, this, config.getEncryptionCtxt());
+ connection = new DataClientConnection(channel, this, config.getEncryptionCtxt());
+
+ // Increase the connection count here since at this point it means that we already have the TCP connection.
+ // Later when connection fails for any reason then we will decrease the counter based on Netty's connection close
+ // handler.
+ connection.incConnectionCounter();
return connection;
}
@@ -129,12 +134,6 @@ public class DataClient extends BasicClient<RpcType, DataClientConnection, BitCl
}
}
- @Override
- protected void finalizeConnection(BitServerHandshake handshake, DataClientConnection connection) {
- // Increment the Data Connection counter.
- connection.incConnectionCounter();
- }
-
protected <M extends MessageLite> RpcCommand<M, DataClientConnection>
getInitialCommand(final RpcCommand<M, DataClientConnection> command) {
final RpcCommand<M, DataClientConnection> initialCommand = super.getInitialCommand(command);
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcMetrics.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcMetrics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcMetrics.java
index 997df57..017b803 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcMetrics.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcMetrics.java
@@ -26,7 +26,7 @@ import org.apache.drill.exec.rpc.RpcMetrics;
/**
* Holds metrics related to bit data rpc layer
*/
-class DataRpcMetrics extends AbstractRpcMetrics {
+public class DataRpcMetrics extends AbstractRpcMetrics {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataRpcMetrics.class);
// Total number of data connection's as client and server for a DrillBit.
@@ -79,6 +79,16 @@ class DataRpcMetrics extends AbstractRpcMetrics {
}
}
+ @Override
+ public long getEncryptedConnectionCount() {
+ return encryptedConnections.getCount();
+ }
+
+ @Override
+ public long getUnEncryptedConnectionCount() {
+ return unencryptedConnection.getCount();
+ }
+
private void registerAllocatorMetrics(final BufferAllocator allocator) {
registerAllocatorMetrics(allocator, ALLOCATOR_METRICS_PREFIX + "bit.data.");
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
index 9e31d6b..c4c8490 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
@@ -58,7 +58,13 @@ public class DataServer extends BasicServer<RpcType, DataServerConnection> {
@Override
protected DataServerConnection initRemoteConnection(SocketChannel channel) {
super.initRemoteConnection(channel);
- return new DataServerConnection(channel, config);
+ final DataServerConnection dataServerConnection = new DataServerConnection(channel, config);
+
+ // Increase the connection count here since at this point it means that we already have the TCP connection.
+ // Later when connection fails for any reason then we will decrease the counter based on Netty's connection close
+ // handler.
+ dataServerConnection.incConnectionCounter();
+ return dataServerConnection;
}
@Override
@@ -83,9 +89,6 @@ public class DataServer extends BasicServer<RpcType, DataServerConnection> {
builder.addAllAuthenticationMechanisms(config.getAuthProvider().getAllFactoryNames());
}
- // Increase the Data Connection counter on server side.
- connection.incConnectionCounter();
-
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcMetrics.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcMetrics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcMetrics.java
index ab93e3d..4c4506d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcMetrics.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserRpcMetrics.java
@@ -26,7 +26,7 @@ import org.apache.drill.exec.rpc.RpcMetrics;
/**
* Holds metrics related to bit user rpc layer
*/
-class UserRpcMetrics extends AbstractRpcMetrics {
+public class UserRpcMetrics extends AbstractRpcMetrics {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UserRpcMetrics.class);
// Total number of user client connection's to a DrillBit.
@@ -78,6 +78,16 @@ class UserRpcMetrics extends AbstractRpcMetrics {
}
}
+ @Override
+ public long getEncryptedConnectionCount() {
+ return encryptedConnections.getCount();
+ }
+
+ @Override
+ public long getUnEncryptedConnectionCount() {
+ return unencryptedConnection.getCount();
+ }
+
private void registerAllocatorMetrics(final BufferAllocator allocator) {
registerAllocatorMetrics(allocator, ALLOCATOR_METRICS_PREFIX + "bit.user.");
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
index 35dbbe9..254fdca 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
@@ -110,6 +110,11 @@ public class UserServer extends BasicServer<RpcType, BitToUserConnection> {
? config.getMessageHandler()
: new ServerAuthenticationHandler<>(config.getMessageHandler(),
RpcType.SASL_MESSAGE_VALUE, RpcType.SASL_MESSAGE));
+
+ // Increase the connection count here since at this point it means that we already have the TCP connection.
+ // Later when connection fails for any reason then we will decrease the counter based on Netty's connection close
+ // handler.
+ incConnectionCounter();
}
void disableReadTimeout() {
@@ -150,10 +155,6 @@ public class UserServer extends BasicServer<RpcType, BitToUserConnection> {
if (config.getImpersonationManager() != null && targetName != null) {
config.getImpersonationManager().replaceUserOnSession(targetName, session);
}
-
- // Increase the corresponding connection counter.
- // For older clients we call this method directly.
- incConnectionCounter();
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
index 3fad005..9da0d86 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
@@ -23,7 +23,10 @@ import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.rpc.control.ControlRpcMetrics;
+import org.apache.drill.exec.rpc.data.DataRpcMetrics;
import org.apache.drill.exec.rpc.security.KerberosHelper;
+import org.apache.drill.exec.rpc.user.UserRpcMetrics;
import org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl;
import org.apache.drill.exec.server.BootStrapContext;
import org.apache.hadoop.security.authentication.util.KerberosName;
@@ -39,6 +42,8 @@ import java.lang.reflect.Field;
import java.security.PrivilegedExceptionAction;
import java.util.Properties;
+import static junit.framework.TestCase.assertTrue;
+
@Ignore("See DRILL-5387")
public class TestUserBitKerberos extends BaseTestQuery {
//private static final org.slf4j.Logger logger =org.slf4j.LoggerFactory.getLogger(TestUserBitKerberos.class);
@@ -137,6 +142,41 @@ public class TestUserBitKerberos extends BaseTestQuery {
test("SELECT * FROM cp.`region.json` LIMIT 5");
}
+ @Test
+ public void testUnecryptedConnectionCounter() throws Exception {
+ final Properties connectionProps = new Properties();
+ connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
+ connectionProps.setProperty(DrillProperties.KERBEROS_FROM_SUBJECT, "true");
+ final Subject clientSubject = JaasKrbUtil.loginUsingKeytab(krbHelper.CLIENT_PRINCIPAL,
+ krbHelper.clientKeytab.getAbsoluteFile());
+
+ Subject.doAs(clientSubject, new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ updateClient(connectionProps);
+ return null;
+ }
+ });
+
+ // Run few queries using the new client
+ testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+
+ // Check encrypted counters value
+ assertTrue(0 == UserRpcMetrics.getInstance().getEncryptedConnectionCount());
+ assertTrue(0 == ControlRpcMetrics.getInstance().getEncryptedConnectionCount());
+ assertTrue(0 == DataRpcMetrics.getInstance().getEncryptedConnectionCount());
+
+ // Check unencrypted counters value
+ assertTrue(1 == UserRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ assertTrue(2 == ControlRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ assertTrue(0 == DataRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ }
+
@AfterClass
public static void cleanTest() throws Exception {
krbHelper.stopKdc();
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
index b9dd705..b4d56ba 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
@@ -25,7 +25,10 @@ import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.rpc.NonTransientRpcException;
import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.control.ControlRpcMetrics;
+import org.apache.drill.exec.rpc.data.DataRpcMetrics;
import org.apache.drill.exec.rpc.security.KerberosHelper;
+import org.apache.drill.exec.rpc.user.UserRpcMetrics;
import org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl;
import org.apache.drill.exec.server.BootStrapContext;
import org.apache.hadoop.security.authentication.util.KerberosName;
@@ -41,6 +44,7 @@ import java.lang.reflect.Field;
import java.security.PrivilegedExceptionAction;
import java.util.Properties;
+import static junit.framework.TestCase.assertTrue;
import static junit.framework.TestCase.fail;
@Ignore("See DRILL-5387")
@@ -121,6 +125,49 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
test("SELECT * FROM cp.`region.json`");
}
+ /**
+ * Test connection counter values for both encrypted and unencrypted connections over all Drillbit channels.
+ * Encryption is enabled only for UserRpc NOT for ControlRpc and DataRpc. Test validates corresponding connection
+ * count for each channel.
+ * For example: There is only 1 DrillClient so encrypted connection count of UserRpcMetrics will be 1. Before
+ * running any query there should not be any connection (control or data) between Drillbits, hence those counters
+ * are 0. After running a simple query since there is only 1 fragment which is root fragment the Control Connection
+ * count is 2 (for unencrypted counter) based on connection for status update of fragment to Foreman. It is 2 because
+ * for Control and Data Server we count total number of client and server connections on a node. There is no
+ * Data Connection because there is no data exchange between multiple fragments.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testConnectionCounters() throws Exception {
+ final Properties connectionProps = new Properties();
+ connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
+ connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
+ connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
+ updateClient(connectionProps);
+
+ assertTrue(UserRpcMetrics.getInstance().getEncryptedConnectionCount() == 1);
+ assertTrue(UserRpcMetrics.getInstance().getUnEncryptedConnectionCount() == 0);
+
+ // Run few queries using the new client
+ testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+
+ // Check encrypted counters value
+ assertTrue(1 == UserRpcMetrics.getInstance().getEncryptedConnectionCount());
+ assertTrue(0 == ControlRpcMetrics.getInstance().getEncryptedConnectionCount());
+ assertTrue(0 == DataRpcMetrics.getInstance().getEncryptedConnectionCount());
+
+ // Check unencrypted counters value
+ assertTrue(0 == UserRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ assertTrue(2 == ControlRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ assertTrue(0 == DataRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ }
+
@Test
public void successTicketWithoutChunking() throws Exception {
final Properties connectionProps = new Properties();
@@ -339,6 +386,69 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
test("SELECT * FROM cp.`region.json` LIMIT 5");
}
+ /**
+ * Test connection counter values for both encrypted and unencrypted connections over all Drillbit channels.
+ * Encryption is enabled for UserRpc, ControlRpc and DataRpc. Test validates corresponding connection
+ * count for each channel.
+ * For example: There is only 1 DrillClient so encrypted connection count of UserRpcMetrics
+ * will be 1. Before running any query there should not be any connection (control or data) between Drillbits,
+ * hence those counters are 0. After running a simple query since there is only 1 fragment which is root fragment
+ * the Control Connection count is 2 (for encrypted counter) based on connection for status update of fragment to
+ * Foreman. It is 2 because for Control and Data Server we count total number of client and server connections on a
+ * node. There is no Data Connection because there is no data exchange between multiple fragments.
+ */
+
+ @Test
+ public void testEncryptedConnectionCountersAllChannel() throws Exception {
+ final Properties connectionProps = new Properties();
+ connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL);
+ connectionProps.setProperty(DrillProperties.USER, krbHelper.CLIENT_PRINCIPAL);
+ connectionProps.setProperty(DrillProperties.KEYTAB, krbHelper.clientKeytab.getAbsolutePath());
+
+ newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
+ .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED,
+ ConfigValueFactory.fromAnyRef(true))
+ .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL,
+ ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))
+ .withValue(BootStrapContext.SERVICE_PRINCIPAL,
+ ConfigValueFactory.fromAnyRef(krbHelper.SERVER_PRINCIPAL))
+ .withValue(BootStrapContext.SERVICE_KEYTAB_LOCATION,
+ ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString()))
+ .withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
+ ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))
+ .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED,
+ ConfigValueFactory.fromAnyRef(true))
+ .withValue(ExecConstants.BIT_AUTHENTICATION_ENABLED,
+ ConfigValueFactory.fromAnyRef(true))
+ .withValue(ExecConstants.BIT_AUTHENTICATION_MECHANISM,
+ ConfigValueFactory.fromAnyRef("kerberos"))
+ .withValue(ExecConstants.USE_LOGIN_PRINCIPAL,
+ ConfigValueFactory.fromAnyRef(true))
+ .withValue(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED,
+ ConfigValueFactory.fromAnyRef(true))
+ ,false);
+
+ updateTestCluster(1, newConfig, connectionProps);
+
+ // Run few queries using the new client
+ testBuilder()
+ .sqlQuery("SELECT session_user FROM (SELECT * FROM sys.drillbits LIMIT 1)")
+ .unOrdered()
+ .baselineColumns("session_user")
+ .baselineValues(krbHelper.CLIENT_SHORT_NAME)
+ .go();
+
+ // Check encrypted counters value
+ assertTrue(1 == UserRpcMetrics.getInstance().getEncryptedConnectionCount());
+ assertTrue(2 == ControlRpcMetrics.getInstance().getEncryptedConnectionCount());
+ assertTrue(0 == DataRpcMetrics.getInstance().getEncryptedConnectionCount());
+
+ // Check unencrypted counters value
+ assertTrue(0 == UserRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ assertTrue(0 == ControlRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ assertTrue(0 == DataRpcMetrics.getInstance().getUnEncryptedConnectionCount());
+ }
+
@Test
public void failurePlainMech() {
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/e91ae509/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMetrics.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMetrics.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMetrics.java
index a737095..b7e2e90 100644
--- a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMetrics.java
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RpcMetrics.java
@@ -26,4 +26,8 @@ public interface RpcMetrics {
void addConnectionCount();
void decConnectionCount();
+
+ long getEncryptedConnectionCount();
+
+ long getUnEncryptedConnectionCount();
}
[3/3] drill git commit: DRILL-5507: Made verbose info logging message
debug level and printed it less frequently
Posted by ar...@apache.org.
DRILL-5507: Made verbose info logging message debug level and printed it less frequently
closes #911
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a3442021
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a3442021
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a3442021
Branch: refs/heads/master
Commit: a3442021bca05fc17f399338c36ce8a6ba140e0c
Parents: e91ae50
Author: Timothy Farkas <ti...@apache.org>
Authored: Thu Aug 17 14:29:38 2017 -0700
Committer: Arina Ielchiieva <ar...@gmail.com>
Committed: Fri Aug 25 14:53:37 2017 +0300
----------------------------------------------------------------------
.../org/apache/drill/exec/store/schedule/BlockMapBuilder.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a3442021/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
index 829bcd8..d898566 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.drill.exec.metrics.DrillMetrics;
@@ -45,6 +46,7 @@ import com.google.common.collect.ImmutableRangeMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
public class BlockMapBuilder {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BlockMapBuilder.class);
@@ -228,6 +230,7 @@ public class BlockMapBuilder {
// Find submap of ranges that intersect with the rowGroup
ImmutableRangeMap<Long,BlockLocation> subRangeMap = blockMap.subRangeMap(rowGroupRange);
+ final Set<String> noDrillbitHosts = logger.isDebugEnabled() ? Sets.<String>newHashSet() : null;
// Iterate through each block in this submap and get the host for the block location
for (Map.Entry<Range<Long>,BlockLocation> block : subRangeMap.asMapOfRanges().entrySet()) {
@@ -246,8 +249,8 @@ public class BlockMapBuilder {
DrillbitEndpoint endpoint = getDrillBitEndpoint(host);
if (endpoint != null) {
endpointByteMap.add(endpoint, bytes);
- } else {
- logger.info("Failure finding Drillbit running on host {}. Skipping affinity to that host.", host);
+ } else if (noDrillbitHosts != null && noDrillbitHosts.add(host)) {
+ logger.debug("Failure finding Drillbit running on host {}. Skipping affinity to that host.", host);
}
}
}