You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by bo...@apache.org on 2017/08/30 09:30:09 UTC
sqoop git commit: SQOOP-3222: Test HBase kerberized connectivity
Repository: sqoop
Updated Branches:
refs/heads/trunk 1378520e4 -> 184452908
SQOOP-3222: Test HBase kerberized connectivity
(Szabolcs Vasas via Egyed Boglarka)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/18445290
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/18445290
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/18445290
Branch: refs/heads/trunk
Commit: 18445290810b1df035e06fb074064d6b9c1d6e90
Parents: 1378520
Author: Boglarka Egyed <bo...@apache.org>
Authored: Wed Aug 30 11:28:31 2017 +0200
Committer: Boglarka Egyed <bo...@apache.org>
Committed: Wed Aug 30 11:28:31 2017 +0200
----------------------------------------------------------------------
build.xml | 11 --
ivy.xml | 2 +
.../hbase/HBaseKerberizedConnectivityTest.java | 33 ++++
.../com/cloudera/sqoop/hbase/HBaseTestCase.java | 163 +++++++++----------
.../kerberos/KerberosConfigurationProvider.java | 29 ++++
.../kerberos/MiniKdcInfrastructure.java | 27 +++
.../kerberos/MiniKdcInfrastructureRule.java | 122 ++++++++++++++
7 files changed, 289 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 5f02dcf..5836a75 100644
--- a/build.xml
+++ b/build.xml
@@ -230,11 +230,6 @@
<property name="sqoop.test.sqlserver.database"
value="sqooptest"/>
- <property name="java.security.krb5.realm"
- value="OX.AC.UK"/>
-
- <property name="java.security.krb5.kdc"
- value="kdc0.ox.ac.uk:kdc1.ox.ac.uk"/>
<property name="ms.sqlserver.username"
value="SQOOPUSER"/>
@@ -865,12 +860,6 @@
<sysproperty key="sqoop.test.msserver.connector.factory"
value="${sqoop.test.msserver.connector.factory}"/>
- <sysproperty key="java.security.krb5.realm"
- value="${java.security.krb5.realm}"/>
-
- <sysproperty key="java.security.krb5.kdc"
- value="${java.security.krb5.kdc}"/>
-
<sysproperty key="sqoop.test.db2.connectstring.host_url" value="${sqoop.test.db2.connectstring.host_url}" />
<sysproperty key="sqoop.test.db2.connectstring.database" value="${sqoop.test.db2.connectstring.database}" />
<sysproperty key="sqoop.test.db2.connectstring.username" value="${sqoop.test.db2.connectstring.username}" />
http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/ivy.xml
----------------------------------------------------------------------
diff --git a/ivy.xml b/ivy.xml
index e4b45bf..601aa01 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -81,6 +81,8 @@ under the License.
<dependency org="org.aspectj" name="aspectjrt" rev="${aspectj.version}"
conf="common->default"/>
+ <dependency org="org.apache.hadoop" name="hadoop-minikdc" rev="${hadoop.version}" conf="test->default" />
+
<!-- Common dependencies for Sqoop -->
<dependency org="commons-cli" name="commons-cli"
rev="${commons-cli.version}" conf="common->default"/>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/com/cloudera/sqoop/hbase/HBaseKerberizedConnectivityTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hbase/HBaseKerberizedConnectivityTest.java b/src/test/com/cloudera/sqoop/hbase/HBaseKerberizedConnectivityTest.java
new file mode 100644
index 0000000..3c027ad
--- /dev/null
+++ b/src/test/com/cloudera/sqoop/hbase/HBaseKerberizedConnectivityTest.java
@@ -0,0 +1,33 @@
+package com.cloudera.sqoop.hbase;
+
+import org.apache.sqoop.infrastructure.kerberos.MiniKdcInfrastructureRule;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class HBaseKerberizedConnectivityTest extends HBaseTestCase {
+
+ private static final String HBASE_TABLE_NAME = "KerberosTest";
+ private static final String HBASE_COLUMN_FAMILY = "TestColumnFamily";
+ private static final String TEST_ROW_KEY = "0";
+ private static final String TEST_ROW_VALUE = "1";
+ private static final String[] COLUMN_TYPES = { "INT", "INT" };
+
+ @ClassRule
+ public static MiniKdcInfrastructureRule miniKdcInfrastructure = new MiniKdcInfrastructureRule();
+
+ public HBaseKerberizedConnectivityTest() {
+ super(miniKdcInfrastructure);
+ }
+
+ @Test
+ public void testSqoopImportWithKerberizedHBaseConnectivitySucceeds() throws IOException {
+ String[] argv = getArgv(true, HBASE_TABLE_NAME, HBASE_COLUMN_FAMILY, true, null);
+ createTableWithColTypes(COLUMN_TYPES, new String[] { TEST_ROW_KEY, TEST_ROW_VALUE });
+
+ runImport(argv);
+
+ verifyHBaseCell(HBASE_TABLE_NAME, TEST_ROW_KEY, HBASE_COLUMN_FAMILY, getColName(1), TEST_ROW_VALUE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
index d9f7495..8b29b5f 100644
--- a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
+++ b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
@@ -18,34 +18,37 @@
package com.cloudera.sqoop.hbase;
+import static org.apache.hadoop.hbase.HConstants.MASTER_INFO_PORT;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_CLIENT_PORT;
+import static org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGION_COPROCESSOR_CONF_KEY;
+import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.KRB_PRINCIPAL;
+import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.MASTER_KRB_PRINCIPAL;
+import static org.apache.hadoop.hbase.security.User.HBASE_SECURITY_CONF_KEY;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_PRINCIPAL;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
-import java.io.File;
import java.io.IOException;
-import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
-import java.util.UUID;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.security.token.TokenProvider;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.sqoop.infrastructure.kerberos.KerberosConfigurationProvider;
import org.junit.After;
import org.junit.Before;
@@ -58,23 +61,22 @@ import com.cloudera.sqoop.testutil.ImportJobTestCase;
*/
public abstract class HBaseTestCase extends ImportJobTestCase {
- /*
- * This is to restore test.build.data system property which gets reset
- * when HBase tests are run. Since other tests in Sqoop also depend upon
- * this property, they can fail if are run subsequently in the same VM.
- */
- private static String testBuildDataProperty = "";
+ public static final Log LOG = LogFactory.getLog(
+ HBaseTestCase.class.getName());
+ private static final int NUM_MASTERS = 1;
+ private static final int NUM_SLAVES = 1;
+ private static final String MASTER_INFO_PORT_DISABLE_WEB_UI = "-1";
- private static void recordTestBuildDataProperty() {
- testBuildDataProperty = System.getProperty("test.build.data", "");
- }
+ private final KerberosConfigurationProvider kerberosConfigurationProvider;
+ private HBaseTestingUtility hbaseTestUtil;
- private static void restoreTestBuidlDataProperty() {
- System.setProperty("test.build.data", testBuildDataProperty);
+ public HBaseTestCase() {
+ this(null);
}
- public static final Log LOG = LogFactory.getLog(
- HBaseTestCase.class.getName());
+ public HBaseTestCase(KerberosConfigurationProvider kerberosConfigurationProvider) {
+ this.kerberosConfigurationProvider = kerberosConfigurationProvider;
+ }
/**
* Create the argv to pass to Sqoop.
@@ -88,8 +90,10 @@ public abstract class HBaseTestCase extends ImportJobTestCase {
if (includeHadoopFlags) {
CommonArgs.addHadoopFlags(args);
+ String zookeeperPort = hbaseTestUtil.getConfiguration().get(ZOOKEEPER_CLIENT_PORT);
args.add("-D");
args.add("hbase.zookeeper.property.clientPort=" + zookeeperPort);
+ args.addAll(getKerberosFlags());
}
if (null != queryStr) {
@@ -146,76 +150,43 @@ public abstract class HBaseTestCase extends ImportJobTestCase {
}
return args.toArray(new String[0]);
}
- // Starts a mini hbase cluster in this process.
- // Starts a mini hbase cluster in this process.
- private HBaseTestingUtility hbaseTestUtil;
- private String workDir = createTempDir().getAbsolutePath();
- private MiniZooKeeperCluster zookeeperCluster;
- private MiniHBaseCluster hbaseCluster;
- private int zookeeperPort;
@Override
@Before
public void setUp() {
try {
- String zookeeperDir = new File(workDir, "zk").getAbsolutePath();
- zookeeperCluster = new MiniZooKeeperCluster();
- zookeeperCluster.startup(new File(zookeeperDir));
- zookeeperPort = zookeeperCluster.getClientPort();
-
- HBaseTestCase.recordTestBuildDataProperty();
- String hbaseDir = new File(workDir, "hbase").getAbsolutePath();
- String hbaseRoot = "file://" + hbaseDir;
- Configuration hbaseConf = HBaseConfiguration.create();
- hbaseConf.set(HConstants.HBASE_DIR, hbaseRoot);
- //Hbase 0.90 does not have HConstants.ZOOKEEPER_CLIENT_PORT
- hbaseConf.setInt("hbase.zookeeper.property.clientPort", zookeeperPort);
- hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "0.0.0.0");
- hbaseConf.setInt("hbase.master.info.port", -1);
- hbaseConf.setInt("hbase.zookeeper.property.maxClientCnxns", 500);
- hbaseCluster = new MiniHBaseCluster(hbaseConf, 1);
- HMaster master = hbaseCluster.getMaster();
- Object serverName = master.getServerName();
+ hbaseTestUtil = new HBaseTestingUtility();
+ // We set the port for the hbase master web UI to -1 because we do not want the info server to run.
+ hbaseTestUtil.getConfiguration().set(MASTER_INFO_PORT, MASTER_INFO_PORT_DISABLE_WEB_UI);
+ setupKerberos();
- String hostAndPort;
- Method m;
- if (serverName instanceof String) {
- System.out.println("Server name is string, using HServerAddress.");
- m = HMaster.class.getDeclaredMethod("getMasterAddress",
- new Class<?>[]{});
- Class<?> clazz = Class.forName("org.apache.hadoop.hbase.HServerAddress");
- /*
- * Call method to get server address
- */
- Object serverAddr = clazz.cast(m.invoke(master, new Object[]{}));
- //returns the address as hostname:port
- hostAndPort = serverAddr.toString();
- } else {
- System.out.println("ServerName is org.apache.hadoop.hbase.ServerName,"
- + "using getHostAndPort()");
- Class<?> clazz = Class.forName("org.apache.hadoop.hbase.ServerName");
- m = clazz.getDeclaredMethod("getHostAndPort", new Class<?>[]{});
- hostAndPort = m.invoke(serverName, new Object[]{}).toString();
- }
- hbaseConf.set("hbase.master", hostAndPort);
- hbaseTestUtil = new HBaseTestingUtility(hbaseConf);
- hbaseTestUtil.setZkCluster(zookeeperCluster);
- hbaseCluster.startMaster();
+ hbaseTestUtil.startMiniZKCluster();
+ hbaseTestUtil.startMiniHBaseCluster(NUM_MASTERS, NUM_SLAVES);
super.setUp();
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
+ private void setupKerberos() {
+ if (!isKerberized()){
+ return;
+ }
+
+ String servicePrincipal = kerberosConfigurationProvider.getTestPrincipal() + "@" + kerberosConfigurationProvider.getRealm();
+ HBaseKerberosUtils.setPrincipalForTesting(servicePrincipal);
+ HBaseKerberosUtils.setKeytabFileForTesting(kerberosConfigurationProvider.getKeytabFilePath());
+ HBaseKerberosUtils.setSecuredConfiguration(hbaseTestUtil.getConfiguration());
+
+ UserGroupInformation.setConfiguration(hbaseTestUtil.getConfiguration());
+ hbaseTestUtil.getConfiguration().setStrings(REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName());
+ }
+
public void shutdown() throws Exception {
LOG.info("In shutdown() method");
- if (null != hbaseTestUtil) {
- LOG.info("Shutting down HBase cluster");
- hbaseCluster.shutdown();
- zookeeperCluster.shutdown();
- hbaseTestUtil = null;
- }
- FileUtils.deleteDirectory(new File(workDir));
+ LOG.info("Shutting down HBase cluster");
+ hbaseTestUtil.shutdownMiniCluster();
+ hbaseTestUtil = null;
LOG.info("shutdown() method returning.");
}
@@ -228,7 +199,6 @@ public abstract class HBaseTestCase extends ImportJobTestCase {
LOG.warn("Error shutting down HBase minicluster: "
+ StringUtils.stringifyException(e));
}
- HBaseTestCase.restoreTestBuidlDataProperty();
super.tearDown();
}
@@ -252,14 +222,6 @@ public abstract class HBaseTestCase extends ImportJobTestCase {
table.close();
}
}
- public static File createTempDir() {
- File baseDir = new File(System.getProperty("java.io.tmpdir"));
- File tempDir = new File(baseDir, UUID.randomUUID().toString());
- if (tempDir.mkdir()) {
- return tempDir;
- }
- throw new IllegalStateException("Failed to create directory");
- }
protected int countHBaseTable(String tableName, String colFamily)
throws IOException {
@@ -278,4 +240,31 @@ public abstract class HBaseTestCase extends ImportJobTestCase {
}
return count;
}
+
+ protected boolean isKerberized() {
+ return kerberosConfigurationProvider != null;
+ }
+
+ private String createFlagWithValue(String flag, String value) {
+ return String.format("%s=%s", flag, value);
+ }
+
+ private List<String> getKerberosFlags() {
+ if (!isKerberized()) {
+ return Collections.emptyList();
+ }
+ List<String> result = new ArrayList<>();
+
+ String principalForTesting = HBaseKerberosUtils.getPrincipalForTesting();
+ result.add("-D");
+ result.add(createFlagWithValue(HBASE_SECURITY_CONF_KEY, "kerberos"));
+ result.add("-D");
+ result.add(createFlagWithValue(MASTER_KRB_PRINCIPAL, principalForTesting));
+ result.add("-D");
+ result.add(createFlagWithValue(KRB_PRINCIPAL, principalForTesting));
+ result.add("-D");
+ result.add(createFlagWithValue(RM_PRINCIPAL, principalForTesting));
+
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java b/src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java
new file mode 100644
index 0000000..c149f1e
--- /dev/null
+++ b/src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.infrastructure.kerberos;
+
+public interface KerberosConfigurationProvider {
+
+ String getTestPrincipal();
+
+ String getRealm();
+
+ String getKeytabFilePath();
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java b/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java
new file mode 100644
index 0000000..5786a47
--- /dev/null
+++ b/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.infrastructure.kerberos;
+
+public interface MiniKdcInfrastructure extends KerberosConfigurationProvider {
+
+ void start();
+
+ void stop();
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/18445290/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructureRule.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructureRule.java b/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructureRule.java
new file mode 100644
index 0000000..a704d0b
--- /dev/null
+++ b/src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructureRule.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.infrastructure.kerberos;
+
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+
+public class MiniKdcInfrastructureRule implements TestRule, MiniKdcInfrastructure {
+
+ private MiniKdc miniKdc;
+
+ private Properties configuration;
+
+ private File workDir;
+
+ private String testPrincipal;
+
+ private File keytabFile;
+
+ public MiniKdcInfrastructureRule() {
+ File baseDir = Files.createTempDir();
+ this.workDir = new File(baseDir, "MiniKdcWorkDir");
+ this.configuration = MiniKdc.createConf();
+ }
+
+ @Override
+ public void start() {
+ try {
+ miniKdc = new MiniKdc(configuration, workDir);
+ miniKdc.start();
+ createTestPrincipal();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void createTestPrincipal() {
+ try {
+ createKeytabFile();
+ testPrincipal = currentUser() + "/" + miniKdc.getHost();
+ miniKdc.createPrincipal(keytabFile, testPrincipal);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void createKeytabFile() {
+ try {
+ keytabFile = new File(workDir.getAbsolutePath(), "keytab");
+ keytabFile.createNewFile();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ try {
+ miniKdc.stop();
+ FileUtils.deleteDirectory(workDir);
+ configuration = null;
+ miniKdc = null;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Statement apply(final Statement base, Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ start();
+ base.evaluate();
+ stop();
+ }
+ };
+ }
+
+ @Override
+ public String getTestPrincipal() {
+ return testPrincipal;
+ }
+
+ @Override
+ public String getRealm() {
+ return miniKdc.getRealm();
+ }
+
+ @Override
+ public String getKeytabFilePath() {
+ return keytabFile.getAbsolutePath();
+ }
+
+ private String currentUser() {
+ return System.getProperty("user.name");
+ }
+}