You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2021/07/09 23:53:06 UTC
[accumulo-testing] branch 2.0 updated: Allow kerberized run (#142)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/2.0 by this push:
new 41ba8e8 Allow kerberized run (#142)
41ba8e8 is described below
commit 41ba8e83114a70fb6aa8340fc332f314d073668e
Author: BukrosSzabolcs <sz...@cloudera.com>
AuthorDate: Sat Jul 10 01:52:56 2021 +0200
Allow kerberized run (#142)
Add login functionality to client creation and pass the delegation
token when yarn is used to allow running tests on a kerberized cluster.
(security tests excluded)
---
.../apache/accumulo/testing/KerberosHelper.java | 83 ++++++++++++++++++++++
.../java/org/apache/accumulo/testing/TestEnv.java | 1 +
.../testing/continuous/ContinuousMoru.java | 8 ++-
.../testing/continuous/ContinuousVerify.java | 8 ++-
.../apache/accumulo/testing/mapreduce/RowHash.java | 9 ++-
.../accumulo/testing/mapreduce/TeraSortIngest.java | 5 +-
.../testing/randomwalk/multitable/CopyTool.java | 11 ++-
.../randomwalk/sequential/MapRedVerifyTool.java | 10 ++-
8 files changed, 121 insertions(+), 14 deletions(-)
diff --git a/src/main/java/org/apache/accumulo/testing/KerberosHelper.java b/src/main/java/org/apache/accumulo/testing/KerberosHelper.java
new file mode 100644
index 0000000..84bc325
--- /dev/null
+++ b/src/main/java/org/apache/accumulo/testing/KerberosHelper.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.security.tokens.DelegationToken;
+import org.apache.accumulo.core.conf.ClientProperty;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KerberosHelper {
+ private static final Logger log = LoggerFactory.getLogger(KerberosHelper.class);
+
+ public static void saslLogin(Properties clientProps, Configuration conf) {
+ if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) {
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ try {
+ UserGroupInformation.loginUserFromKeytab(
+ ClientProperty.AUTH_PRINCIPAL.getValue(clientProps),
+ ClientProperty.AUTH_TOKEN.getValue(clientProps));
+ } catch (IOException e) {
+ log.warn("Sasl login failed", e);
+ }
+ }
+ }
+
+ public static Properties configDelegationToken(Properties clientProps)
+ throws AccumuloException, AccumuloSecurityException {
+ if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) {
+ AccumuloClient client = Accumulo.newClient().from(clientProps).build();
+ DelegationToken dt = client.securityOperations().getDelegationToken(null);
+ clientProps = Accumulo.newClientProperties().from(clientProps)
+ .as(ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), dt).build();
+ client.close();
+ }
+ return clientProps;
+ }
+
+ public static Properties configDelegationToken(TestEnv env)
+ throws AccumuloException, AccumuloSecurityException {
+ Properties clientProps = env.getClientProps();
+ if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) {
+ AccumuloClient client = env.getAccumuloClient();
+ clientProps = configDelegationToken(clientProps, client);
+ client.close();
+ }
+ return clientProps;
+ }
+
+ public static Properties configDelegationToken(Properties clientProps, AccumuloClient client)
+ throws AccumuloException, AccumuloSecurityException {
+ if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) {
+ DelegationToken dt = client.securityOperations().getDelegationToken(null);
+ clientProps = Accumulo.newClientProperties().from(clientProps)
+ .as(ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), dt).build();
+ }
+ return clientProps;
+ }
+}
diff --git a/src/main/java/org/apache/accumulo/testing/TestEnv.java b/src/main/java/org/apache/accumulo/testing/TestEnv.java
index d395ad4..73ee307 100644
--- a/src/main/java/org/apache/accumulo/testing/TestEnv.java
+++ b/src/main/java/org/apache/accumulo/testing/TestEnv.java
@@ -171,6 +171,7 @@ public class TestEnv implements AutoCloseable {
*/
public synchronized AccumuloClient getAccumuloClient() {
if (client == null) {
+ KerberosHelper.saslLogin(clientProps, getHadoopConfiguration());
client = Accumulo.newClient().from(clientProps).build();
}
return client;
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
index 99e2fdf..a9fb439 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.testing.continuous;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
+import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
@@ -30,6 +31,7 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
import org.apache.accumulo.testing.TestProps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -123,14 +125,16 @@ public class ContinuousMoru extends Configured implements Tool {
Set<Range> ranges = env.getAccumuloClient().tableOperations()
.splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps);
- AccumuloInputFormat.configure().clientProperties(env.getClientProps())
+ Properties clientProps = KerberosHelper.configDelegationToken(env);
+
+ AccumuloInputFormat.configure().clientProperties(clientProps)
.table(env.getAccumuloTableName()).ranges(ranges).autoAdjustRanges(false).store(job);
job.setMapperClass(CMapper.class);
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
+ AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true)
.defaultTable(env.getAccumuloTableName()).store(job);
Configuration conf = job.getConfiguration();
diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
index 5db4350..fd05c7e 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Properties;
import java.util.Random;
import java.util.Set;
@@ -30,6 +31,7 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
import org.apache.accumulo.testing.TestProps;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -173,8 +175,10 @@ public class ContinuousVerify extends Configured implements Tool {
table = tableName;
}
- AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(table)
- .ranges(ranges).autoAdjustRanges(false).offlineScan(scanOffline).store(job);
+ Properties clientProps = KerberosHelper.configDelegationToken(env.getClientProps(), client);
+
+ AccumuloInputFormat.configure().clientProperties(clientProps).table(table).ranges(ranges)
+ .autoAdjustRanges(false).offlineScan(scanOffline).store(job);
job.setMapperClass(CMapper.class);
job.setMapOutputKeyClass(LongWritable.class);
diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
index 45e24f0..e7363ae 100644
--- a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
+++ b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java
@@ -28,6 +28,7 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
import org.apache.accumulo.testing.TestEnv;
import org.apache.accumulo.testing.TestProps;
import org.apache.hadoop.conf.Configured;
@@ -76,10 +77,12 @@ public class RowHash extends Configured implements Tool {
String inputTable = props.getProperty(TestProps.ROWHASH_INPUT_TABLE);
String outputTable = props.getProperty(TestProps.ROWHASH_OUTPUT_TABLE);
- AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(inputTable)
+ Properties clientProps = KerberosHelper.configDelegationToken(env);
+
+ AccumuloInputFormat.configure().clientProperties(clientProps).table(inputTable)
.fetchColumns(cols).store(job);
- AccumuloOutputFormat.configure().clientProperties(env.getClientProps())
- .defaultTable(outputTable).createTables(true).store(job);
+ AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(outputTable)
+ .createTables(true).store(job);
job.getConfiguration().set("mapreduce.job.classloader", "true");
job.setMapperClass(HashDataMapper.class);
diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
index 1961b93..8dc915b 100644
--- a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java
@@ -30,6 +30,7 @@ import java.util.Random;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
import org.apache.accumulo.testing.TestEnv;
import org.apache.accumulo.testing.TestProps;
import org.apache.hadoop.conf.Configuration;
@@ -376,7 +377,9 @@ public class TeraSortIngest extends Configured implements Tool {
String tableName = testProps.getProperty(TestProps.TERASORT_TABLE);
Objects.requireNonNull(tableName);
- AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
+ Properties clientProps = KerberosHelper.configDelegationToken(env);
+
+ AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true)
.defaultTable(tableName).store(job);
Configuration conf = job.getConfiguration();
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java
index 54866f2..44167a0 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java
@@ -26,6 +26,8 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
@@ -56,10 +58,13 @@ public class CopyTool extends Configured implements Tool {
return 1;
}
- Properties props = Accumulo.newClientProperties().from(args[0]).build();
job.setInputFormatClass(AccumuloInputFormat.class);
- AccumuloInputFormat.configure().clientProperties(props).table(args[1])
+ Properties clientProps = Accumulo.newClientProperties().from(args[0]).build();
+ KerberosHelper.saslLogin(clientProps, new Configuration(false));
+ clientProps = KerberosHelper.configDelegationToken(clientProps);
+
+ AccumuloInputFormat.configure().clientProperties(clientProps).table(args[1])
.auths(Authorizations.EMPTY).store(job);
job.setMapperClass(SeqMapClass.class);
@@ -69,7 +74,7 @@ public class CopyTool extends Configured implements Tool {
job.getConfiguration().set("mapreduce.job.classloader", "true");
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.configure().clientProperties(props).createTables(true)
+ AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true)
.defaultTable(args[2]).store(job);
job.waitForCompletion(true);
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java b/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java
index f1e560c..454bd83 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java
@@ -26,6 +26,8 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
@@ -89,10 +91,12 @@ public class MapRedVerifyTool extends Configured implements Tool {
return 1;
}
- Properties props = Accumulo.newClientProperties().from(args[0]).build();
+ Properties clientProps = Accumulo.newClientProperties().from(args[0]).build();
+ KerberosHelper.saslLogin(clientProps, new Configuration(false));
+ clientProps = KerberosHelper.configDelegationToken(clientProps);
- AccumuloInputFormat.configure().clientProperties(props).table(args[1]).store(job);
- AccumuloOutputFormat.configure().clientProperties(props).defaultTable(args[2])
+ AccumuloInputFormat.configure().clientProperties(clientProps).table(args[1]).store(job);
+ AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(args[2])
.createTables(true).store(job);
job.setInputFormatClass(AccumuloInputFormat.class);