You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2014/08/26 00:11:28 UTC
[2/8] git commit: SLIDER-254 bump Accumulo version to 1.6.1-SNAPSHOT
and configure app package to use CredentialProvider
SLIDER-254 bump Accumulo version to 1.6.1-SNAPSHOT and configure app package to use CredentialProvider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6e0e7261
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6e0e7261
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6e0e7261
Branch: refs/heads/develop
Commit: 6e0e72613085fc242f1602ca455a30403f267511
Parents: 02e9f83
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Aug 4 15:11:15 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Aug 4 15:11:15 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 6 ++--
.../slider/accumulo/CustomAuthenticator.java | 31 +++++++++++---------
.../funtest/accumulo/AccumuloBasicIT.groovy | 23 +++++++++------
pom.xml | 2 +-
.../org/apache/slider/client/SliderClient.java | 12 ++------
5 files changed, 37 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/app-packages/accumulo/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
index 83d35c7..c024c97 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -27,13 +27,11 @@
"site.global.monitor_protocol": "http",
"site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
- "site.accumulo-site.instance.secret": "DEFAULT",
"site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
- "site.accumulo-site.instance.security.credential.provider": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
- "site.accumulo-site.trace.token.property.password": "secret",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
"site.accumulo-site.tserver.walog.max.size": "100M",
@@ -46,7 +44,7 @@
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
- "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root", "instance.secret"]
+ "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"]
},
"components": {
"ACCUMULO_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
index 45df8cb..dd18a8e 100644
--- a/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
+++ b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
@@ -38,8 +38,8 @@ import java.util.List;
import java.util.Set;
public final class CustomAuthenticator implements Authenticator {
- private static final String CREDENTIAL_PROVIDER_PROPERTY =
- "instance.security.credential.provider";
+ public static final String ROOT_INITIAL_PASSWORD_PROPERTY =
+ "root.initial.password";
private static ZKAuthenticator zkAuthenticator = null;
public CustomAuthenticator() {
@@ -55,14 +55,15 @@ public final class CustomAuthenticator implements Authenticator {
public void initializeSecurity(TCredentials credentials, String principal,
byte[] token) throws AccumuloSecurityException {
char[] pass = null;
- SiteConfiguration siteconf = SiteConfiguration.getInstance
- (DefaultConfiguration.getInstance());
- String jksFile = siteconf.getAllPropertiesWithPrefix(
- Property.INSTANCE_PREFIX).get(CREDENTIAL_PROVIDER_PROPERTY);
+ SiteConfiguration siteconf = SiteConfiguration.getInstance(
+ DefaultConfiguration.getInstance());
+ String jksFile = siteconf.get(
+ Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS);
if (jksFile == null) {
- throw new RuntimeException("instance.security.credential.provider not " +
- "specified in accumulo-site.xml");
+ throw new RuntimeException(
+ Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS +
+ " not specified in accumulo-site.xml");
}
try {
Configuration conf = new Configuration();
@@ -73,25 +74,27 @@ public final class CustomAuthenticator implements Authenticator {
if (providers != null) {
for (CredentialProvider provider : providers) {
try {
- CredentialEntry entry = provider.getCredentialEntry(principal);
+ CredentialEntry entry = provider.getCredentialEntry(
+ ROOT_INITIAL_PASSWORD_PROPERTY);
if (entry != null) {
pass = entry.getCredential();
break;
}
}
catch (IOException ioe) {
- throw new IOException("Can't get key " + principal + " from " +
+ throw new IOException("Can't get key " +
+ ROOT_INITIAL_PASSWORD_PROPERTY + " from " +
provider.getClass().getName() + ", " + jksFile, ioe);
}
}
}
} catch (IOException ioe) {
- throw new RuntimeException("Can't get key " + principal + " from " +
- jksFile, ioe);
+ throw new RuntimeException("Can't get key " +
+ ROOT_INITIAL_PASSWORD_PROPERTY + " from " + jksFile, ioe);
}
if (pass == null) {
- throw new RuntimeException("Can't get key " + principal + " from " +
- jksFile);
+ throw new RuntimeException("Can't get key " +
+ ROOT_INITIAL_PASSWORD_PROPERTY + " from " + jksFile);
}
zkAuthenticator.initializeSecurity(credentials, principal,
new String(pass).getBytes(StandardCharsets.UTF_8));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index b8e0841..63bb091 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -17,12 +17,14 @@
package org.apache.slider.funtest.accumulo
import groovy.util.logging.Slf4j
+import org.apache.accumulo.core.conf.Property
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.security.ProviderUtils
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.security.alias.CredentialProvider
import org.apache.hadoop.security.alias.CredentialProviderFactory
+import org.apache.slider.accumulo.CustomAuthenticator
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
@@ -38,8 +40,8 @@ import org.junit.Test
@Slf4j
class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
- protected static final String PROVIDER =
- "site.accumulo-site.instance.security.credential.provider"
+ protected static final String PROVIDER_PROPERTY = "site.accumulo-site." +
+ Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS
protected ConfTree tree
@Before
@@ -50,12 +52,11 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
"skipping creation of credentials"
SliderClient.replaceTokens(tree, UserGroupInformation.getCurrentUser()
.getShortUserName(), getClusterName())
- String jks = tree.global.get(PROVIDER)
+ String jks = tree.global.get(PROVIDER_PROPERTY)
def keys = tree.credentials.get(jks)
- assert keys!=null, "jks specified in $PROVIDER wasn't requested in " +
- "credentials"
- assert keys.size()==2, "test expects root and instance.secret to be " +
- "requested"
+ assert keys!=null, "jks specified in $PROVIDER_PROPERTY wasn't requested " +
+ "in credentials"
+ assert keys.size()==3, "test expects 3 passwords to be requested"
Path jksPath = ProviderUtils.unnestUri(new URI(jks))
if (clusterFS.exists(jksPath)) {
clusterFS.delete(jksPath, false)
@@ -64,8 +65,12 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, jks)
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0)
- provider.createCredentialEntry(USER, PASSWORD.toCharArray())
- provider.createCredentialEntry("instance.secret", INSTANCE_SECRET.toCharArray())
+ provider.createCredentialEntry(
+ CustomAuthenticator.ROOT_INITIAL_PASSWORD_PROPERTY, PASSWORD.toCharArray())
+ provider.createCredentialEntry(Property.INSTANCE_SECRET.toString(),
+ INSTANCE_SECRET.toCharArray())
+ provider.createCredentialEntry(Property.TRACE_TOKEN_PROPERTY_PREFIX
+ .toString() + "password", PASSWORD.toCharArray())
provider.flush()
assert clusterFS.exists(jksPath), "jks $jks not created"
log.info("Created credential provider $jks for test")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7e2063c..ab6e9a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -120,7 +120,7 @@
<hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
<hbase.version>0.98.4-hadoop2</hbase.version>
- <accumulo.version>1.6.0</accumulo.version>
+ <accumulo.version>1.6.1-SNAPSHOT</accumulo.version>
<!--
artifact versions
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ed5b74c..99acfbc 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.CredentialShell;
-import org.apache.hadoop.security.alias.CredentialShell.PasswordReader;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -110,27 +109,20 @@ import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.services.curator.CuratorServiceInstance;
import org.apache.slider.server.services.registry.SliderRegistryService;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
-
-import static org.apache.slider.common.params.SliderActions.*;
-
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
-import java.io.Console;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -141,6 +133,8 @@ import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
+import static org.apache.slider.common.params.SliderActions.*;
+
/**
* Client service for Slider
*/
@@ -497,7 +491,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
continue;
}
String[] csarg = new String[]{
- "create", alias, "--provider", provider};
+ "create", alias, "-provider", provider};
log.info("Creating credentials for {} in {}", alias, provider);
credentialShell.run(csarg);
}