You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2015/02/27 01:02:07 UTC
accumulo git commit: ACCUMULO-3629 Use ClientConf from job,
not default.
Repository: accumulo
Updated Branches:
refs/heads/master 4b68322ff -> bac5bc5b2
ACCUMULO-3629 Use ClientConf from job, not default.
Noticed as a part of ACCUMULO-3599. AccumuloInputFormatIT was
failing when run with kerberos on. It hung indefinitely because
it constructed a default client conf which did not have
instance.rpc.sasl.enabled=true.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bac5bc5b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bac5bc5b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bac5bc5b
Branch: refs/heads/master
Commit: bac5bc5b29ee5412d687eaa18899de38c489b310
Parents: 4b68322
Author: Josh Elser <el...@apache.org>
Authored: Thu Feb 26 18:39:44 2015 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Feb 26 19:01:56 2015 -0500
----------------------------------------------------------------------
.../core/client/mapred/AbstractInputFormat.java | 14 +++++++-
.../client/mapreduce/AbstractInputFormat.java | 14 +++++++-
.../mapreduce/lib/impl/ConfiguratorBase.java | 38 +++++++++++++++-----
3 files changed, 56 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
index 0ce05d7..ebfbe1b 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
@@ -305,6 +305,18 @@ public abstract class AbstractInputFormat<K,V> implements InputFormat<K,V> {
return InputConfigurator.getTabletLocator(CLASS, job, tableId);
}
+ /**
+ * Fetch the client configuration from the job.
+ *
+ * @param job
+ * The job
+ * @return The client configuration for the job
+ * @since 1.7.0
+ */
+ protected static ClientConfiguration getClientConfiguration(JobConf job) {
+ return InputConfigurator.getClientConfiguration(CLASS, job);
+ }
+
// InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
/**
* Check whether a configuration is fully configured to be used with an Accumulo {@link InputFormat}.
@@ -578,7 +590,7 @@ public abstract class AbstractInputFormat<K,V> implements InputFormat<K,V> {
tl.invalidateCache();
ClientContext context = new ClientContext(getInstance(job), new Credentials(getPrincipal(job), getAuthenticationToken(job)),
- ClientConfiguration.loadDefault());
+ getClientConfiguration(job));
while (!tl.binRanges(context, ranges, binnedRanges).isEmpty()) {
if (!(instance instanceof MockInstance)) {
if (!Tables.exists(instance, tableId))
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
index e1b35b2..255e555 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
@@ -388,6 +388,18 @@ public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V> {
}
/**
+ * Construct the {@link ClientConfiguration} given the provided context.
+ *
+ * @param context
+ * The Job
+ * @return The ClientConfiguration
+ * @since 1.7.0
+ */
+ protected static ClientConfiguration getClientConfiguration(JobContext context) {
+ return InputConfigurator.getClientConfiguration(CLASS, context.getConfiguration());
+ }
+
+ /**
* An abstract base class to be used to create {@link org.apache.hadoop.mapreduce.RecordReader} instances that convert from Accumulo
* {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value} pairs to the user's K/V types.
*
@@ -634,7 +646,7 @@ public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V> {
tl.invalidateCache();
ClientContext clientContext = new ClientContext(getInstance(context), new Credentials(getPrincipal(context), getAuthenticationToken(context)),
- ClientConfiguration.loadDefault());
+ getClientConfiguration(context));
while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) {
if (!(instance instanceof MockInstance)) {
if (!Tables.exists(instance, tableId))
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bac5bc5b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index 3b5fa3a..b32d1b3 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@ -358,14 +358,7 @@ public class ConfiguratorBase {
if ("MockInstance".equals(instanceType))
return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
else if ("ZooKeeperInstance".equals(instanceType)) {
- String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
- if (clientConfigString == null) {
- String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
- String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
- return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
- } else {
- return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
- }
+ return new ZooKeeperInstance(getClientConfiguration(implementingClass, conf));
} else if (instanceType.isEmpty())
throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
else
@@ -373,6 +366,35 @@ public class ConfiguratorBase {
}
/**
+ * Obtain a {@link ClientConfiguration} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ *
+ * @return A {@link ClientConfiguration}
+ * @since 1.7.0
+ */
+ public static ClientConfiguration getClientConfiguration(Class<?> implementingClass, Configuration conf) {
+ String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
+ if (null != clientConfigString) {
+ return ClientConfiguration.deserialize(clientConfigString);
+ }
+
+ String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
+ String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
+ ClientConfiguration clientConf = ClientConfiguration.loadDefault();
+ if (null != instanceName) {
+ clientConf.withInstance(instanceName);
+ }
+ if (null != zookeepers) {
+ clientConf.withZkHosts(zookeepers);
+ }
+ return clientConf;
+ }
+
+ /**
* Sets the log level for this job.
*
* @param implementingClass