You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by cj...@apache.org on 2013/09/29 05:29:01 UTC
[5/5] git commit: Removing deprecation for now until we have some
discussions. Updating/adding comments. ACCUMULO-391
Removing deprecation for now until we have some discussions. Updating/adding comments. ACCUMULO-391
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/75cccccb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/75cccccb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/75cccccb
Branch: refs/heads/ACCUMULO-391
Commit: 75cccccb3e73d86aab77728a2813d8ade79efbe7
Parents: d340d82
Author: Corey J. Nolet <cj...@gmail.com>
Authored: Sat Sep 28 23:23:48 2013 -0400
Committer: Corey J. Nolet <cj...@gmail.com>
Committed: Sat Sep 28 23:23:48 2013 -0400
----------------------------------------------------------------------
.../core/client/mapred/InputFormatBase.java | 2 +
.../core/client/mapreduce/InputFormatBase.java | 71 +++++++++++++-------
.../mapreduce/lib/util/InputConfigurator.java | 40 +++--------
.../mapreduce/AccumuloInputFormatTest.java | 15 ++---
4 files changed, 64 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/75cccccb/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
index fb1b193..15c2dd2 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
@@ -314,6 +314,7 @@ public abstract class InputFormatBase<K,V> implements InputFormat<K,V> {
* the ranges that will be mapped over
* @since 1.5.0
*/
+ @Deprecated
public static void setRanges(JobConf job, Collection<Range> ranges) {
InputConfigurator.setRanges(CLASS, job, ranges);
}
@@ -329,6 +330,7 @@ public abstract class InputFormatBase<K,V> implements InputFormat<K,V> {
* @since 1.5.0
* @see #setRanges(JobConf, Collection)
*/
+ @Deprecated
protected static List<Range> getRanges(JobConf job) throws IOException {
return InputConfigurator.getRanges(CLASS,job);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/75cccccb/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index d5208b4..5392459 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -157,7 +157,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @see #setConnectorInfo(Job, String, AuthenticationToken)
*/
protected static String getPrincipal(JobContext context) {
- return InputConfigurator.getPrincipal(CLASS, getConfiguration(context));
+ return InputConfigurator.getPrincipal(CLASS,getConfiguration(context));
}
/**
@@ -170,7 +170,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @see #setInputTableName(Job, String)
*/
protected static String getInputTableName(JobContext context) {
- return InputConfigurator.getInputTableName(CLASS, getConfiguration(context));
+ return InputConfigurator.getInputTableName(CLASS,getConfiguration(context));
}
/**
@@ -206,7 +206,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @see #setConnectorInfo(Job, String, String)
*/
protected static AuthenticationToken getAuthenticationToken(JobContext context) {
- return InputConfigurator.getAuthenticationToken(CLASS, getConfiguration(context));
+ return InputConfigurator.getAuthenticationToken(CLASS,getConfiguration(context));
}
/**
@@ -234,7 +234,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @since 1.5.0
*/
public static void setMockInstance(Job job, String instanceName) {
- InputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
+ InputConfigurator.setMockInstance(CLASS,job.getConfiguration(),instanceName);
}
/**
@@ -286,7 +286,6 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* the table to use when the tablename is null in the write call
* @since 1.5.0
*/
- @Deprecated
public static void setInputTableName(Job job, String tableName) {
InputConfigurator.setInputTableName(CLASS, job.getConfiguration(), tableName);
}
@@ -301,7 +300,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @since 1.5.0
*/
public static void setScanAuthorizations(Job job, Authorizations auths) {
- InputConfigurator.setScanAuthorizations(CLASS, job.getConfiguration(), auths);
+ InputConfigurator.setScanAuthorizations(CLASS,job.getConfiguration(),auths);
}
/**
@@ -327,7 +326,6 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* the ranges that will be mapped over
* @since 1.5.0
*/
- @Deprecated
public static void setRanges(Job job, Collection<Range> ranges) {
InputConfigurator.setRanges(CLASS, job.getConfiguration(), ranges);
}
@@ -342,7 +340,6 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @see #setRanges(Job, Collection)
* @see #setRanges(org.apache.hadoop.mapreduce.Job, java.util.Collection)
*/
- @Deprecated
protected static List<Range> getRanges(JobContext context) throws IOException {
return InputConfigurator.getRanges(CLASS, getConfiguration(context));
}
@@ -357,7 +354,6 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* selected. An empty set is the default and is equivalent to scanning the all columns.
* @since 1.5.0
*/
- @Deprecated
public static void fetchColumns(Job job, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
InputConfigurator.fetchColumns(CLASS, job.getConfiguration(), columnFamilyColumnQualifierPairs);
}
@@ -384,14 +380,12 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* the configuration of the iterator
* @since 1.5.0
*/
- @Deprecated
public static void addIterator(Job job, IteratorSetting cfg) {
InputConfigurator.addIterator(CLASS, job.getConfiguration(), cfg);
}
/**
- * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration. This will only return iterators that have not been set
- * for a specific table.
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration.
*
* @param context
* the Hadoop context for the configured job
@@ -399,7 +393,6 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @since 1.5.0
* @see #addIterator(Job, IteratorSetting)
*/
- @Deprecated
protected static List<IteratorSetting> getIterators(JobContext context) {
return InputConfigurator.getIterators(CLASS, getConfiguration(context));
}
@@ -418,7 +411,6 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @see #setRanges(Job, Collection)
* @since 1.5.0
*/
- @Deprecated
public static void setAutoAdjustRanges(Job job, boolean enableFeature) {
InputConfigurator.setAutoAdjustRanges(CLASS, job.getConfiguration(), enableFeature);
}
@@ -432,21 +424,52 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* @since 1.5.0
* @see #setAutoAdjustRanges(Job, boolean)
*/
- @Deprecated
protected static boolean getAutoAdjustRanges(JobContext context) {
return InputConfigurator.getAutoAdjustRanges(CLASS, getConfiguration(context));
}
-
- protected static void setTableQueryConfigurations(JobContext job, TableQueryConfig... configs) {
+
+ /**
+ * Sets the {@link TableQueryConfig} objects on the given Hadoop configuration
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param configs
+ * the table query configs to be set on the configuration.
+ */
+ public static void setTableQueryConfigs(JobContext job, TableQueryConfig... configs) {
checkNotNull(configs);
InputConfigurator.setTableQueryConfigs(CLASS, getConfiguration(job), configs);
}
-
- public static List<TableQueryConfig> getTableQueryConfigurations(JobContext job) {
+
+ /**
+ * Fetches all {@link TableQueryConfig}s that have been set on the given Hadoop configuration.
+ *
+ * <p>
+ * Note this also returns the {@link TableQueryConfig} representing the table configurations set through the single
+ * table input methods ({@link #setInputTableName(org.apache.hadoop.mapreduce.Job, String)},
+ * {@link #setRanges(org.apache.hadoop.mapreduce.Job, java.util.Collection)},
+ * {@link #fetchColumns(org.apache.hadoop.mapreduce.Job, java.util.Collection)},
+ * {@link #addIterator(org.apache.hadoop.mapreduce.Job, org.apache.accumulo.core.client.IteratorSetting)}, etc...)
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @return
+ */
+ public static List<TableQueryConfig> getTableQueryConfigs(JobContext job) {
return InputConfigurator.getTableQueryConfigs(CLASS, getConfiguration(job));
}
-
- protected static TableQueryConfig getTableQueryConfiguration(JobContext job, String tableName) {
+
+ /**
+ * Fetches a {@link TableQueryConfig} that has been set on the configuration for a specific table.
+ *
+ * <p>
+ * null is returned in the event that the table doesn't exist.
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table name for which to grab the config object
+ * @return the {@link TableQueryConfig} for the given table
+ */
+ public static TableQueryConfig getTableQueryConfig(JobContext job, String tableName) {
return InputConfigurator.getTableQueryConfigs(CLASS, getConfiguration(job), tableName);
}
@@ -615,7 +638,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
* the table name for which to set up the iterators
*/
protected void setupIterators(TaskAttemptContext context, Scanner scanner, String tableName) {
- TableQueryConfig config = getTableQueryConfiguration(context, tableName);
+ TableQueryConfig config = getTableQueryConfig(context,tableName);
List<IteratorSetting> iterators = config.getIterators();
for (IteratorSetting iterator : iterators)
scanner.addScanIterator(iterator);
@@ -633,7 +656,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
Instance instance = getInstance(attempt);
String principal = getPrincipal(attempt);
- TableQueryConfig tableConfig = getTableQueryConfiguration(attempt, split.getTableName());
+ TableQueryConfig tableConfig = getTableQueryConfig(attempt,split.getTableName());
// in case the table name changed, we can still use the previous name for terms of configuration,
// but for the scanner, we'll need to reference the new table name.
@@ -828,7 +851,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
validateOptions(conf);
LinkedList<InputSplit> splits = new LinkedList<InputSplit>();
- List<TableQueryConfig> tableConfigs = getTableQueryConfigurations(conf);
+ List<TableQueryConfig> tableConfigs = getTableQueryConfigs(conf);
for (TableQueryConfig tableConfig : tableConfigs) {
boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/75cccccb/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
index 5e02177..82df3a1 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
@@ -89,9 +89,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @param tableName
* the table to use when the tablename is null in the write call
* @since 1.5.0
- * @deprecated since 1.6.0
*/
- @Deprecated
public static void setInputTableName(Class<?> implementingClass, Configuration conf, String tableName) {
notNull(tableName);
conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE), tableName);
@@ -105,9 +103,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @param conf
* the Hadoop configuration object to configure
* @since 1.5.0
- * @deprecated since 1.6.0
*/
- @Deprecated
public static String getInputTableName(Class<?> implementingClass, Configuration conf) {
return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE));
}
@@ -156,9 +152,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @throws IllegalArgumentException
* if the ranges cannot be encoded into base 64
* @since 1.5.0
- * @deprecated since 1.6.0
*/
- @Deprecated
public static void setRanges(Class<?> implementingClass, Configuration conf, Collection<Range> ranges) {
notNull(ranges);
@@ -186,10 +180,8 @@ public class InputConfigurator extends ConfiguratorBase {
* @throws IOException
* if the ranges have been encoded improperly
* @since 1.6.0
- * @deprecated since 1.6.0
* @see #setRanges(Class, Configuration, Collection)
*/
- @Deprecated
public static List<Range> getRanges(Class<?> implementingClass, Configuration conf) throws IOException {
Collection<String> encodedRanges = conf.getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES));
@@ -254,7 +246,6 @@ public class InputConfigurator extends ConfiguratorBase {
* @since 1.5.0
* @deprecated since 1.6.0
*/
- @Deprecated
public static void fetchColumns(Class<?> implementingClass, Configuration conf, Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
notNull(columnFamilyColumnQualifierPairs);
ArrayList<String> columnStrings = new ArrayList<String>();
@@ -306,9 +297,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @throws IllegalArgumentException
* if the iterator can't be serialized into the configuration
* @since 1.5.0
- * @deprecated since 1.6.0
*/
- @Deprecated
public static void addIterator(Class<?> implementingClass, Configuration conf, IteratorSetting cfg) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
String newIter;
@@ -348,9 +337,7 @@ public class InputConfigurator extends ConfiguratorBase {
* the feature is enabled if true, disabled otherwise
* @see #setRanges(Class, Configuration, Collection)
* @since 1.5.0
- * @deprecated since 1.6.0
*/
- @Deprecated
public static void setAutoAdjustRanges(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature);
}
@@ -365,9 +352,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @return false if the feature is disabled, true otherwise
* @since 1.5.0
* @see #setAutoAdjustRanges(Class, Configuration, boolean)
- * @deprecated since 1.6.0
*/
- @Deprecated
public static Boolean getAutoAdjustRanges(Class<?> implementingClass, Configuration conf) {
return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true);
}
@@ -385,9 +370,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @param enableFeature
* the feature is enabled if true, disabled otherwise
* @since 1.5.0
- * @deprecated since 1.6.0
*/
- @Deprecated
public static void setScanIsolation(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature);
}
@@ -402,9 +385,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @return true if the feature is enabled, false otherwise
* @since 1.5.0
* @see #setScanIsolation(Class, Configuration, boolean)
- * @deprecated since 1.6.0
*/
- @Deprecated
public static Boolean isIsolated(Class<?> implementingClass, Configuration conf) {
return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false);
}
@@ -423,9 +404,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @param enableFeature
* the feature is enabled if true, disabled otherwise
* @since 1.5.0
- * @deprecated since 1.6.0
*/
- @Deprecated
public static void setLocalIterators(Class<?> implementingClass, Configuration conf, boolean enableFeature) {
conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature);
}
@@ -440,9 +419,7 @@ public class InputConfigurator extends ConfiguratorBase {
* @return true if the feature is enabled, false otherwise
* @since 1.5.0
* @see #setLocalIterators(Class, Configuration, boolean)
- * @deprecated since 1.6.0
*/
- @Deprecated
public static Boolean usesLocalIterators(Class<?> implementingClass, Configuration conf) {
return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false);
}
@@ -615,9 +592,6 @@ public class InputConfigurator extends ConfiguratorBase {
}
}
}
-
- // TODO: Check for the "default table case"
-
} catch (AccumuloException e) {
throw new IOException(e);
} catch (AccumuloSecurityException e) {
@@ -627,7 +601,17 @@ public class InputConfigurator extends ConfiguratorBase {
}
}
-
+ /**
+ * Returns the {@link TableQueryConfig} for the configuration based on the properties set using the single-table
+ * input methods.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop instance for which to retrieve the configuration
+ * @return the config object built from the single input table properties set on the job
+ * @throws IOException
+ */
protected static TableQueryConfig getDefaultTableConfig(Class<?> implementingClass, Configuration conf) throws IOException {
String tableName = getInputTableName(implementingClass, conf);
if(tableName != null) {
@@ -650,6 +634,4 @@ public class InputConfigurator extends ConfiguratorBase {
}
return null;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/75cccccb/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index 1b4aa26..f9dd9ec 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@ -25,15 +25,9 @@ import static org.junit.Assert.fail;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
@@ -47,7 +41,6 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.RegExFilter;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
-import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Pair;
import org.apache.commons.codec.binary.Base64;
@@ -264,7 +257,7 @@ public class AccumuloInputFormatTest {
TableQueryConfig tableConfig1 = new TableQueryConfig(table1);
TableQueryConfig tableConfig2 = new TableQueryConfig(table2);
- AccumuloInputFormat.setTableQueryConfigurations(job, tableConfig1, tableConfig2);
+ AccumuloInputFormat.setTableQueryConfigs(job,tableConfig1,tableConfig2);
AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
job.setMapperClass(TestMapper.class);
@@ -326,9 +319,9 @@ public class AccumuloInputFormatTest {
.setColumns(Collections.singleton(new Pair<Text,Text>(new Text("CF1"), new Text("CQ1"))))
.setIterators(Collections.singletonList(new IteratorSetting(50, "iter1", "iterclass1")));
- AccumuloInputFormat.setTableQueryConfigurations(job, table1, table2);
+ AccumuloInputFormat.setTableQueryConfigs(job,table1,table2);
- assertEquals(table1, AccumuloInputFormat.getTableQueryConfiguration(job, TEST_TABLE_1));
- assertEquals(table2, AccumuloInputFormat.getTableQueryConfiguration(job, TEST_TABLE_2));
+ assertEquals(table1, AccumuloInputFormat.getTableQueryConfig(job,TEST_TABLE_1));
+ assertEquals(table2, AccumuloInputFormat.getTableQueryConfig(job,TEST_TABLE_2));
}
}