You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by GitBox <gi...@apache.org> on 2018/10/25 22:39:47 UTC
[GitHub] milleruntime closed pull request #712: Cleanup MapReduce package
and API
milleruntime closed pull request #712: Cleanup MapReduce package and API
URL: https://github.com/apache/accumulo/pull/712
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/assemble/pom.xml b/assemble/pom.xml
index b56643da59..27062f54bb 100644
--- a/assemble/pom.xml
+++ b/assemble/pom.xml
@@ -140,17 +140,17 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-client-mapreduce</artifactId>
+ <artifactId>accumulo-core</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
+ <artifactId>accumulo-gc</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-gc</artifactId>
+ <artifactId>accumulo-hadoop-mapreduce</artifactId>
<optional>true</optional>
</dependency>
<dependency>
diff --git a/core/pom.xml b/core/pom.xml
index 057ae12358..ff059f608d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -198,10 +198,15 @@
<allow>org[.]apache[.]hadoop[.]conf[.]Configuration</allow>
<allow>org[.]apache[.]hadoop[.]fs[.](FileSystem|Path)</allow>
<allow>org[.]apache[.]hadoop[.]io[.](Text|Writable|WritableComparable|WritableComparator)</allow>
+ <allow>org[.]apache[.]hadoop[.]mapred[.](JobConf|RecordReader|InputSplit|RecordWriter|Reporter)</allow>
+ <allow>org[.]apache[.]hadoop[.]mapreduce[.](Job|JobContext|RecordReader|InputSplit|TaskAttemptContext|RecordWriter|OutputCommitter|TaskInputOutputContext)</allow>
+ <allow>org[.]apache[.]hadoop[.]util[.]Progressable</allow>
<!--ugghhh-->
<allow>org[.]apache[.]log4j[.](Level|Logger)</allow>
<!-- allow javax security exceptions for Authentication tokens -->
<allow>javax[.]security[.]auth[.]DestroyFailedException</allow>
+ <!-- allow questionable Hadoop exceptions for mapreduce -->
+ <allow>org[.]apache[.]hadoop[.]mapred[.](FileAlreadyExistsException|InvalidJobConfException)</allow>
<!-- allow lexicoders to throw iterator exceptions -->
<allow>org[.]apache[.]accumulo[.]core[.]iterators[.]ValueFormatException</allow>
</allows>
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/AbstractInputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/RangeInputSplit.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/RangeInputSplit.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/RangeInputSplit.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/RangeInputSplit.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/impl/BatchInputSplit.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/impl/BatchInputSplit.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/impl/BatchInputSplit.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapred/impl/BatchInputSplit.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloMultiTableInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloMultiTableInputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloMultiTableInputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloMultiTableInputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormat.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormat.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormat.java
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
new file mode 100644
index 0000000000..bb6ef71e48
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.InputConfigurator;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * This abstract {@link InputFormat} class allows MapReduce jobs to use Accumulo as the source of
+ * K,V pairs.
+ * <p>
+ * Subclasses must implement a {@link #createRecordReader(InputSplit, TaskAttemptContext)} to
+ * provide a {@link RecordReader} for K,V.
+ * <p>
+ * A static base class, RecordReaderBase, is provided to retrieve Accumulo {@link Key}/{@link Value}
+ * pairs, but one must implement its {@link RecordReaderBase#nextKeyValue()} to transform them to
+ * the desired generic types K,V.
+ * <p>
+ * See {@link AccumuloInputFormat} for an example implementation.
+ */
+public abstract class InputFormatBase<K,V> extends AbstractInputFormat<K,V> {
+
+ /**
+ * Gets the table name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the table name
+ * @since 1.5.0
+ * @see #setInputTableName(Job, String)
+ */
+ protected static String getInputTableName(JobContext context) {
+ return InputConfigurator.getInputTableName(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setInputTableName(Job job, String tableName) {
+ InputConfigurator.setInputTableName(CLASS, job.getConfiguration(), tableName);
+ }
+
+ /**
+ * Sets the input ranges to scan for the single input table associated with this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param ranges
+ * the ranges that will be mapped over
+ * @see TableOperations#splitRangeByTablets(String, Range, int)
+ * @since 1.5.0
+ */
+ public static void setRanges(Job job, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, job.getConfiguration(), ranges);
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the ranges
+ * @since 1.5.0
+ * @see #setRanges(Job, Collection)
+ */
+ protected static List<Range> getRanges(JobContext context) throws IOException {
+ return InputConfigurator.getRanges(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job for the default input table.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If
+ * the column qualifier is null, the entire column family is selected. An empty set is
+ * the default and is equivalent to scanning the all columns.
+ * @since 1.5.0
+ */
+ public static void fetchColumns(Job job,
+ Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, job.getConfiguration(), columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return a set of columns
+ * @since 1.5.0
+ * @see #fetchColumns(Job, Collection)
+ */
+ protected static Set<Pair<Text,Text>> getFetchedColumns(JobContext context) {
+ return InputConfigurator.getFetchedColumns(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Encode an iterator on the single input table for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param cfg
+ * the configuration of the iterator
+ * @since 1.5.0
+ */
+ public static void addIterator(Job job, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, job.getConfiguration(), cfg);
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this
+ * configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return a list of iterators
+ * @since 1.5.0
+ * @see #addIterator(Job, IteratorSetting)
+ */
+ protected static List<IteratorSetting> getIterators(JobContext context) {
+ return InputConfigurator.getIterators(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping
+ * ranges, then splits them to align with tablet boundaries. Disabling this feature will cause
+ * exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(Job, Collection)
+ * @since 1.5.0
+ */
+ public static void setAutoAdjustRanges(Job job, boolean enableFeature) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled. Must be enabled when
+ * {@link #setBatchScan(Job, boolean)} is true.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.5.0
+ * @see #setAutoAdjustRanges(Job, boolean)
+ */
+ protected static boolean getAutoAdjustRanges(JobContext context) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setScanIsolation(Job job, boolean enableFeature) {
+ InputConfigurator.setScanIsolation(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setScanIsolation(Job, boolean)
+ */
+ protected static boolean isIsolated(JobContext context) {
+ return InputConfigurator.isIsolated(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature
+ * will cause the iterator stack to be constructed within the Map task, rather than within the
+ * Accumulo TServer. To use this feature, all classes needed for those iterators must be available
+ * on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setLocalIterators(Job job, boolean enableFeature) {
+ InputConfigurator.setLocalIterators(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setLocalIterators(Job, boolean)
+ */
+ protected static boolean usesLocalIterators(JobContext context) {
+ return InputConfigurator.usesLocalIterators(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are
+ * scanned. This will make the map reduce job directly read the table's files. If the table is not
+ * offline, then the job will fail. If the table comes online during the map reduce job, it is
+ * likely that the job will fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in
+ * HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any
+ * iterators that are configured for the table will need to be on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as
+ * the input table for a map reduce job. If you plan to map reduce over the data many times, it
+ * may be better to the compact the table, clone it, take it offline, and use the clone for all
+ * map reduce jobs. The reason to do this is that compaction will reduce each tablet in the table
+ * to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may
+ * see better read performance. Second, it will support speculative execution better. When reading
+ * an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setOfflineTableScan(Job job, boolean enableFeature) {
+ InputConfigurator.setOfflineTableScan(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setOfflineTableScan(Job, boolean)
+ */
+ protected static boolean isOfflineScan(JobContext context) {
+ return InputConfigurator.isOfflineScan(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Controls the use of the {@link org.apache.accumulo.core.client.BatchScanner} in this job. Using
+ * this feature will group Ranges by their source tablet, producing an InputSplit per tablet
+ * rather than per Range. This batching helps to reduce overhead when querying a large number of
+ * small ranges. (ex: when doing quad-tree decomposition for spatial queries)
+ * <p>
+ * In order to achieve good locality of InputSplits this option always clips the input Ranges to
+ * tablet boundaries. This may result in one input Range contributing to several InputSplits.
+ * <p>
+ * Note: that the value of {@link #setAutoAdjustRanges(Job, boolean)} is ignored and is assumed to
+ * be true when BatchScan option is enabled.
+ * <p>
+ * This configuration is incompatible with:
+ * <ul>
+ * <li>{@link #setOfflineTableScan(org.apache.hadoop.mapreduce.Job, boolean)}</li>
+ * <li>{@link #setLocalIterators(org.apache.hadoop.mapreduce.Job, boolean)}</li>
+ * <li>{@link #setScanIsolation(org.apache.hadoop.mapreduce.Job, boolean)}</li>
+ * </ul>
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.7.0
+ */
+ public static void setBatchScan(Job job, boolean enableFeature) {
+ InputConfigurator.setBatchScan(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the {@link org.apache.accumulo.core.client.BatchScanner}
+ * feature enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @since 1.7.0
+ * @see #setBatchScan(Job, boolean)
+ */
+ public static boolean isBatchScan(JobContext context) {
+ return InputConfigurator.isBatchScan(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Causes input format to read sample data. If sample data was created using a different
+ * configuration or a tables sampler configuration changes while reading data, then the input
+ * format will throw an error.
+ *
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param samplerConfig
+ * The sampler configuration that sample must have been created with inorder for reading
+ * sample data to succeed.
+ *
+ * @since 1.8.0
+ * @see ScannerBase#setSamplerConfiguration(SamplerConfiguration)
+ */
+ public static void setSamplerConfiguration(Job job, SamplerConfiguration samplerConfig) {
+ InputConfigurator.setSamplerConfiguration(CLASS, job.getConfiguration(), samplerConfig);
+ }
+
+ /**
+ * Set these execution hints on scanners created for input splits. See
+ * {@link ScannerBase#setExecutionHints(java.util.Map)}
+ *
+ * @since 2.0.0
+ */
+ public static void setExecutionHints(JobConf job, Map<String,String> hints) {
+ InputConfigurator.setExecutionHints(CLASS, job, hints);
+ }
+
+ protected abstract static class RecordReaderBase<K,V> extends AbstractRecordReader<K,V> {
+
+ @Override
+ protected List<IteratorSetting> contextIterators(TaskAttemptContext context, String tableName) {
+ return getIterators(context);
+ }
+ }
+}
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputTableConfig.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/BatchInputSplit.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/BatchInputSplit.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/BatchInputSplit.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/BatchInputSplit.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/DelegationTokenStub.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/DelegationTokenStub.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/DelegationTokenStub.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/DelegationTokenStub.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/SplitUtils.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/SplitUtils.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/SplitUtils.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/impl/SplitUtils.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/DistributedCacheHelper.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/DistributedCacheHelper.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/DistributedCacheHelper.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/DistributedCacheHelper.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
new file mode 100644
index 0000000000..d381b6d6d4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
@@ -0,0 +1,990 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Objects.requireNonNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.ClientContext;
+import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.collect.Maps;
+
+/**
+ * @since 1.6.0
+ */
+public class InputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Scanner}.
+ *
+ * @since 1.6.0
+ */
+ public static enum ScanOpts {
+ TABLE_NAME,
+ AUTHORIZATIONS,
+ RANGES,
+ COLUMNS,
+ ITERATORS,
+ TABLE_CONFIGS,
+ SAMPLER_CONFIG,
+ CLASSLOADER_CONTEXT,
+ EXECUTION_HINTS
+ }
+
+ /**
+ * Configuration keys for various features.
+ *
+ * @since 1.6.0
+ */
+ public static enum Features {
+ AUTO_ADJUST_RANGES,
+ SCAN_ISOLATION,
+ USE_LOCAL_ITERATORS,
+ SCAN_OFFLINE,
+ BATCH_SCANNER,
+ BATCH_SCANNER_THREADS
+ }
+
+ /**
+ * Sets the name of the context classloader to use for scans
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param context
+ * the name of the context classloader
+ * @since 1.8.0
+ */
+ public static void setClassLoaderContext(Class<?> implementingClass, Configuration conf,
+ String context) {
+ checkArgument(context != null, "context is null");
+ conf.set(enumToConfKey(implementingClass, ScanOpts.CLASSLOADER_CONTEXT), context);
+ }
+
+ /**
+ * Gets the name of the context classloader to use for scans
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the classloader context name
+ * @since 1.8.0
+ */
+ public static String getClassLoaderContext(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ScanOpts.CLASSLOADER_CONTEXT), null);
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.6.0
+ */
+ public static void setInputTableName(Class<?> implementingClass, Configuration conf,
+ String tableName) {
+ checkArgument(tableName != null, "tableName is null");
+ conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME), tableName);
+ }
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @since 1.6.0
+ */
+ public static String getInputTableName(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_NAME));
+ }
+
+ /**
+ * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorization.
+ * Defaults to the empty set.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param auths
+ * the user's authorizations
+ * @since 1.6.0
+ */
+ public static void setScanAuthorizations(Class<?> implementingClass, Configuration conf,
+ Authorizations auths) {
+ if (auths != null && !auths.isEmpty())
+ conf.set(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS), auths.serialize());
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the Accumulo scan authorizations
+ * @since 1.6.0
+ * @see #setScanAuthorizations(Class, Configuration, Authorizations)
+ */
+ public static Authorizations getScanAuthorizations(Class<?> implementingClass,
+ Configuration conf) {
+ String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
+ return authString == null ? Authorizations.EMPTY
+ : new Authorizations(authString.getBytes(UTF_8));
+ }
+
+ /**
+ * Sets the input ranges to scan on all input tables for this job. If not set, the entire table
+ * will be scanned.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param ranges
+ * the ranges that will be mapped over
+ * @throws IllegalArgumentException
+ * if the ranges cannot be encoded into base 64
+ * @since 1.6.0
+ */
+ public static void setRanges(Class<?> implementingClass, Configuration conf,
+ Collection<Range> ranges) {
+ checkArgument(ranges != null, "ranges is null");
+
+ ArrayList<String> rangeStrings = new ArrayList<>(ranges.size());
+ try {
+ for (Range r : ranges) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ r.write(new DataOutputStream(baos));
+ rangeStrings.add(Base64.getEncoder().encodeToString(baos.toByteArray()));
+ }
+ conf.setStrings(enumToConfKey(implementingClass, ScanOpts.RANGES),
+ rangeStrings.toArray(new String[0]));
+ } catch (IOException ex) {
+ throw new IllegalArgumentException("Unable to encode ranges to Base64", ex);
+ }
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.6.0
+ * @see #setRanges(Class, Configuration, Collection)
+ */
+ public static List<Range> getRanges(Class<?> implementingClass, Configuration conf)
+ throws IOException {
+
+ Collection<String> encodedRanges = conf
+ .getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES));
+ List<Range> ranges = new ArrayList<>();
+ for (String rangeString : encodedRanges) {
+ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.getDecoder().decode(rangeString));
+ Range range = new Range();
+ range.readFields(new DataInputStream(bais));
+ ranges.add(range);
+ }
+ return ranges;
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this
+ * configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return a list of iterators
+ * @since 1.6.0
+ * @see #addIterator(Class, Configuration, IteratorSetting)
+ */
+ public static List<IteratorSetting> getIterators(Class<?> implementingClass, Configuration conf) {
+ String iterators = conf.get(enumToConfKey(implementingClass, ScanOpts.ITERATORS));
+
+ // If no iterators are present, return an empty list
+ if (iterators == null || iterators.isEmpty())
+ return new ArrayList<>();
+
+ // Compose the set of iterators encoded in the job configuration
+ StringTokenizer tokens = new StringTokenizer(iterators, StringUtils.COMMA_STR);
+ List<IteratorSetting> list = new ArrayList<>();
+ try {
+ while (tokens.hasMoreTokens()) {
+ String itstring = tokens.nextToken();
+ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.getDecoder().decode(itstring));
+ list.add(new IteratorSetting(new DataInputStream(bais)));
+ bais.close();
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("couldn't decode iterator settings");
+ }
+ return list;
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for the single input table on this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If
+ * the column qualifier is null, the entire column family is selected. An empty set is
+ * the default and is equivalent to scanning the all columns.
+ * @throws IllegalArgumentException
+ * if the column family is null
+ * @since 1.6.0
+ */
+ public static void fetchColumns(Class<?> implementingClass, Configuration conf,
+ Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ checkArgument(columnFamilyColumnQualifierPairs != null,
+ "columnFamilyColumnQualifierPairs is null");
+ String[] columnStrings = serializeColumns(columnFamilyColumnQualifierPairs);
+ conf.setStrings(enumToConfKey(implementingClass, ScanOpts.COLUMNS), columnStrings);
+ }
+
+ public static String[] serializeColumns(
+ Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ checkArgument(columnFamilyColumnQualifierPairs != null,
+ "columnFamilyColumnQualifierPairs is null");
+ ArrayList<String> columnStrings = new ArrayList<>(columnFamilyColumnQualifierPairs.size());
+ for (Pair<Text,Text> column : columnFamilyColumnQualifierPairs) {
+
+ if (column.getFirst() == null)
+ throw new IllegalArgumentException("Column family can not be null");
+
+ String col = Base64.getEncoder().encodeToString(TextUtil.getBytes(column.getFirst()));
+ if (column.getSecond() != null)
+ col += ":" + Base64.getEncoder().encodeToString(TextUtil.getBytes(column.getSecond()));
+ columnStrings.add(col);
+ }
+
+ return columnStrings.toArray(new String[0]);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return a set of columns
+ * @since 1.6.0
+ * @see #fetchColumns(Class, Configuration, Collection)
+ */
+ public static Set<Pair<Text,Text>> getFetchedColumns(Class<?> implementingClass,
+ Configuration conf) {
+ checkArgument(conf != null, "conf is null");
+ String confValue = conf.get(enumToConfKey(implementingClass, ScanOpts.COLUMNS));
+ List<String> serialized = new ArrayList<>();
+ if (confValue != null) {
+ // Split and include any trailing empty strings to allow empty column families
+ for (String val : confValue.split(",", -1)) {
+ serialized.add(val);
+ }
+ }
+ return deserializeFetchedColumns(serialized);
+ }
+
+ public static Set<Pair<Text,Text>> deserializeFetchedColumns(Collection<String> serialized) {
+ Set<Pair<Text,Text>> columns = new HashSet<>();
+
+ if (null == serialized) {
+ return columns;
+ }
+
+ for (String col : serialized) {
+ int idx = col.indexOf(":");
+ Text cf = new Text(idx < 0 ? Base64.getDecoder().decode(col)
+ : Base64.getDecoder().decode(col.substring(0, idx)));
+ Text cq = idx < 0 ? null : new Text(Base64.getDecoder().decode(col.substring(idx + 1)));
+ columns.add(new Pair<>(cf, cq));
+ }
+ return columns;
+ }
+
+ /**
+ * Encode an iterator on the input for the single input table associated with this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param cfg
+ * the configuration of the iterator
+ * @throws IllegalArgumentException
+ * if the iterator can't be serialized into the configuration
+ * @since 1.6.0
+ */
+ public static void addIterator(Class<?> implementingClass, Configuration conf,
+ IteratorSetting cfg) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String newIter;
+ try {
+ cfg.write(new DataOutputStream(baos));
+ newIter = Base64.getEncoder().encodeToString(baos.toByteArray());
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("unable to serialize IteratorSetting");
+ }
+
+ String confKey = enumToConfKey(implementingClass, ScanOpts.ITERATORS);
+ String iterators = conf.get(confKey);
+ // No iterators specified yet, create a new string
+ if (iterators == null || iterators.isEmpty()) {
+ iterators = newIter;
+ } else {
+ // append the next iterator & reset
+ iterators = iterators.concat(StringUtils.COMMA_STR + newIter);
+ }
+ // Store the iterators w/ the job
+ conf.set(confKey, iterators);
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping
+ * ranges, then splits them to align with tablet boundaries. Disabling this feature will cause
+ * exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(Class, Configuration, Collection)
+ * @since 1.6.0
+ */
+ public static void setAutoAdjustRanges(Class<?> implementingClass, Configuration conf,
+ boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.6.0
+ * @see #setAutoAdjustRanges(Class, Configuration, boolean)
+ */
+ public static Boolean getAutoAdjustRanges(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true);
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setScanIsolation(Class<?> implementingClass, Configuration conf,
+ boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setScanIsolation(Class, Configuration, boolean)
+ */
+ public static Boolean isIsolated(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false);
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature
+ * will cause the iterator stack to be constructed within the Map task, rather than within the
+ * Accumulo TServer. To use this feature, all classes needed for those iterators must be available
+ * on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setLocalIterators(Class<?> implementingClass, Configuration conf,
+ boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setLocalIterators(Class, Configuration, boolean)
+ */
+ public static Boolean usesLocalIterators(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false);
+ }
+
+ /**
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are
+ * scanned. This will make the map reduce job directly read the table's files. If the table is not
+ * offline, then the job will fail. If the table comes online during the map reduce job, it is
+ * likely that the job will fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in
+ * HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any
+ * iterators that are configured for the table will need to be on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as
+ * the input table for a map reduce job. If you plan to map reduce over the data many times, it
+ * may be better to the compact the table, clone it, take it offline, and use the clone for all
+ * map reduce jobs. The reason to do this is that compaction will reduce each tablet in the table
+ * to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may
+ * see better read performance. Second, it will support speculative execution better. When reading
+ * an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setOfflineTableScan(Class<?> implementingClass, Configuration conf,
+ boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setOfflineTableScan(Class, Configuration, boolean)
+ */
+ public static Boolean isOfflineScan(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_OFFLINE), false);
+ }
+
+ /**
+ * Controls the use of the {@link BatchScanner} in this job. Using this feature will group ranges
+ * by their source tablet per InputSplit and use BatchScanner to read them.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.7.0
+ */
+ public static void setBatchScan(Class<?> implementingClass, Configuration conf,
+ boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.BATCH_SCANNER), enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the BatchScanner feature enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.7.0
+ * @see #setBatchScan(Class, Configuration, boolean)
+ */
+ public static Boolean isBatchScan(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.BATCH_SCANNER), false);
+ }
+
+ /**
+ * Sets configurations for multiple tables at a time.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param configs
+ * an array of {@link InputTableConfig} objects to associate with the job
+ * @since 1.6.0
+ */
+ public static void setInputTableConfigs(Class<?> implementingClass, Configuration conf,
+ Map<String,InputTableConfig> configs) {
+ MapWritable mapWritable = new MapWritable();
+ for (Map.Entry<String,InputTableConfig> tableConfig : configs.entrySet())
+ mapWritable.put(new Text(tableConfig.getKey()), tableConfig.getValue());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ mapWritable.write(new DataOutputStream(baos));
+ } catch (IOException e) {
+ throw new IllegalStateException("Table configuration could not be serialized.");
+ }
+
+ String confKey = enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS);
+ conf.set(confKey, Base64.getEncoder().encodeToString(baos.toByteArray()));
+ }
+
+ /**
+ * Returns all {@link InputTableConfig} objects associated with this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return all of the table query configs for the job
+ * @since 1.6.0
+ */
+ public static Map<String,InputTableConfig> getInputTableConfigs(Class<?> implementingClass,
+ Configuration conf) {
+ Map<String,InputTableConfig> configs = new HashMap<>();
+ Map.Entry<String,InputTableConfig> defaultConfig = getDefaultInputTableConfig(implementingClass,
+ conf);
+ if (defaultConfig != null)
+ configs.put(defaultConfig.getKey(), defaultConfig.getValue());
+ String configString = conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE_CONFIGS));
+ MapWritable mapWritable = new MapWritable();
+ if (configString != null) {
+ try {
+ byte[] bytes = Base64.getDecoder().decode(configString);
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ mapWritable.readFields(new DataInputStream(bais));
+ bais.close();
+ } catch (IOException e) {
+ throw new IllegalStateException("The table query configurations could not be deserialized"
+ + " from the given configuration");
+ }
+ }
+ for (Map.Entry<Writable,Writable> entry : mapWritable.entrySet())
+ configs.put(entry.getKey().toString(), (InputTableConfig) entry.getValue());
+
+ return configs;
+ }
+
+ /**
+ * Returns the {@link InputTableConfig} for the given table
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table name for which to fetch the table query config
+ * @return the table query config for the given table name (if it exists) and null if it does not
+ * @since 1.6.0
+ */
+ public static InputTableConfig getInputTableConfig(Class<?> implementingClass, Configuration conf,
+ String tableName) {
+ Map<String,InputTableConfig> queryConfigs = getInputTableConfigs(implementingClass, conf);
+ return queryConfigs.get(tableName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link TabletLocator} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableId
+ * The table id for which to initialize the {@link TabletLocator}
+ * @return an Accumulo tablet locator
+ * @since 1.6.0
+ */
+ public static TabletLocator getTabletLocator(Class<?> implementingClass, Configuration conf,
+ Table.ID tableId) {
+ ClientInfo info = getClientInfo(implementingClass, conf);
+ ClientContext context = new ClientContext(info);
+ return TabletLocator.getLocator(context, tableId);
+ }
+
+ /**
+ * Validates that the user has permissions on the requested tables
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param client
+ * the Accumulo client
+ * @since 1.7.0
+ */
+ public static void validatePermissions(Class<?> implementingClass, Configuration conf,
+ AccumuloClient client) throws IOException {
+ Map<String,InputTableConfig> inputTableConfigs = getInputTableConfigs(implementingClass, conf);
+ try {
+ if (getInputTableConfigs(implementingClass, conf).size() == 0)
+ throw new IOException("No table set.");
+
+ String principal = getPrincipal(implementingClass, conf);
+ if (principal == null) {
+ principal = getClientInfo(implementingClass, conf).getPrincipal();
+ }
+
+ for (Map.Entry<String,InputTableConfig> tableConfig : inputTableConfigs.entrySet()) {
+ if (!client.securityOperations().hasTablePermission(principal, tableConfig.getKey(),
+ TablePermission.READ))
+ throw new IOException("Unable to access table");
+ }
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : inputTableConfigs.entrySet()) {
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+ if (!tableConfig.shouldUseLocalIterators()) {
+ if (tableConfig.getIterators() != null) {
+ for (IteratorSetting iter : tableConfig.getIterators()) {
+ if (!client.tableOperations().testClassLoad(tableConfigEntry.getKey(),
+ iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
+ throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass()
+ + " as a " + SortedKeyValueIterator.class.getName());
+ }
+ }
+ }
+ }
+ } catch (AccumuloException | TableNotFoundException | AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Returns the {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} for the
+ * configuration based on the properties set using the single-table input methods.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop instance for which to retrieve the configuration
+ * @return the config object built from the single input table properties set on the job
+ * @since 1.6.0
+ */
+ protected static Map.Entry<String,InputTableConfig> getDefaultInputTableConfig(
+ Class<?> implementingClass, Configuration conf) {
+ String tableName = getInputTableName(implementingClass, conf);
+ if (tableName != null) {
+ InputTableConfig queryConfig = new InputTableConfig();
+ List<IteratorSetting> itrs = getIterators(implementingClass, conf);
+ if (itrs != null)
+ queryConfig.setIterators(itrs);
+ Set<Pair<Text,Text>> columns = getFetchedColumns(implementingClass, conf);
+ if (columns != null)
+ queryConfig.fetchColumns(columns);
+ List<Range> ranges = null;
+ try {
+ ranges = getRanges(implementingClass, conf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ if (ranges != null)
+ queryConfig.setRanges(ranges);
+
+ SamplerConfiguration samplerConfig = getSamplerConfiguration(implementingClass, conf);
+ if (samplerConfig != null) {
+ queryConfig.setSamplerConfiguration(samplerConfig);
+ }
+
+ queryConfig.setAutoAdjustRanges(getAutoAdjustRanges(implementingClass, conf))
+ .setUseIsolatedScanners(isIsolated(implementingClass, conf))
+ .setUseLocalIterators(usesLocalIterators(implementingClass, conf))
+ .setOfflineScan(isOfflineScan(implementingClass, conf))
+ .setExecutionHints(getExecutionHints(implementingClass, conf));
+ return Maps.immutableEntry(tableName, queryConfig);
+ }
+ return null;
+ }
+
+ public static Map<String,Map<KeyExtent,List<Range>>> binOffline(Table.ID tableId,
+ List<Range> ranges, ClientContext context)
+ throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
+
+ if (Tables.getTableState(context, tableId) != TableState.OFFLINE) {
+ Tables.clearCache(context);
+ if (Tables.getTableState(context, tableId) != TableState.OFFLINE) {
+ throw new AccumuloException(
+ "Table is online tableId:" + tableId + " cannot scan table in offline mode ");
+ }
+ }
+
+ for (Range range : ranges) {
+ Text startRow;
+
+ if (range.getStartKey() != null)
+ startRow = range.getStartKey().getRow();
+ else
+ startRow = new Text();
+
+ Range metadataRange = new Range(new KeyExtent(tableId, startRow, null).getMetadataEntry(),
+ true, null, false);
+ Scanner scanner = context.getClient().createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME);
+ scanner.setRange(metadataRange);
+
+ RowIterator rowIter = new RowIterator(scanner);
+ KeyExtent lastExtent = null;
+ while (rowIter.hasNext()) {
+ Iterator<Map.Entry<Key,Value>> row = rowIter.next();
+ String last = "";
+ KeyExtent extent = null;
+ String location = null;
+
+ while (row.hasNext()) {
+ Map.Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (key.getColumnFamily()
+ .equals(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME)) {
+ last = entry.getValue().toString();
+ }
+
+ if (key.getColumnFamily()
+ .equals(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME)
+ || key.getColumnFamily()
+ .equals(MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME)) {
+ location = entry.getValue().toString();
+ }
+
+ if (MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
+ extent = new KeyExtent(key.getRow(), entry.getValue());
+ }
+
+ }
+
+ if (location != null)
+ return null;
+
+ if (!extent.getTableId().equals(tableId)) {
+ throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
+ }
+
+ if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
+ throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
+ }
+
+ Map<KeyExtent,List<Range>> tabletRanges = binnedRanges.get(last);
+ if (tabletRanges == null) {
+ tabletRanges = new HashMap<>();
+ binnedRanges.put(last, tabletRanges);
+ }
+
+ List<Range> rangeList = tabletRanges.get(extent);
+ if (rangeList == null) {
+ rangeList = new ArrayList<>();
+ tabletRanges.put(extent, rangeList);
+ }
+
+ rangeList.add(range);
+
+ if (extent.getEndRow() == null
+ || range.afterEndKey(new Key(extent.getEndRow()).followingKey(PartialKey.ROW))) {
+ break;
+ }
+
+ lastExtent = extent;
+ }
+
+ }
+ return binnedRanges;
+ }
+
+ private static String toBase64(Writable writable) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ try {
+ writable.write(dos);
+ dos.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return Base64.getEncoder().encodeToString(baos.toByteArray());
+ }
+
+ private static <T extends Writable> T fromBase64(T writable, String enc) {
+ ByteArrayInputStream bais = new ByteArrayInputStream(Base64.getDecoder().decode(enc));
+ DataInputStream dis = new DataInputStream(bais);
+ try {
+ writable.readFields(dis);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return writable;
+ }
+
+ public static void setSamplerConfiguration(Class<?> implementingClass, Configuration conf,
+ SamplerConfiguration samplerConfig) {
+ requireNonNull(samplerConfig);
+
+ String key = enumToConfKey(implementingClass, ScanOpts.SAMPLER_CONFIG);
+ String val = toBase64(new SamplerConfigurationImpl(samplerConfig));
+
+ conf.set(key, val);
+ }
+
+ public static SamplerConfiguration getSamplerConfiguration(Class<?> implementingClass,
+ Configuration conf) {
+ String key = enumToConfKey(implementingClass, ScanOpts.SAMPLER_CONFIG);
+
+ String encodedSC = conf.get(key);
+ if (encodedSC == null)
+ return null;
+
+ return fromBase64(new SamplerConfigurationImpl(), encodedSC).toSamplerConfiguration();
+ }
+
+ public static void setExecutionHints(Class<?> implementingClass, Configuration conf,
+ Map<String,String> hints) {
+ MapWritable mapWritable = new MapWritable();
+ hints.forEach((k, v) -> mapWritable.put(new Text(k), new Text(v)));
+
+ String key = enumToConfKey(implementingClass, ScanOpts.EXECUTION_HINTS);
+ String val = toBase64(mapWritable);
+
+ conf.set(key, val);
+ }
+
+ public static Map<String,String> getExecutionHints(Class<?> implementingClass,
+ Configuration conf) {
+ String key = enumToConfKey(implementingClass, ScanOpts.EXECUTION_HINTS);
+ String encodedEH = conf.get(key);
+ if (encodedEH == null) {
+ return Collections.emptyMap();
+ }
+
+ MapWritable mapWritable = new MapWritable();
+ fromBase64(mapWritable, encodedEH);
+
+ HashMap<String,String> hints = new HashMap<>();
+ mapWritable.forEach((k, v) -> hints.put(k.toString(), v.toString()));
+ return hints;
+ }
+}
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOnDefaultTable.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOnDefaultTable.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOnDefaultTable.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOnDefaultTable.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOnRequiredTable.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOnRequiredTable.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOnRequiredTable.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOnRequiredTable.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOpts.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOpts.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOpts.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/MapReduceClientOpts.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/OutputConfigurator.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/package-info.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/package-info.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/package-info.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/package-info.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/KeyRangePartitioner.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/KeyRangePartitioner.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/KeyRangePartitioner.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/KeyRangePartitioner.java
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
similarity index 100%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
rename to core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormatTest.java
similarity index 95%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormatTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormatTest.java
index b4430a1df5..1c548ce037 100644
--- a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloMultiTableInputFormatTest.java
@@ -39,8 +39,7 @@
public TestName testName = new TestName();
/**
- * Verify {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} objects get correctly
- * serialized in the JobContext.
+ * Verify {@link InputTableConfig} objects get correctly serialized in the JobContext.
*/
@Test
public void testTableQueryConfigSerialization() throws IOException {
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloMultiTableInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloMultiTableInputFormatTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloMultiTableInputFormatTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloMultiTableInputFormatTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/InputTableConfigTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/InputTableConfigTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/InputTableConfigTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/InputTableConfigTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplitTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/impl/BatchInputSplitTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/impl/BatchInputSplitTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/impl/BatchInputSplitTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/impl/BatchInputSplitTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
diff --git a/client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java
similarity index 100%
rename from client/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java
rename to core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java
diff --git a/client/mapreduce/.gitignore b/hadoop-mapreduce/.gitignore
similarity index 100%
rename from client/mapreduce/.gitignore
rename to hadoop-mapreduce/.gitignore
diff --git a/client/mapreduce/pom.xml b/hadoop-mapreduce/pom.xml
similarity index 96%
rename from client/mapreduce/pom.xml
rename to hadoop-mapreduce/pom.xml
index 4bb9dd6196..a3a5060997 100644
--- a/client/mapreduce/pom.xml
+++ b/hadoop-mapreduce/pom.xml
@@ -23,9 +23,9 @@
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>accumulo-client-mapreduce</artifactId>
- <name>Apache Accumulo MapReduce APIs</name>
- <description>Apache Accumulo core libraries.</description>
+ <artifactId>accumulo-hadoop-mapreduce</artifactId>
+ <name>Apache Accumulo Hadoop MapReduce</name>
+ <description>Apache Accumulo MapReduce bindings.</description>
<dependencies>
<dependency>
<groupId>com.beust</groupId>
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AbstractInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AbstractInputFormat.java
new file mode 100644
index 0000000000..0587096e6d
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AbstractInputFormat.java
@@ -0,0 +1,790 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.admin.SecurityOperations;
+import org.apache.accumulo.core.client.impl.AuthenticationTokenIdentifier;
+import org.apache.accumulo.core.client.impl.ClientContext;
+import org.apache.accumulo.core.client.impl.DelegationTokenImpl;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.DelegationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.hadoop.mapreduce.InputTableConfig;
+import org.apache.accumulo.hadoopImpl.mapred.BatchInputSplit;
+import org.apache.accumulo.hadoopImpl.mapreduce.SplitUtils;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.ConfiguratorBase;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.security.token.Token;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * An abstract input format to provide shared methods common to all other input format classes. At
+ * the very least, any classes inheriting from this class will need to define their own
+ * {@link RecordReader}.
+ */
+public abstract class AbstractInputFormat<K,V> implements InputFormat<K,V> {
+ protected static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the name of the classloader context on this scanner
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param context
+ * name of the classloader context
+ * @since 1.8.0
+ */
+ public static void setClassLoaderContext(JobConf job, String context) {
+ InputConfigurator.setClassLoaderContext(CLASS, job, context);
+ }
+
+ /**
+ * Returns the name of the current classloader context set on this scanner
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @return name of the current context
+ * @since 1.8.0
+ */
+ public static String getClassLoaderContext(JobConf job) {
+ return InputConfigurator.getClassLoaderContext(CLASS, job);
+ }
+
+ /**
+ * Sets connection information needed to communicate with Accumulo for this job
+ *
+ * @param job
+ * Hadoop job instance to be configured
+ * @param info
+ * Connection information for Accumulo
+ * @since 2.0.0
+ */
+ public static void setClientInfo(JobConf job, ClientInfo info) {
+ ClientInfo inputInfo = InputConfigurator.updateToken(job.getCredentials(), info);
+ InputConfigurator.setClientInfo(CLASS, job, inputInfo);
+ }
+
+ /**
+ * Set Accumulo client properties file used to connect to Accumulo
+ *
+ * @param job
+ * Hadoop job to be configured
+ * @param clientPropsFile
+ * URL to Accumulo client properties file
+ * @since 2.0.0
+ */
+ public static void setClientPropertiesFile(JobConf job, String clientPropsFile) {
+ InputConfigurator.setClientPropertiesFile(CLASS, job, clientPropsFile);
+ }
+
+ /**
+ * Retrieves {@link ClientInfo} from the configuration
+ *
+ * @param job
+ * Hadoop job instance configuration
+ * @return {@link ClientInfo} object
+ * @since 2.0.0
+ */
+ protected static ClientInfo getClientInfo(JobConf job) {
+ return InputConfigurator.getClientInfo(CLASS, job);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> Some tokens, when serialized, divulge sensitive information in the
+ * configuration as a means to pass the token to MapReduce tasks. This information is BASE64
+ * encoded to provide a charset safe conversion to a string, but this conversion is not intended
+ * to be secure. {@link PasswordToken} is one example that is insecure in this way; however
+ * {@link DelegationToken}s, acquired using
+ * {@link SecurityOperations#getDelegationToken(DelegationTokenConfig)}, is not subject to this
+ * concern.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
+ * @since 1.5.0
+ * @deprecated since 2.0.0, use {@link #setClientInfo(JobConf, ClientInfo)} instead
+ */
+ @Deprecated
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (token instanceof KerberosToken) {
+ log.info("Received KerberosToken, attempting to fetch DelegationToken");
+ try {
+ AccumuloClient client = Accumulo.newClient().usingClientInfo(getClientInfo(job))
+ .usingToken(principal, token).build();
+ token = client.securityOperations().getDelegationToken(new DelegationTokenConfig());
+ } catch (Exception e) {
+ log.warn("Failed to automatically obtain DelegationToken, Mappers/Reducers will likely"
+ + " fail to communicate with Accumulo", e);
+ }
+ }
+ // DelegationTokens can be passed securely from user to task without serializing insecurely in
+ // the configuration
+ if (token instanceof DelegationTokenImpl) {
+ DelegationTokenImpl delegationToken = (DelegationTokenImpl) token;
+
+ // Convert it into a Hadoop Token
+ AuthenticationTokenIdentifier identifier = delegationToken.getIdentifier();
+ Token<AuthenticationTokenIdentifier> hadoopToken = new Token<>(identifier.getBytes(),
+ delegationToken.getPassword(), identifier.getKind(), delegationToken.getServiceName());
+
+ // Add the Hadoop Token to the Job so it gets serialized and passed along.
+ job.getCredentials().addToken(hadoopToken.getService(), hadoopToken);
+ }
+
+ InputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt
+ * to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param tokenFile
+ * the path to the token file
+ * @since 1.6.0
+ * @deprecated since 2.0.0, use {@link #setClientPropertiesFile(JobConf, String)} instead
+ */
+ @Deprecated
+ public static void setConnectorInfo(JobConf job, String principal, String tokenFile)
+ throws AccumuloSecurityException {
+ setClientPropertiesFile(job, tokenFile);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return InputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the
+ * configuration, whichever was used when the job was configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ * @see #setConnectorInfo(JobConf, String, String)
+ */
+ protected static AuthenticationToken getAuthenticationToken(JobConf job) {
+ AuthenticationToken token = InputConfigurator.getAuthenticationToken(CLASS, job);
+ return ConfiguratorBase.unwrapAuthenticationToken(job, token);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param clientConfig
+ * client configuration containing connection options
+ * @since 1.6.0
+ * @deprecated since 2.0.0; Use {@link #setClientInfo(JobConf, ClientInfo)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(JobConf job,
+ org.apache.accumulo.core.client.ClientConfiguration clientConfig) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job, clientConfig);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the
+ * configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @deprecated since 2.0.0, Use {@link #getClientInfo(JobConf)} instead
+ */
+ @Deprecated
+ protected static org.apache.accumulo.core.client.Instance getInstance(JobConf job) {
+ return InputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return InputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the {@link org.apache.accumulo.core.security.Authorizations} used to scan. Must be a
+ * subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ * @since 1.5.0
+ */
+ public static void setScanAuthorizations(JobConf job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job, auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(JobConf, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobConf job) {
+ return InputConfigurator.getScanAuthorizations(CLASS, job);
+ }
+
+ /**
+ * Fetch the client configuration from the job.
+ *
+ * @param job
+ * The job
+ * @return The client configuration for the job
+ * @since 1.7.0
+ * @deprecated since 2.0.0, replaced by {@link #getClientInfo(JobConf)}
+ */
+ @Deprecated
+ protected static org.apache.accumulo.core.client.ClientConfiguration getClientConfiguration(
+ JobConf job) {
+ return InputConfigurator.getClientConfiguration(CLASS, job);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo
+ * {@link InputFormat}.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @throws java.io.IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobConf job) throws IOException {
+ AccumuloClient client = InputConfigurator.getClient(CLASS, job);
+ InputConfigurator.validatePermissions(CLASS, job, client);
+ }
+
+ /**
+ * Fetches all {@link InputTableConfig}s that have been set on the given Hadoop job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @return the {@link InputTableConfig} objects set on the job
+ * @since 1.6.0
+ */
+ public static Map<String,InputTableConfig> getInputTableConfigs(JobConf job) {
+ return InputConfigurator.getInputTableConfigs(CLASS, job);
+ }
+
+ /**
+ * Fetches a {@link InputTableConfig} that has been set on the configuration for a specific table.
+ *
+ * <p>
+ * null is returned in the event that the table doesn't exist.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table name for which to grab the config object
+ * @return the {@link InputTableConfig} for the given table
+ * @since 1.6.0
+ */
+ public static InputTableConfig getInputTableConfig(JobConf job, String tableName) {
+ return InputConfigurator.getInputTableConfig(CLASS, job, tableName);
+ }
+
+ /**
+ * An abstract base class to be used to create {@link org.apache.hadoop.mapred.RecordReader}
+ * instances that convert from Accumulo
+ * {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value} pairs to
+ * the user's K/V types.
+ *
+ * Subclasses must implement {@link #next(Object, Object)} to update key and value, and also to
+ * update the following variables:
+ * <ul>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class AbstractRecordReader<K,V> implements RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Map.Entry<Key,Value>> scannerIterator;
+ protected RangeInputSplit split;
+ private org.apache.accumulo.hadoop.mapreduce.RangeInputSplit baseSplit;
+ protected ScannerBase scannerBase;
+
+ /**
+ * Extracts Iterators settings from the context to be used by RecordReader.
+ *
+ * @param job
+ * the Hadoop job configuration
+ * @param tableName
+ * the table name for which the scanner is configured
+ * @return List of iterator settings for given table
+ * @since 1.7.0
+ */
+ protected abstract List<IteratorSetting> jobIterators(JobConf job, String tableName);
+
+ /**
+ * Configures the iterators on a scanner for the given table name.
+ *
+ * @param job
+ * the Hadoop job configuration
+ * @param scanner
+ * the scanner for which to configure the iterators
+ * @param tableName
+ * the table name for which the scanner is configured
+ * @since 1.7.0
+ */
+ private void setupIterators(JobConf job, ScannerBase scanner, String tableName,
+ org.apache.accumulo.hadoop.mapreduce.RangeInputSplit split) {
+ List<IteratorSetting> iterators = null;
+
+ if (null == split) {
+ iterators = jobIterators(job, tableName);
+ } else {
+ iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = jobIterators(job, tableName);
+ }
+ }
+
+ for (IteratorSetting iterator : iterators)
+ scanner.addScanIterator(iterator);
+ }
+
+ /**
+ * Initialize a scanner over the given input split using this task attempt configuration.
+ */
+ public void initialize(InputSplit inSplit, JobConf job) throws IOException {
+ baseSplit = (org.apache.accumulo.hadoop.mapreduce.RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + baseSplit);
+
+ ClientContext context = new ClientContext(getClientInfo(job));
+ AccumuloClient client;
+ try {
+ client = context.getClient();
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw new IllegalStateException(e);
+ }
+ Authorizations authorizations = getScanAuthorizations(job);
+ String classLoaderContext = getClassLoaderContext(job);
+ String table = baseSplit.getTableName();
+
+ // in case the table name changed, we can still use the previous name for terms of
+ // configuration, but the scanner will use the table id resolved at job setup time
+ InputTableConfig tableConfig = getInputTableConfig(job, baseSplit.getTableName());
+
+ log.debug("Created client with user: " + client.whoami());
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+
+ if (baseSplit instanceof BatchInputSplit) {
+ BatchScanner scanner;
+ BatchInputSplit multiRangeSplit = (BatchInputSplit) baseSplit;
+
+ try {
+ // Note: BatchScanner will use at most one thread per tablet, currently BatchInputSplit
+ // will not span tablets
+ int scanThreads = 1;
+ scanner = client.createBatchScanner(baseSplit.getTableName(), authorizations,
+ scanThreads);
+ setupIterators(job, scanner, baseSplit.getTableName(), baseSplit);
+ if (null != classLoaderContext) {
+ scanner.setClassLoaderContext(classLoaderContext);
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ scanner.setRanges(multiRangeSplit.getRanges());
+ scannerBase = scanner;
+
+ } else if (baseSplit instanceof RangeInputSplit) {
+ split = (RangeInputSplit) baseSplit;
+ Boolean isOffline = baseSplit.isOffline();
+ if (null == isOffline) {
+ isOffline = tableConfig.isOfflineScan();
+ }
+
+ Boolean isIsolated = baseSplit.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = tableConfig.shouldUseIsolatedScanners();
+ }
+
+ Boolean usesLocalIterators = baseSplit.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = tableConfig.shouldUseLocalIterators();
+ }
+
+ Scanner scanner;
+
+ try {
+ if (isOffline) {
+ scanner = new OfflineScanner(context, Table.ID.of(baseSplit.getTableId()),
+ authorizations);
+ } else {
+ scanner = new ScannerImpl(context, Table.ID.of(baseSplit.getTableId()), authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+ setupIterators(job, scanner, baseSplit.getTableName(), baseSplit);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ scanner.setRange(baseSplit.getRange());
+ scannerBase = scanner;
+ } else {
+ throw new IllegalArgumentException("Can not initialize from " + baseSplit.getClass());
+ }
+
+ Collection<Pair<Text,Text>> columns = baseSplit.getFetchedColumns();
+ if (null == columns) {
+ columns = tableConfig.getFetchedColumns();
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scannerBase.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scannerBase.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ SamplerConfiguration samplerConfig = baseSplit.getSamplerConfiguration();
+ if (null == samplerConfig) {
+ samplerConfig = tableConfig.getSamplerConfiguration();
+ }
+
+ if (samplerConfig != null) {
+ scannerBase.setSamplerConfiguration(samplerConfig);
+ }
+
+ Map<String,String> executionHints = baseSplit.getExecutionHints();
+ if (executionHints == null || executionHints.size() == 0) {
+ executionHints = tableConfig.getExecutionHints();
+ }
+
+ if (executionHints != null) {
+ scannerBase.setExecutionHints(executionHints);
+ }
+
+ scannerIterator = scannerBase.iterator();
+ numKeysRead = 0;
+ }
+
+ @Override
+ public void close() {
+ if (null != scannerBase) {
+ scannerBase.close();
+ }
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return numKeysRead;
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return baseSplit.getProgress(currentKey);
+ }
+
+ protected Key currentKey = null;
+
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobConf job, Table.ID tableId,
+ List<Range> ranges)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ClientContext context = new ClientContext(getClientInfo(job));
+ return InputConfigurator.binOffline(tableId, ranges, context);
+ }
+
+ /**
+ * Gets the splits of the tables that have been set on the job by reading the metadata table for
+ * the specified ranges.
+ *
+ * @return the splits from the tables based on the ranges.
+ * @throws java.io.IOException
+ * if a table set on the job doesn't exist or an error occurs initializing the tablet
+ * locator
+ */
+ @Override
+ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+ Level logLevel = getLogLevel(job);
+ log.setLevel(logLevel);
+ validateOptions(job);
+
+ Random random = new SecureRandom();
+ LinkedList<InputSplit> splits = new LinkedList<>();
+ Map<String,InputTableConfig> tableConfigs = getInputTableConfigs(job);
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
+ String tableName = tableConfigEntry.getKey();
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+
+ ClientContext context = new ClientContext(getClientInfo(job));
+ Table.ID tableId;
+ // resolve table name to id once, and use id from this point forward
+ try {
+ tableId = Tables.getTableId(context, tableName);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+
+ boolean batchScan = InputConfigurator.isBatchScan(CLASS, job);
+ boolean supportBatchScan = !(tableConfig.isOfflineScan()
+ || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
+ if (batchScan && !supportBatchScan)
+ throw new IllegalArgumentException("BatchScanner optimization not available for offline"
+ + " scan, isolated, or local iterators");
+
+ boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
+ if (batchScan && !autoAdjust)
+ throw new IllegalArgumentException(
+ "AutoAdjustRanges must be enabled when using BatchScanner optimization");
+
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges())
+ : tableConfig.getRanges();
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
+ TabletLocator tl;
+ try {
+ if (tableConfig.isOfflineScan()) {
+ binnedRanges = binOfflineTable(job, tableId, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ // sleep randomly between 100 and 200 ms
+ sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
+ binnedRanges = binOfflineTable(job, tableId, ranges);
+ }
+ } else {
+ tl = InputConfigurator.getTabletLocator(CLASS, job, tableId);
+ // its possible that the cache could contain complete, but old information about a tables
+ // tablets... so clear it
+ tl.invalidateCache();
+
+ while (!tl.binRanges(context, ranges, binnedRanges).isEmpty()) {
+ String tableIdStr = tableId.canonicalID();
+ if (!Tables.exists(context, tableId))
+ throw new TableDeletedException(tableIdStr);
+ if (Tables.getTableState(context, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(Tables.getTableOfflineMsg(context, tableId));
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ // sleep randomly between 100 and 200 ms
+ sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<>();
+
+ HashMap<String,String> hostNameCache = new HashMap<>();
+ for (Map.Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getCanonicalHostName();
+ hostNameCache.put(ip, location);
+ }
+ for (Map.Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ if (batchScan) {
+ // group ranges by tablet to be read by a BatchScanner
+ ArrayList<Range> clippedRanges = new ArrayList<>();
+ for (Range r : extentRanges.getValue())
+ clippedRanges.add(ke.clip(r));
+
+ BatchInputSplit split = new BatchInputSplit(tableName, tableId, clippedRanges,
+ new String[] {location});
+ SplitUtils.updateSplit(split, tableConfig, logLevel);
+
+ splits.add(split);
+ } else {
+ // not grouping by tablet
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonicalID(),
+ ke.clip(r), new String[] {location});
+ SplitUtils.updateSplit(split, tableConfig, logLevel);
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+
+ splits.add(split);
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Map.Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet()) {
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonicalID(),
+ entry.getKey(), entry.getValue().toArray(new String[0]));
+ SplitUtils.updateSplit(split, tableConfig, logLevel);
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+
+ splits.add(split);
+ }
+ }
+
+ return splits.toArray(new InputSplit[splits.size()]);
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloFileOutputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloFileOutputFormat.java
new file mode 100644
index 0000000000..a4664d126c
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloFileOutputFormat.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.accumulo.core.client.rfile.RFile;
+import org.apache.accumulo.core.client.rfile.RFileWriter;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.client.summary.Summarizer;
+import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.ConfiguratorBase;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.FileOutputConfigurator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Progressable;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to write output in the Accumulo data file format.<br>
+ * Care should be taken to write only sorted data (sorted by {@link Key}), as this is an important
+ * requirement of Accumulo data files.
+ *
+ * <p>
+ * The output path to be created must be specified via
+ * {@link AccumuloFileOutputFormat#setOutputPath(JobConf, Path)}. This is inherited from
+ * {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Other methods from
+ * {@link FileOutputFormat} are not supported and may be ignored or cause failures. Using other
+ * Hadoop configuration options that affect the behavior of the underlying files directly in the
+ * Job's configuration may work, but are not directly supported at this time.
+ */
+public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
+
+ private static final Class<?> CLASS = AccumuloFileOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the compression type to use for data blocks. Specifying a compression may require
+ * additional libraries to be available to your Job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param compressionType
+ * one of "none", "gz", "lzo", or "snappy"
+ * @since 1.5.0
+ */
+ public static void setCompressionType(JobConf job, String compressionType) {
+ FileOutputConfigurator.setCompressionType(CLASS, job, compressionType);
+ }
+
+ /**
+ * Sets the size for data blocks within each file.<br>
+ * Data blocks are a span of key/value pairs stored in the file that are compressed and indexed as
+ * a group.
+ *
+ * <p>
+ * Making this value smaller may increase seek performance, but at the cost of increasing the size
+ * of the indexes (which can also affect seek performance).
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param dataBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setDataBlockSize(JobConf job, long dataBlockSize) {
+ FileOutputConfigurator.setDataBlockSize(CLASS, job, dataBlockSize);
+ }
+
+ /**
+ * Sets the size for file blocks in the file system; file blocks are managed, and replicated, by
+ * the underlying file system.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param fileBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setFileBlockSize(JobConf job, long fileBlockSize) {
+ FileOutputConfigurator.setFileBlockSize(CLASS, job, fileBlockSize);
+ }
+
+ /**
+ * Sets the size for index blocks within each file; smaller blocks means a deeper index hierarchy
+ * within the file, while larger blocks mean a more shallow index hierarchy within the file. This
+ * can affect the performance of queries.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param indexBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setIndexBlockSize(JobConf job, long indexBlockSize) {
+ FileOutputConfigurator.setIndexBlockSize(CLASS, job, indexBlockSize);
+ }
+
+ /**
+ * Sets the file system replication factor for the resulting file, overriding the file system
+ * default.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param replication
+ * the number of replicas for produced files
+ * @since 1.5.0
+ */
+ public static void setReplication(JobConf job, int replication) {
+ FileOutputConfigurator.setReplication(CLASS, job, replication);
+ }
+
+ /**
+ * Specify a sampler to be used when writing out data. This will result in the output file having
+ * sample data.
+ *
+ * @param job
+ * The Hadoop job instance to be configured
+ * @param samplerConfig
+ * The configuration for creating sample data in the output file.
+ * @since 1.8.0
+ */
+
+ public static void setSampler(JobConf job, SamplerConfiguration samplerConfig) {
+ FileOutputConfigurator.setSampler(CLASS, job, samplerConfig);
+ }
+
+ /**
+ * Specifies a list of summarizer configurations to create summary data in the output file. Each
+ * Key Value written will be passed to the configured {@link Summarizer}'s.
+ *
+ * @param job
+ * The Hadoop job instance to be configured
+ * @param summarizerConfigs
+ * summarizer configurations
+ * @since 2.0.0
+ */
+ public static void setSummarizers(JobConf job, SummarizerConfiguration... summarizerConfigs) {
+ FileOutputConfigurator.setSummarizers(CLASS, job, summarizerConfigs);
+ }
+
+ @Override
+ public RecordWriter<Key,Value> getRecordWriter(FileSystem ignored, JobConf job, String name,
+ Progressable progress) throws IOException {
+ // get the path of the temporary output file
+ final Configuration conf = job;
+ final AccumuloConfiguration acuConf = FileOutputConfigurator.getAccumuloConfiguration(CLASS,
+ job);
+
+ final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
+ final Path file = new Path(getWorkOutputPath(job),
+ getUniqueName(job, "part") + "." + extension);
+ final int visCacheSize = ConfiguratorBase.getVisibilityCacheSize(conf);
+
+ return new RecordWriter<Key,Value>() {
+ RFileWriter out = null;
+
+ @Override
+ public void close(Reporter reporter) throws IOException {
+ if (out != null)
+ out.close();
+ }
+
+ @Override
+ public void write(Key key, Value value) throws IOException {
+ if (out == null) {
+ out = RFile.newWriter().to(file.toString()).withFileSystem(file.getFileSystem(conf))
+ .withTableProperties(acuConf).withVisibilityCacheSize(visCacheSize).build();
+ out.startDefaultLocalityGroup();
+ }
+ out.append(key, value);
+ }
+ };
+ }
+
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormat.java
new file mode 100644
index 0000000000..8a7e6a9734
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormat.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.accumulo.hadoop.mapreduce.RangeInputSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.log4j.Level;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the source of data. This {@link InputFormat}
+ * provides keys and values of type {@link Key} and {@link Value} to the Map function.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloInputFormat#setClientInfo(JobConf, ClientInfo)}
+ * <li>{@link AccumuloInputFormat#setInputTableName(JobConf, String)}</li>
+ * <li>{@link AccumuloInputFormat#setScanAuthorizations(JobConf, Authorizations)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloInputFormat extends InputFormatBase<Key,Value> {
+
+ @Override
+ public RecordReader<Key,Value> getRecordReader(InputSplit split, JobConf job, Reporter reporter)
+ throws IOException {
+ log.setLevel(getLogLevel(job));
+
+ // Override the log level from the configuration as if the RangeInputSplit has one it's the more
+ // correct one to use.
+ if (split instanceof RangeInputSplit) {
+ RangeInputSplit accSplit = (RangeInputSplit) split;
+ Level level = accSplit.getLogLevel();
+ if (null != level) {
+ log.setLevel(level);
+ }
+ } else {
+ throw new IllegalArgumentException("No RecordReader for " + split.getClass());
+ }
+
+ RecordReaderBase<Key,Value> recordReader = new RecordReaderBase<Key,Value>() {
+
+ @Override
+ public boolean next(Key key, Value value) throws IOException {
+ if (scannerIterator.hasNext()) {
+ ++numKeysRead;
+ Entry<Key,Value> entry = scannerIterator.next();
+ key.set(currentKey = entry.getKey());
+ value.set(entry.getValue().get());
+ if (log.isTraceEnabled())
+ log.trace("Processing key/value pair: " + DefaultFormatter.formatEntry(entry, true));
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public Key createKey() {
+ return new Key();
+ }
+
+ @Override
+ public Value createValue() {
+ return new Value();
+ }
+
+ };
+ recordReader.initialize(split, job);
+ return recordReader;
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloMultiTableInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloMultiTableInputFormat.java
new file mode 100644
index 0000000000..e34d307007
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloMultiTableInputFormat.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.accumulo.hadoop.mapred.InputFormatBase.RecordReaderBase;
+import org.apache.accumulo.hadoop.mapreduce.InputTableConfig;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * This class allows MapReduce jobs to use multiple Accumulo tables as the source of data. This
+ * {@link org.apache.hadoop.mapred.InputFormat} provides keys and values of type {@link Key} and
+ * {@link Value} to the Map function.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloInputFormat#setClientInfo(JobConf, ClientInfo)}
+ * <li>{@link AccumuloInputFormat#setScanAuthorizations(JobConf, org.apache.accumulo.core.security.Authorizations)}
+ * <li>{@link AccumuloMultiTableInputFormat#setInputTableConfigs(org.apache.hadoop.mapred.JobConf, java.util.Map)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+
+public class AccumuloMultiTableInputFormat extends AbstractInputFormat<Key,Value> {
+
+ /**
+ * Sets the {@link InputTableConfig} objects on the given Hadoop configuration
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param configs
+ * the table query configs to be set on the configuration.
+ * @since 1.6.0
+ */
+ public static void setInputTableConfigs(JobConf job, Map<String,InputTableConfig> configs) {
+ InputConfigurator.setInputTableConfigs(CLASS, job, configs);
+ }
+
+ @Override
+ public RecordReader<Key,Value> getRecordReader(InputSplit split, JobConf job, Reporter reporter)
+ throws IOException {
+ log.setLevel(getLogLevel(job));
+ RecordReaderBase<Key,Value> recordReader = new RecordReaderBase<Key,Value>() {
+
+ @Override
+ public boolean next(Key key, Value value) throws IOException {
+ if (scannerIterator.hasNext()) {
+ ++numKeysRead;
+ Map.Entry<Key,Value> entry = scannerIterator.next();
+ key.set(currentKey = entry.getKey());
+ value.set(entry.getValue().get());
+ if (log.isTraceEnabled())
+ log.trace("Processing key/value pair: " + DefaultFormatter.formatEntry(entry, true));
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public Key createKey() {
+ return new Key();
+ }
+
+ @Override
+ public Value createValue() {
+ return new Value();
+ }
+
+ };
+ recordReader.initialize(split, job);
+ return recordReader;
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloOutputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloOutputFormat.java
new file mode 100644
index 0000000000..da8fa1b953
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloOutputFormat.java
@@ -0,0 +1,587 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.admin.SecurityOperations;
+import org.apache.accumulo.core.client.impl.AuthenticationTokenIdentifier;
+import org.apache.accumulo.core.client.impl.DelegationTokenImpl;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.DelegationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.TabletId;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.ConfiguratorBase;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.OutputConfigurator;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Progressable;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the sink for data. This {@link OutputFormat}
+ * accepts keys and values of type {@link Text} (for a table name) and {@link Mutation} from the Map
+ * and Reduce functions.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloOutputFormat#setClientInfo(JobConf, ClientInfo)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloOutputFormat implements OutputFormat<Text,Mutation> {
+
+ private static final Class<?> CLASS = AccumuloOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Set the connection information needed to communicate with Accumulo in this job.
+ *
+ * @param job
+ * Hadoop job to be configured
+ * @param info
+ * Accumulo connection information
+ * @since 2.0.0
+ */
+ public static void setClientInfo(JobConf job, ClientInfo info) {
+ ClientInfo outInfo = OutputConfigurator.updateToken(job.getCredentials(), info);
+ OutputConfigurator.setClientInfo(CLASS, job, outInfo);
+ }
+
+ /**
+ * Get the connection information needed to communication with Accumulo
+ *
+ * @param job
+ * Hadoop job to be configured
+ * @since 2.0.0
+ */
+ protected static ClientInfo getClientInfo(JobConf job) {
+ return OutputConfigurator.getClientInfo(CLASS, job);
+ }
+
+ /**
+ * Set Accumulo client properties file used to connect to Accumulo
+ *
+ * @param job
+ * Hadoop job to be configured
+ * @param clientPropsFile
+ * URL (hdfs:// or http://) to Accumulo client properties file
+ * @since 2.0.0
+ */
+ public static void setClientPropertiesFile(JobConf job, String clientPropsFile) {
+ OutputConfigurator.setClientPropertiesFile(CLASS, job, clientPropsFile);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> Some tokens, when serialized, divulge sensitive information in the
+ * configuration as a means to pass the token to MapReduce tasks. This information is BASE64
+ * encoded to provide a charset safe conversion to a string, but this conversion is not intended
+ * to be secure. {@link PasswordToken} is one example that is insecure in this way; however
+ * {@link DelegationToken}s, acquired using
+ * {@link SecurityOperations#getDelegationToken(DelegationTokenConfig)}, is not subject to this
+ * concern.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if
+ * {@link #setCreateTables(JobConf, boolean)} is set to true)
+ * @param token
+ * the user's password
+ * @since 1.5.0
+ * @deprecated since 2.0.0, use {@link #setClientInfo(JobConf, ClientInfo)} instead.
+ */
+ @Deprecated
+ public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ // DelegationTokens can be passed securely from user to task without serializing insecurely in
+ // the configuration
+ if (token instanceof DelegationTokenImpl) {
+ DelegationTokenImpl delegationToken = (DelegationTokenImpl) token;
+
+ // Convert it into a Hadoop Token
+ AuthenticationTokenIdentifier identifier = delegationToken.getIdentifier();
+ Token<AuthenticationTokenIdentifier> hadoopToken = new Token<>(identifier.getBytes(),
+ delegationToken.getPassword(), identifier.getKind(), delegationToken.getServiceName());
+
+ // Add the Hadoop Token to the Job so it gets serialized and passed along.
+ job.getCredentials().addToken(hadoopToken.getService(), hadoopToken);
+ }
+
+ OutputConfigurator.setConnectorInfo(CLASS, job, principal, token);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt
+ * to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if
+ * {@link #setCreateTables(JobConf, boolean)} is set to true)
+ * @param tokenFile
+ * the path to the password file
+ * @since 1.6.0
+ * @deprecated since 2.0.0, use {@link #setClientPropertiesFile(JobConf, String)} instead
+ */
+ @Deprecated
+ public static void setConnectorInfo(JobConf job, String principal, String tokenFile)
+ throws AccumuloSecurityException {
+ setClientPropertiesFile(job, tokenFile);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobConf job) {
+ return OutputConfigurator.isConnectorInfoSet(CLASS, job);
+ }
+
+ /**
+ * Gets the principal from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobConf job) {
+ return OutputConfigurator.getPrincipal(CLASS, job);
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the
+ * configuration, whichever was used when the job was configured.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+ * @see #setConnectorInfo(JobConf, String, String)
+ */
+ protected static AuthenticationToken getAuthenticationToken(JobConf job) {
+ AuthenticationToken token = OutputConfigurator.getAuthenticationToken(CLASS, job);
+ return ConfiguratorBase.unwrapAuthenticationToken(job, token);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ *
+ * @param clientConfig
+ * client configuration for specifying connection timeouts, SSL connection options, etc.
+ * @since 1.6.0
+ * @deprecated since 2.0.0; Use {@link #setClientInfo(JobConf, ClientInfo)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(JobConf job,
+ org.apache.accumulo.core.client.ClientConfiguration clientConfig) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, job, clientConfig);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the
+ * configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @deprecated since 2.0.0; Use {@link #getClientInfo(JobConf)} instead
+ */
+ @Deprecated
+ protected static org.apache.accumulo.core.client.Instance getInstance(JobConf job) {
+ return OutputConfigurator.getInstance(CLASS, job);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(JobConf job, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, job, level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(JobConf, Level)
+ */
+ protected static Level getLogLevel(JobConf job) {
+ return OutputConfigurator.getLogLevel(CLASS, job);
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given
+ * mutation. Table names can only be alpha-numeric and underscores.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setDefaultTableName(JobConf job, String tableName) {
+ OutputConfigurator.setDefaultTableName(CLASS, job, tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the default table name
+ * @since 1.5.0
+ * @see #setDefaultTableName(JobConf, String)
+ */
+ protected static String getDefaultTableName(JobConf job) {
+ return OutputConfigurator.getDefaultTableName(CLASS, job);
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new
+ * {@link BatchWriterConfig}, with sensible built-in defaults is used. Setting the configuration
+ * multiple times overwrites any previous configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.5.0
+ */
+ public static void setBatchWriterOptions(JobConf job, BatchWriterConfig bwConfig) {
+ OutputConfigurator.setBatchWriterOptions(CLASS, job, bwConfig);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the configuration object
+ * @since 1.5.0
+ * @see #setBatchWriterOptions(JobConf, BatchWriterConfig)
+ */
+ protected static BatchWriterConfig getBatchWriterOptions(JobConf job) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, job);
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric
+ * and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setCreateTables(JobConf job, boolean enableFeature) {
+ OutputConfigurator.setCreateTables(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.5.0
+ * @see #setCreateTables(JobConf, boolean)
+ */
+ protected static Boolean canCreateTables(JobConf job) {
+ return OutputConfigurator.canCreateTables(CLASS, job);
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is
+ * produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setSimulationMode(JobConf job, boolean enableFeature) {
+ OutputConfigurator.setSimulationMode(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setSimulationMode(JobConf, boolean)
+ */
+ protected static Boolean getSimulationMode(JobConf job) {
+ return OutputConfigurator.getSimulationMode(CLASS, job);
+ }
+
+ /**
+ * A base class to be used to create {@link RecordWriter} instances that write to Accumulo.
+ */
+ protected static class AccumuloRecordWriter implements RecordWriter<Text,Mutation> {
+ private MultiTableBatchWriter mtbw = null;
+ private HashMap<Text,BatchWriter> bws = null;
+ private Text defaultTableName = null;
+
+ private boolean simulate = false;
+ private boolean createTables = false;
+
+ private long mutCount = 0;
+ private long valCount = 0;
+
+ private AccumuloClient client;
+
+ protected AccumuloRecordWriter(JobConf job)
+ throws AccumuloException, AccumuloSecurityException, IOException {
+ Level l = getLogLevel(job);
+ if (l != null)
+ log.setLevel(getLogLevel(job));
+ this.simulate = getSimulationMode(job);
+ this.createTables = canCreateTables(job);
+
+ if (simulate)
+ log.info("Simulating output only. No writes to tables will occur");
+
+ this.bws = new HashMap<>();
+
+ String tname = getDefaultTableName(job);
+ this.defaultTableName = (tname == null) ? null : new Text(tname);
+
+ if (!simulate) {
+ this.client = Accumulo.newClient().usingClientInfo(getClientInfo(job)).build();
+ mtbw = client.createMultiTableBatchWriter(getBatchWriterOptions(job));
+ }
+ }
+
+ /**
+ * Push a mutation into a table. If table is null, the defaultTable will be used. If
+ * {@link AccumuloOutputFormat#canCreateTables(JobConf)} is set, the table will be created if it
+ * does not exist. The table name must only contain alphanumerics and underscore.
+ */
+ @Override
+ public void write(Text table, Mutation mutation) throws IOException {
+ if (table == null || table.toString().isEmpty())
+ table = this.defaultTableName;
+
+ if (!simulate && table == null)
+ throw new IOException("No table or default table specified. Try simulation mode next time");
+
+ ++mutCount;
+ valCount += mutation.size();
+ printMutation(table, mutation);
+
+ if (simulate)
+ return;
+
+ if (!bws.containsKey(table))
+ try {
+ addTable(table);
+ } catch (final Exception e) {
+ log.error("Could not add table '" + table + "'", e);
+ throw new IOException(e);
+ }
+
+ try {
+ bws.get(table).addMutation(mutation);
+ } catch (MutationsRejectedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
+ if (simulate) {
+ log.info("Simulating adding table: " + tableName);
+ return;
+ }
+
+ log.debug("Adding table: " + tableName);
+ BatchWriter bw = null;
+ String table = tableName.toString();
+
+ if (createTables && !client.tableOperations().exists(table)) {
+ try {
+ client.tableOperations().create(table);
+ } catch (AccumuloSecurityException e) {
+ log.error("Accumulo security violation creating " + table, e);
+ throw e;
+ } catch (TableExistsException e) {
+ // Shouldn't happen
+ }
+ }
+
+ try {
+ bw = mtbw.getBatchWriter(table);
+ } catch (TableNotFoundException e) {
+ log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
+ throw new AccumuloException(e);
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw e;
+ }
+
+ if (bw != null)
+ bws.put(tableName, bw);
+ }
+
+ private int printMutation(Text table, Mutation m) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
+ for (ColumnUpdate cu : m.getUpdates()) {
+ log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()),
+ hexDump(cu.getColumnQualifier())));
+ log.trace(String.format("Table %s security: %s", table,
+ new ColumnVisibility(cu.getColumnVisibility()).toString()));
+ log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
+ }
+ }
+ return m.getUpdates().size();
+ }
+
+ private String hexDump(byte[] ba) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : ba) {
+ if ((b > 0x20) && (b < 0x7e))
+ sb.append((char) b);
+ else
+ sb.append(String.format("x%02x", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close(Reporter reporter) throws IOException {
+ log.debug("mutations written: " + mutCount + ", values written: " + valCount);
+ if (simulate)
+ return;
+
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getSecurityErrorCodes().size() >= 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<>();
+ for (Entry<TabletId,Set<SecurityErrorCode>> ke : e.getSecurityErrorCodes().entrySet()) {
+ String tableId = ke.getKey().getTableId().toString();
+ Set<SecurityErrorCode> secCodes = tables.get(tableId);
+ if (secCodes == null) {
+ secCodes = new HashSet<>();
+ tables.put(tableId, secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+
+ log.error("Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ log.error("Constraint violations : " + e.getConstraintViolationSummaries().size());
+ }
+ throw new IOException(e);
+ }
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
+ if (!isConnectorInfoSet(job))
+ throw new IOException("Connector info has not been set.");
+ try {
+ // if the instance isn't configured, it will complain here
+ AccumuloClient c = Accumulo.newClient().usingClientInfo(getClientInfo(job)).build();
+ String principal = getPrincipal(job);
+ AuthenticationToken token = getAuthenticationToken(job);
+ if (!c.securityOperations().authenticateUser(principal, token))
+ throw new IOException("Unable to authenticate user");
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(FileSystem ignored, JobConf job, String name,
+ Progressable progress) throws IOException {
+ try {
+ return new AccumuloRecordWriter(job);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloRowInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloRowInputFormat.java
new file mode 100644
index 0000000000..c9e4a213ce
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/AccumuloRowInputFormat.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.PeekingIterator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the source of data. This {@link InputFormat}
+ * provides row names as {@link Text} as keys, and a corresponding {@link PeekingIterator} as a
+ * value, which in turn makes the {@link Key}/{@link Value} pairs for that row available to the Map
+ * function.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloRowInputFormat#setClientInfo(JobConf, ClientInfo)}
+ * <li>{@link AccumuloRowInputFormat#setInputTableName(JobConf, String)}
+ * <li>{@link AccumuloRowInputFormat#setScanAuthorizations(JobConf, Authorizations)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloRowInputFormat
+ extends InputFormatBase<Text,PeekingIterator<Entry<Key,Value>>> {
+ @Override
+ public RecordReader<Text,PeekingIterator<Entry<Key,Value>>> getRecordReader(InputSplit split,
+ JobConf job, Reporter reporter) throws IOException {
+ log.setLevel(getLogLevel(job));
+ // @formatter:off
+ RecordReaderBase<Text,PeekingIterator<Entry<Key,Value>>> recordReader =
+ new RecordReaderBase<Text,PeekingIterator<Entry<Key,Value>>>() {
+ // @formatter:on
+ RowIterator rowIterator;
+
+ @Override
+ public void initialize(InputSplit inSplit, JobConf job) throws IOException {
+ super.initialize(inSplit, job);
+ rowIterator = new RowIterator(scannerIterator);
+ }
+
+ @Override
+ public boolean next(Text key, PeekingIterator<Entry<Key,Value>> value)
+ throws IOException {
+ if (!rowIterator.hasNext())
+ return false;
+ value.initialize(rowIterator.next());
+ numKeysRead = rowIterator.getKVCount();
+ key.set((currentKey = value.peek().getKey()).getRow());
+ return true;
+ }
+
+ @Override
+ public Text createKey() {
+ return new Text();
+ }
+
+ @Override
+ public PeekingIterator<Entry<Key,Value>> createValue() {
+ return new PeekingIterator<>();
+ }
+ };
+ recordReader.initialize(split, job);
+ return recordReader;
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/InputFormatBase.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/InputFormatBase.java
new file mode 100644
index 0000000000..677049e0a6
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/InputFormatBase.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * This abstract {@link InputFormat} class allows MapReduce jobs to use Accumulo as the source of
+ * K,V pairs.
+ * <p>
+ * Subclasses must implement a {@link #getRecordReader(InputSplit, JobConf, Reporter)} to provide a
+ * {@link RecordReader} for K,V.
+ * <p>
+ * A static base class, RecordReaderBase, is provided to retrieve Accumulo {@link Key}/{@link Value}
+ * pairs, but one must implement its {@link RecordReaderBase#next(Object, Object)} to transform them
+ * to the desired generic types K,V.
+ * <p>
+ * See {@link AccumuloInputFormat} for an example implementation.
+ */
+public abstract class InputFormatBase<K,V> extends AbstractInputFormat<K,V> {
+
+ /**
+ * Sets the name of the input table, over which this job will scan.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setInputTableName(JobConf job, String tableName) {
+ InputConfigurator.setInputTableName(CLASS, job, tableName);
+ }
+
+ /**
+ * Gets the table name from the configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the table name
+ * @since 1.5.0
+ * @see #setInputTableName(JobConf, String)
+ */
+ protected static String getInputTableName(JobConf job) {
+ return InputConfigurator.getInputTableName(CLASS, job);
+ }
+
+ /**
+ * Sets the input ranges to scan for this job. If not set, the entire table will be scanned.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param ranges
+ * the ranges that will be mapped over
+ * @see TableOperations#splitRangeByTablets(String, Range, int)
+ * @since 1.5.0
+ */
+ public static void setRanges(JobConf job, Collection<Range> ranges) {
+ InputConfigurator.setRanges(CLASS, job, ranges);
+ }
+
+ /**
+ * Gets the ranges to scan over from a job.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return the ranges
+ * @throws IOException
+ * if the ranges have been encoded improperly
+ * @since 1.5.0
+ * @see #setRanges(JobConf, Collection)
+ */
+ protected static List<Range> getRanges(JobConf job) throws IOException {
+ return InputConfigurator.getRanges(CLASS, job);
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param columnFamilyColumnQualifierPairs
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If
+ * the column qualifier is null, the entire column family is selected. An empty set is
+ * the default and is equivalent to scanning the all columns.
+ * @since 1.5.0
+ */
+ public static void fetchColumns(JobConf job,
+ Collection<Pair<Text,Text>> columnFamilyColumnQualifierPairs) {
+ InputConfigurator.fetchColumns(CLASS, job, columnFamilyColumnQualifierPairs);
+ }
+
+ /**
+ * Gets the columns to be mapped over from this job.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return a set of columns
+ * @since 1.5.0
+ * @see #fetchColumns(JobConf, Collection)
+ */
+ protected static Set<Pair<Text,Text>> getFetchedColumns(JobConf job) {
+ return InputConfigurator.getFetchedColumns(CLASS, job);
+ }
+
+ /**
+ * Encode an iterator on the input for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param cfg
+ * the configuration of the iterator
+ * @since 1.5.0
+ */
+ public static void addIterator(JobConf job, IteratorSetting cfg) {
+ InputConfigurator.addIterator(CLASS, job, cfg);
+ }
+
+ /**
+ * Gets a list of the iterator settings (for iterators to apply to a scanner) from this
+ * configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return a list of iterators
+ * @since 1.5.0
+ * @see #addIterator(JobConf, IteratorSetting)
+ */
+ protected static List<IteratorSetting> getIterators(JobConf job) {
+ return InputConfigurator.getIterators(CLASS, job);
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping
+ * ranges, then splits them to align with tablet boundaries. Disabling this feature will cause
+ * exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(JobConf, Collection)
+ * @since 1.5.0
+ */
+ public static void setAutoAdjustRanges(JobConf job, boolean enableFeature) {
+ InputConfigurator.setAutoAdjustRanges(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled. Must be enabled when
+ * {@link #setBatchScan(JobConf, boolean)} is true.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.5.0
+ * @see #setAutoAdjustRanges(JobConf, boolean)
+ */
+ protected static boolean getAutoAdjustRanges(JobConf job) {
+ return InputConfigurator.getAutoAdjustRanges(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setScanIsolation(JobConf job, boolean enableFeature) {
+ InputConfigurator.setScanIsolation(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setScanIsolation(JobConf, boolean)
+ */
+ protected static boolean isIsolated(JobConf job) {
+ return InputConfigurator.isIsolated(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link ClientSideIteratorScanner} in this job. Enabling this feature
+ * will cause the iterator stack to be constructed within the Map task, rather than within the
+ * Accumulo TServer. To use this feature, all classes needed for those iterators must be available
+ * on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setLocalIterators(JobConf job, boolean enableFeature) {
+ InputConfigurator.setLocalIterators(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setLocalIterators(JobConf, boolean)
+ */
+ protected static boolean usesLocalIterators(JobConf job) {
+ return InputConfigurator.usesLocalIterators(CLASS, job);
+ }
+
+ /**
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are
+ * scanned. This will make the map reduce job directly read the table's files. If the table is not
+ * offline, then the job will fail. If the table comes online during the map reduce job, it is
+ * likely that the job will fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in
+ * HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any
+ * iterators that are configured for the table will need to be on the mapper's classpath.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as
+ * the input table for a map reduce job. If you plan to map reduce over the data many times, it
+ * may be better to the compact the table, clone it, take it offline, and use the clone for all
+ * map reduce jobs. The reason to do this is that compaction will reduce each tablet in the table
+ * to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may
+ * see better read performance. Second, it will support speculative execution better. When reading
+ * an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setOfflineTableScan(JobConf job, boolean enableFeature) {
+ InputConfigurator.setOfflineTableScan(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setOfflineTableScan(JobConf, boolean)
+ */
+ protected static boolean isOfflineScan(JobConf job) {
+ return InputConfigurator.isOfflineScan(CLASS, job);
+ }
+
+ /**
+ * Controls the use of the {@link org.apache.accumulo.core.client.BatchScanner} in this job. Using
+ * this feature will group Ranges by their source tablet, producing an InputSplit per tablet
+ * rather than per Range. This batching helps to reduce overhead when querying a large number of
+ * small ranges. (ex: when doing quad-tree decomposition for spatial queries)
+ * <p>
+ * In order to achieve good locality of InputSplits this option always clips the input Ranges to
+ * tablet boundaries. This may result in one input Range contributing to several InputSplits.
+ * <p>
+ * Note: that the value of {@link #setAutoAdjustRanges(JobConf, boolean)} is ignored and is
+ * assumed to be true when BatchScan option is enabled.
+ * <p>
+ * This configuration is incompatible with:
+ * <ul>
+ * <li>{@link #setOfflineTableScan(JobConf, boolean)}</li>
+ * <li>{@link #setLocalIterators(JobConf, boolean)}</li>
+ * <li>{@link #setScanIsolation(JobConf, boolean)}</li>
+ * </ul>
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.7.0
+ */
+ public static void setBatchScan(JobConf job, boolean enableFeature) {
+ InputConfigurator.setBatchScan(CLASS, job, enableFeature);
+ }
+
+ /**
+ * Determines whether a configuration has the {@link org.apache.accumulo.core.client.BatchScanner}
+ * feature enabled.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @since 1.7.0
+ * @see #setBatchScan(JobConf, boolean)
+ */
+ public static boolean isBatchScan(JobConf job) {
+ return InputConfigurator.isBatchScan(CLASS, job);
+ }
+
+ /**
+ * Causes input format to read sample data. If sample data was created using a different
+ * configuration or a tables sampler configuration changes while reading data, then the input
+ * format will throw an error.
+ *
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param samplerConfig
+ * The sampler configuration that sample must have been created with inorder for reading
+ * sample data to succeed.
+ *
+ * @since 1.8.0
+ * @see ScannerBase#setSamplerConfiguration(SamplerConfiguration)
+ */
+ public static void setSamplerConfiguration(JobConf job, SamplerConfiguration samplerConfig) {
+ InputConfigurator.setSamplerConfiguration(CLASS, job, samplerConfig);
+ }
+
+ /**
+ * Set these execution hints on scanners created for input splits. See
+ * {@link ScannerBase#setExecutionHints(java.util.Map)}
+ *
+ * @since 2.0.0
+ */
+ public static void setExecutionHints(JobConf job, Map<String,String> hints) {
+ InputConfigurator.setExecutionHints(CLASS, job, hints);
+ }
+
+ protected abstract static class RecordReaderBase<K,V> extends AbstractRecordReader<K,V> {
+
+ @Override
+ protected List<IteratorSetting> jobIterators(JobConf job, String tableName) {
+ return getIterators(job);
+ }
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/RangeInputSplit.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/RangeInputSplit.java
new file mode 100644
index 0000000000..9d163d7db4
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapred/RangeInputSplit.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Range;
+import org.apache.hadoop.mapred.InputSplit;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+/**
+ * The Class RangeInputSplit. Encapsulates an Accumulo range for use in Map Reduce jobs.
+ */
+@SuppressFBWarnings(value = "NM_SAME_SIMPLE_NAME_AS_SUPERCLASS",
+ justification = "Intended to share code between mapred and mapreduce")
+public class RangeInputSplit extends org.apache.accumulo.hadoop.mapreduce.RangeInputSplit
+ implements InputSplit {
+
+ public RangeInputSplit() {
+ super();
+ }
+
+ public RangeInputSplit(RangeInputSplit split) throws IOException {
+ super(split);
+ }
+
+ protected RangeInputSplit(String table, String tableId, Range range, String[] locations) {
+ super(table, tableId, range, locations);
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AbstractInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AbstractInputFormat.java
new file mode 100644
index 0000000000..d0f3be1b33
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AbstractInputFormat.java
@@ -0,0 +1,818 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.ClientSideIteratorScanner;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.admin.SecurityOperations;
+import org.apache.accumulo.core.client.impl.AuthenticationTokenIdentifier;
+import org.apache.accumulo.core.client.impl.ClientContext;
+import org.apache.accumulo.core.client.impl.DelegationTokenImpl;
+import org.apache.accumulo.core.client.impl.OfflineScanner;
+import org.apache.accumulo.core.client.impl.ScannerImpl;
+import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.DelegationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.hadoopImpl.mapreduce.BatchInputSplit;
+import org.apache.accumulo.hadoopImpl.mapreduce.SplitUtils;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.ConfiguratorBase;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.security.token.Token;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * An abstract input format to provide shared methods common to all other input format classes. At
+ * the very least, any classes inheriting from this class will need to define their own
+ * {@link RecordReader}.
+ */
+public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V> {
+
+ protected static final Class<?> CLASS = AccumuloInputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the name of the classloader context on this scanner
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param context
+ * name of the classloader context
+ * @since 1.8.0
+ */
+ public static void setClassLoaderContext(Job job, String context) {
+ InputConfigurator.setClassLoaderContext(CLASS, job.getConfiguration(), context);
+ }
+
+ /**
+ * Returns the name of the current classloader context set on this scanner
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @return name of the current context
+ * @since 1.8.0
+ */
+ public static String getClassLoaderContext(JobContext job) {
+ return InputConfigurator.getClassLoaderContext(CLASS, job.getConfiguration());
+ }
+
+ /**
+ * Sets connection information needed to communicate with Accumulo for this job
+ *
+ * @param job
+ * Hadoop job instance to be configured
+ * @param info
+ * Connection information for Accumulo
+ * @since 2.0.0
+ */
+ public static void setClientInfo(Job job, ClientInfo info) {
+ ClientInfo inputInfo = InputConfigurator.updateToken(job.getCredentials(), info);
+ InputConfigurator.setClientInfo(CLASS, job.getConfiguration(), inputInfo);
+ }
+
+ /**
+ * Set Accumulo client properties file used to connect to Accumulo
+ *
+ * @param job
+ * Hadoop job to be configured
+ * @param clientPropsFile
+ * URL (hdfs:// or http://) to Accumulo client properties file
+ * @since 2.0.0
+ */
+ public static void setClientPropertiesFile(Job job, String clientPropsFile) {
+ InputConfigurator.setClientPropertiesFile(CLASS, job.getConfiguration(), clientPropsFile);
+ }
+
+ /**
+ * Gets the {@link ClientInfo} from the configuration
+ *
+ * @param context
+ * Hadoop job context
+ * @return ClientInfo
+ * @since 2.0.0
+ */
+ protected static ClientInfo getClientInfo(JobContext context) {
+ return InputConfigurator.getClientInfo(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> Some tokens, when serialized, divulge sensitive information in the
+ * configuration as a means to pass the token to MapReduce tasks. This information is BASE64
+ * encoded to provide a charset safe conversion to a string, but this conversion is not intended
+ * to be secure. {@link PasswordToken} is one example that is insecure in this way; however
+ * {@link DelegationToken}s, acquired using
+ * {@link SecurityOperations#getDelegationToken(DelegationTokenConfig)}, is not subject to this
+ * concern.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param token
+ * the user's password
+ * @since 1.5.0
+ * @deprecated since 2.0.0; use {@link #setClientInfo(Job, ClientInfo)} instead.
+ */
+ @Deprecated
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (token instanceof KerberosToken) {
+ log.info("Received KerberosToken, attempting to fetch DelegationToken");
+ try {
+ AccumuloClient client = Accumulo.newClient().usingClientInfo(getClientInfo(job))
+ .usingToken(principal, token).build();
+ token = client.securityOperations().getDelegationToken(new DelegationTokenConfig());
+ } catch (Exception e) {
+ log.warn("Failed to automatically obtain DelegationToken, "
+ + "Mappers/Reducers will likely fail to communicate with Accumulo", e);
+ }
+ }
+ // DelegationTokens can be passed securely from user to task without serializing insecurely in
+ // the configuration
+ if (token instanceof DelegationTokenImpl) {
+ DelegationTokenImpl delegationToken = (DelegationTokenImpl) token;
+
+ // Convert it into a Hadoop Token
+ AuthenticationTokenIdentifier identifier = delegationToken.getIdentifier();
+ Token<AuthenticationTokenIdentifier> hadoopToken = new Token<>(identifier.getBytes(),
+ delegationToken.getPassword(), identifier.getKind(), delegationToken.getServiceName());
+
+ // Add the Hadoop Token to the Job so it gets serialized and passed along.
+ job.getCredentials().addToken(hadoopToken.getService(), hadoopToken);
+ }
+
+ InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt
+ * to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission)
+ * @param tokenFile
+ * the path to the token file
+ * @since 1.6.0
+ * @deprecated since 2.0.0, use {@link #setClientPropertiesFile(Job, String)}
+ */
+ @Deprecated
+ public static void setConnectorInfo(Job job, String principal, String tokenFile)
+ throws AccumuloSecurityException {
+ setClientPropertiesFile(job, tokenFile);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return InputConfigurator.isConnectorInfoSet(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return InputConfigurator.getPrincipal(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the
+ * configuration, whichever was used when the job was configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ * @see #setConnectorInfo(Job, String, String)
+ */
+ protected static AuthenticationToken getAuthenticationToken(JobContext context) {
+ AuthenticationToken token = InputConfigurator.getAuthenticationToken(CLASS,
+ context.getConfiguration());
+ return ConfiguratorBase.unwrapAuthenticationToken(context, token);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ *
+ * @param clientConfig
+ * client configuration containing connection options
+ * @since 1.6.0
+ * @deprecated since 2.0.0; Use {@link #setClientInfo(Job, ClientInfo)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Job job,
+ org.apache.accumulo.core.client.ClientConfiguration clientConfig) {
+ InputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), clientConfig);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the
+ * configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @deprecated since 2.0.0, use {@link #getClientInfo(JobContext)} instead
+ */
+ @Deprecated
+ protected static org.apache.accumulo.core.client.Instance getInstance(JobContext context) {
+ return InputConfigurator.getInstance(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ InputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return InputConfigurator.getLogLevel(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the {@link org.apache.accumulo.core.security.Authorizations} used to scan. Must be a
+ * subset of the user's authorization. Defaults to the empty set.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param auths
+ * the user's authorizations
+ */
+ public static void setScanAuthorizations(Job job, Authorizations auths) {
+ InputConfigurator.setScanAuthorizations(CLASS, job.getConfiguration(), auths);
+ }
+
+ /**
+ * Gets the authorizations to set for the scans from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the Accumulo scan authorizations
+ * @since 1.5.0
+ * @see #setScanAuthorizations(Job, Authorizations)
+ */
+ protected static Authorizations getScanAuthorizations(JobContext context) {
+ return InputConfigurator.getScanAuthorizations(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Fetches all {@link InputTableConfig}s that have been set on the given job.
+ *
+ * @param context
+ * the Hadoop job instance to be configured
+ * @return the {@link InputTableConfig} objects for the job
+ * @since 1.6.0
+ */
+ protected static Map<String,InputTableConfig> getInputTableConfigs(JobContext context) {
+ return InputConfigurator.getInputTableConfigs(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Fetches a {@link InputTableConfig} that has been set on the configuration for a specific table.
+ *
+ * <p>
+ * null is returned in the event that the table doesn't exist.
+ *
+ * @param context
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table name for which to grab the config object
+ * @return the {@link InputTableConfig} for the given table
+ * @since 1.6.0
+ */
+ protected static InputTableConfig getInputTableConfig(JobContext context, String tableName) {
+ return InputConfigurator.getInputTableConfig(CLASS, context.getConfiguration(), tableName);
+ }
+
+ // InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
+ /**
+ * Check whether a configuration is fully configured to be used with an Accumulo
+ * {@link org.apache.hadoop.mapreduce.InputFormat}.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @throws java.io.IOException
+ * if the context is improperly configured
+ * @since 1.5.0
+ */
+ protected static void validateOptions(JobContext context) throws IOException {
+ AccumuloClient client = InputConfigurator.getClient(CLASS, context.getConfiguration());
+ InputConfigurator.validatePermissions(CLASS, context.getConfiguration(), client);
+ }
+
+ /**
+ * Construct the ClientConfiguration given the provided context.
+ *
+ * @param context
+ * The Job
+ * @return The ClientConfiguration
+ * @since 1.7.0
+ * @deprecated since 2.0.0; use {@link #getClientInfo(JobContext)} instead
+ */
+ @Deprecated
+ protected static org.apache.accumulo.core.client.ClientConfiguration getClientConfiguration(
+ JobContext context) {
+ return InputConfigurator.getClientConfiguration(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * An abstract base class to be used to create {@link org.apache.hadoop.mapreduce.RecordReader}
+ * instances that convert from Accumulo
+ * {@link org.apache.accumulo.core.data.Key}/{@link org.apache.accumulo.core.data.Value} pairs to
+ * the user's K/V types.
+ *
+ * Subclasses must implement {@link #nextKeyValue()} and use it to update the following variables:
+ * <ul>
+ * <li>K {@link #currentK}</li>
+ * <li>V {@link #currentV}</li>
+ * <li>Key {@link #currentKey} (used for progress reporting)</li>
+ * <li>int {@link #numKeysRead} (used for progress reporting)</li>
+ * </ul>
+ */
+ protected abstract static class AbstractRecordReader<K,V> extends RecordReader<K,V> {
+ protected long numKeysRead;
+ protected Iterator<Map.Entry<Key,Value>> scannerIterator;
+ protected ScannerBase scannerBase;
+ protected RangeInputSplit split;
+
+ /**
+ * Extracts Iterators settings from the context to be used by RecordReader.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @param tableName
+ * the table name for which the scanner is configured
+ * @return List of iterator settings for given table
+ * @since 1.7.0
+ */
+ protected abstract List<IteratorSetting> contextIterators(TaskAttemptContext context,
+ String tableName);
+
+ /**
+ * Configures the iterators on a scanner for the given table name. Will attempt to use
+ * configuration from the InputSplit, on failure will try to extract them from
+ * TaskAttemptContext.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @param tableName
+ * the table name for which the scanner is configured
+ * @param scanner
+ * the scanner for which to configure the iterators
+ * @param split
+ * InputSplit containing configurations
+ * @since 1.7.0
+ */
+ private void setupIterators(TaskAttemptContext context, ScannerBase scanner, String tableName,
+ RangeInputSplit split) {
+ List<IteratorSetting> iterators = null;
+
+ if (null == split) {
+ iterators = contextIterators(context, tableName);
+ } else {
+ iterators = split.getIterators();
+ if (null == iterators) {
+ iterators = contextIterators(context, tableName);
+ }
+ }
+
+ for (IteratorSetting iterator : iterators)
+ scanner.addScanIterator(iterator);
+ }
+
+ @Override
+ public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
+
+ split = (RangeInputSplit) inSplit;
+ log.debug("Initializing input split: " + split);
+
+ ClientInfo info = getClientInfo(attempt);
+ ClientContext context = new ClientContext(info);
+ AccumuloClient client;
+ try {
+ client = context.getClient();
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw new IllegalStateException(e);
+ }
+ Authorizations authorizations = getScanAuthorizations(attempt);
+ String classLoaderContext = getClassLoaderContext(attempt);
+ String table = split.getTableName();
+
+ // in case the table name changed, we can still use the previous name for terms of
+ // configuration,
+ // but the scanner will use the table id resolved at job setup time
+ InputTableConfig tableConfig = getInputTableConfig(attempt, split.getTableName());
+
+ log.debug("Creating client with user: " + info.getPrincipal());
+ log.debug("Creating scanner for table: " + table);
+ log.debug("Authorizations are: " + authorizations);
+
+ if (split instanceof BatchInputSplit) {
+ BatchInputSplit batchSplit = (BatchInputSplit) split;
+
+ BatchScanner scanner;
+ try {
+ // Note: BatchScanner will use at most one thread per tablet, currently BatchInputSplit
+ // will not span tablets
+ int scanThreads = 1;
+ scanner = client.createBatchScanner(split.getTableName(), authorizations, scanThreads);
+ setupIterators(attempt, scanner, split.getTableName(), split);
+ if (null != classLoaderContext) {
+ scanner.setClassLoaderContext(classLoaderContext);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ scanner.setRanges(batchSplit.getRanges());
+ scannerBase = scanner;
+ } else {
+ Scanner scanner;
+
+ Boolean isOffline = split.isOffline();
+ if (null == isOffline) {
+ isOffline = tableConfig.isOfflineScan();
+ }
+
+ Boolean isIsolated = split.isIsolatedScan();
+ if (null == isIsolated) {
+ isIsolated = tableConfig.shouldUseIsolatedScanners();
+ }
+
+ Boolean usesLocalIterators = split.usesLocalIterators();
+ if (null == usesLocalIterators) {
+ usesLocalIterators = tableConfig.shouldUseLocalIterators();
+ }
+
+ try {
+ if (isOffline) {
+ scanner = new OfflineScanner(context, Table.ID.of(split.getTableId()), authorizations);
+ } else {
+ // Not using public API to create scanner so that we can use table ID
+ // Table ID is used in case of renames during M/R job
+ scanner = new ScannerImpl(context, Table.ID.of(split.getTableId()), authorizations);
+ }
+ if (isIsolated) {
+ log.info("Creating isolated scanner");
+ scanner = new IsolatedScanner(scanner);
+ }
+ if (usesLocalIterators) {
+ log.info("Using local iterators");
+ scanner = new ClientSideIteratorScanner(scanner);
+ }
+
+ setupIterators(attempt, scanner, split.getTableName(), split);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ scanner.setRange(split.getRange());
+ scannerBase = scanner;
+
+ }
+
+ Collection<Pair<Text,Text>> columns = split.getFetchedColumns();
+ if (null == columns) {
+ columns = tableConfig.getFetchedColumns();
+ }
+
+ // setup a scanner within the bounds of this split
+ for (Pair<Text,Text> c : columns) {
+ if (c.getSecond() != null) {
+ log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
+ scannerBase.fetchColumn(c.getFirst(), c.getSecond());
+ } else {
+ log.debug("Fetching column family " + c.getFirst());
+ scannerBase.fetchColumnFamily(c.getFirst());
+ }
+ }
+
+ SamplerConfiguration samplerConfig = split.getSamplerConfiguration();
+ if (null == samplerConfig) {
+ samplerConfig = tableConfig.getSamplerConfiguration();
+ }
+
+ if (samplerConfig != null) {
+ scannerBase.setSamplerConfiguration(samplerConfig);
+ }
+
+ Map<String,String> executionHints = split.getExecutionHints();
+ if (executionHints == null || executionHints.isEmpty()) {
+ executionHints = tableConfig.getExecutionHints();
+ }
+
+ if (executionHints != null) {
+ scannerBase.setExecutionHints(executionHints);
+ }
+
+ scannerIterator = scannerBase.iterator();
+ numKeysRead = 0;
+ }
+
+ @Override
+ public void close() {
+ if (null != scannerBase) {
+ scannerBase.close();
+ }
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (numKeysRead > 0 && currentKey == null)
+ return 1.0f;
+ return split.getProgress(currentKey);
+ }
+
+ /**
+ * The Key that should be returned to the client
+ */
+ protected K currentK = null;
+
+ /**
+ * The Value that should be return to the client
+ */
+ protected V currentV = null;
+
+ /**
+ * The Key that is used to determine progress in the current InputSplit. It is not returned to
+ * the client and is only used internally
+ */
+ protected Key currentKey = null;
+
+ @Override
+ public K getCurrentKey() throws IOException, InterruptedException {
+ return currentK;
+ }
+
+ @Override
+ public V getCurrentValue() throws IOException, InterruptedException {
+ return currentV;
+ }
+ }
+
+ Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobContext context, Table.ID tableId,
+ List<Range> ranges)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ClientContext clientContext = new ClientContext(getClientInfo(context));
+ return InputConfigurator.binOffline(tableId, ranges, clientContext);
+ }
+
+ /**
+ * Gets the splits of the tables that have been set on the job by reading the metadata table for
+ * the specified ranges.
+ *
+ * @return the splits from the tables based on the ranges.
+ * @throws java.io.IOException
+ * if a table set on the job doesn't exist or an error occurs initializing the tablet
+ * locator
+ */
+ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException {
+ Level logLevel = getLogLevel(context);
+ log.setLevel(logLevel);
+ validateOptions(context);
+ Random random = new SecureRandom();
+ LinkedList<InputSplit> splits = new LinkedList<>();
+ Map<String,InputTableConfig> tableConfigs = getInputTableConfigs(context);
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
+
+ String tableName = tableConfigEntry.getKey();
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+
+ ClientContext clientContext = new ClientContext(getClientInfo(context));
+ Table.ID tableId;
+ // resolve table name to id once, and use id from this point forward
+ try {
+ tableId = Tables.getTableId(clientContext, tableName);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+
+ boolean batchScan = InputConfigurator.isBatchScan(CLASS, context.getConfiguration());
+ boolean supportBatchScan = !(tableConfig.isOfflineScan()
+ || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
+ if (batchScan && !supportBatchScan)
+ throw new IllegalArgumentException("BatchScanner optimization not available for offline"
+ + " scan, isolated, or local iterators");
+
+ boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
+ if (batchScan && !autoAdjust)
+ throw new IllegalArgumentException(
+ "AutoAdjustRanges must be enabled when using BatchScanner optimization");
+
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges())
+ : tableConfig.getRanges();
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<>(1);
+ ranges.add(new Range());
+ }
+
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
+ TabletLocator tl;
+ try {
+ if (tableConfig.isOfflineScan()) {
+ binnedRanges = binOfflineTable(context, tableId, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ // sleep randomly between 100 and 200 ms
+ sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
+ binnedRanges = binOfflineTable(context, tableId, ranges);
+
+ }
+ } else {
+ tl = InputConfigurator.getTabletLocator(CLASS, context.getConfiguration(), tableId);
+ // its possible that the cache could contain complete, but old information about a tables
+ // tablets... so clear it
+ tl.invalidateCache();
+
+ while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) {
+ String tableIdStr = tableId.canonicalID();
+ if (!Tables.exists(clientContext, tableId))
+ throw new TableDeletedException(tableIdStr);
+ if (Tables.getTableState(clientContext, tableId) == TableState.OFFLINE)
+ throw new TableOfflineException(Tables.getTableOfflineMsg(clientContext, tableId));
+ binnedRanges.clear();
+ log.warn("Unable to locate bins for specified ranges. Retrying.");
+ // sleep randomly between 100 and 200 ms
+ sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
+ tl.invalidateCache();
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // all of this code will add either range per each locations or split ranges and add
+ // range-location split
+ // Map from Range to Array of Locations, we only use this if we're don't split
+ HashMap<Range,ArrayList<String>> splitsToAdd = null;
+
+ if (!autoAdjust)
+ splitsToAdd = new HashMap<>();
+
+ HashMap<String,String> hostNameCache = new HashMap<>();
+ for (Map.Entry<String,Map<KeyExtent,List<Range>>> tserverBin : binnedRanges.entrySet()) {
+ String ip = tserverBin.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getCanonicalHostName();
+ hostNameCache.put(ip, location);
+ }
+ for (Map.Entry<KeyExtent,List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
+ Range ke = extentRanges.getKey().toDataRange();
+ if (batchScan) {
+ // group ranges by tablet to be read by a BatchScanner
+ ArrayList<Range> clippedRanges = new ArrayList<>();
+ for (Range r : extentRanges.getValue())
+ clippedRanges.add(ke.clip(r));
+ BatchInputSplit split = new BatchInputSplit(tableName, tableId, clippedRanges,
+ new String[] {location});
+ SplitUtils.updateSplit(split, tableConfig, logLevel);
+
+ splits.add(split);
+ } else {
+ // not grouping by tablet
+ for (Range r : extentRanges.getValue()) {
+ if (autoAdjust) {
+ // divide ranges into smaller ranges, based on the tablets
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonicalID(),
+ ke.clip(r), new String[] {location});
+ SplitUtils.updateSplit(split, tableConfig, logLevel);
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+ splits.add(split);
+ } else {
+ // don't divide ranges
+ ArrayList<String> locations = splitsToAdd.get(r);
+ if (locations == null)
+ locations = new ArrayList<>(1);
+ locations.add(location);
+ splitsToAdd.put(r, locations);
+ }
+ }
+ }
+ }
+ }
+
+ if (!autoAdjust)
+ for (Map.Entry<Range,ArrayList<String>> entry : splitsToAdd.entrySet()) {
+ RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonicalID(),
+ entry.getKey(), entry.getValue().toArray(new String[0]));
+ SplitUtils.updateSplit(split, tableConfig, logLevel);
+ split.setOffline(tableConfig.isOfflineScan());
+ split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
+ split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
+
+ splits.add(split);
+ }
+ }
+ return splits;
+ }
+
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloFileOutputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloFileOutputFormat.java
new file mode 100644
index 0000000000..a47c5aa8fa
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloFileOutputFormat.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.accumulo.core.client.rfile.RFile;
+import org.apache.accumulo.core.client.rfile.RFileWriter;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.client.summary.Summarizer;
+import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.ConfiguratorBase;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.FileOutputConfigurator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to write output in the Accumulo data file format.<br>
+ * Care should be taken to write only sorted data (sorted by {@link Key}), as this is an important
+ * requirement of Accumulo data files.
+ *
+ * <p>
+ * The output path to be created must be specified via
+ * {@link AccumuloFileOutputFormat#setOutputPath(Job, Path)}. This is inherited from
+ * {@link FileOutputFormat#setOutputPath(Job, Path)}. Other methods from {@link FileOutputFormat}
+ * are not supported and may be ignored or cause failures. Using other Hadoop configuration options
+ * that affect the behavior of the underlying files directly in the Job's configuration may work,
+ * but are not directly supported at this time.
+ */
+public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
+
+ private static final Class<?> CLASS = AccumuloFileOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Sets the compression type to use for data blocks. Specifying a compression may require
+ * additional libraries to be available to your Job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param compressionType
+ * one of "none", "gz", "lzo", or "snappy"
+ * @since 1.5.0
+ */
+ public static void setCompressionType(Job job, String compressionType) {
+ FileOutputConfigurator.setCompressionType(CLASS, job.getConfiguration(), compressionType);
+ }
+
+ /**
+ * Sets the size for data blocks within each file.<br>
+ * Data blocks are a span of key/value pairs stored in the file that are compressed and indexed as
+ * a group.
+ *
+ * <p>
+ * Making this value smaller may increase seek performance, but at the cost of increasing the size
+ * of the indexes (which can also affect seek performance).
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param dataBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setDataBlockSize(Job job, long dataBlockSize) {
+ FileOutputConfigurator.setDataBlockSize(CLASS, job.getConfiguration(), dataBlockSize);
+ }
+
+ /**
+ * Sets the size for file blocks in the file system; file blocks are managed, and replicated, by
+ * the underlying file system.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param fileBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setFileBlockSize(Job job, long fileBlockSize) {
+ FileOutputConfigurator.setFileBlockSize(CLASS, job.getConfiguration(), fileBlockSize);
+ }
+
+ /**
+ * Sets the size for index blocks within each file; smaller blocks means a deeper index hierarchy
+ * within the file, while larger blocks mean a more shallow index hierarchy within the file. This
+ * can affect the performance of queries.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param indexBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setIndexBlockSize(Job job, long indexBlockSize) {
+ FileOutputConfigurator.setIndexBlockSize(CLASS, job.getConfiguration(), indexBlockSize);
+ }
+
+ /**
+ * Sets the file system replication factor for the resulting file, overriding the file system
+ * default.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param replication
+ * the number of replicas for produced files
+ * @since 1.5.0
+ */
+ public static void setReplication(Job job, int replication) {
+ FileOutputConfigurator.setReplication(CLASS, job.getConfiguration(), replication);
+ }
+
+ /**
+ * Specify a sampler to be used when writing out data. This will result in the output file having
+ * sample data.
+ *
+ * @param job
+ * The Hadoop job instance to be configured
+ * @param samplerConfig
+ * The configuration for creating sample data in the output file.
+ * @since 1.8.0
+ */
+
+ public static void setSampler(Job job, SamplerConfiguration samplerConfig) {
+ FileOutputConfigurator.setSampler(CLASS, job.getConfiguration(), samplerConfig);
+ }
+
+ /**
+ * Specifies a list of summarizer configurations to create summary data in the output file. Each
+ * Key Value written will be passed to the configured {@link Summarizer}'s.
+ *
+ * @param job
+ * The Hadoop job instance to be configured
+ * @param summarizerConfigs
+ * summarizer configurations
+ * @since 2.0.0
+ */
+ public static void setSummarizers(Job job, SummarizerConfiguration... summarizerConfigs) {
+ FileOutputConfigurator.setSummarizers(CLASS, job.getConfiguration(), summarizerConfigs);
+ }
+
+ @Override
+ public RecordWriter<Key,Value> getRecordWriter(TaskAttemptContext context) throws IOException {
+ // get the path of the temporary output file
+ final Configuration conf = context.getConfiguration();
+ final AccumuloConfiguration acuConf = FileOutputConfigurator.getAccumuloConfiguration(CLASS,
+ context.getConfiguration());
+
+ final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
+ final Path file = this.getDefaultWorkFile(context, "." + extension);
+ final int visCacheSize = ConfiguratorBase.getVisibilityCacheSize(conf);
+
+ return new RecordWriter<Key,Value>() {
+ RFileWriter out = null;
+
+ @Override
+ public void close(TaskAttemptContext context) throws IOException {
+ if (out != null)
+ out.close();
+ }
+
+ @Override
+ public void write(Key key, Value value) throws IOException {
+ if (out == null) {
+ out = RFile.newWriter().to(file.toString()).withFileSystem(file.getFileSystem(conf))
+ .withTableProperties(acuConf).withVisibilityCacheSize(visCacheSize).build();
+ out.startDefaultLocalityGroup();
+ }
+ out.append(key, value);
+ }
+ };
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java
new file mode 100644
index 0000000000..d4fe4274d0
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormat.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.log4j.Level;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the source of data. This {@link InputFormat}
+ * provides keys and values of type {@link Key} and {@link Value} to the Map function.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloInputFormat#setClientInfo(Job, ClientInfo)}
+ * <li>{@link AccumuloInputFormat#setInputTableName(Job, String)}
+ * <li>{@link AccumuloInputFormat#setScanAuthorizations(Job, Authorizations)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloInputFormat extends InputFormatBase<Key,Value> {
+
+ @Override
+ public RecordReader<Key,Value> createRecordReader(InputSplit split, TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ log.setLevel(getLogLevel(context));
+
+ // Override the log level from the configuration as if the InputSplit has one it's the more
+ // correct one to use.
+ if (split instanceof RangeInputSplit) {
+ RangeInputSplit accSplit = (RangeInputSplit) split;
+ Level level = accSplit.getLogLevel();
+ if (null != level) {
+ log.setLevel(level);
+ }
+ } else {
+ throw new IllegalArgumentException("No RecordReader for " + split.getClass());
+ }
+
+ return new RecordReaderBase<Key,Value>() {
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ if (scannerIterator.hasNext()) {
+ ++numKeysRead;
+ Entry<Key,Value> entry = scannerIterator.next();
+ currentK = currentKey = entry.getKey();
+ currentV = entry.getValue();
+ if (log.isTraceEnabled())
+ log.trace("Processing key/value pair: " + DefaultFormatter.formatEntry(entry, true));
+ return true;
+ }
+ return false;
+ }
+ };
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloMultiTableInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloMultiTableInputFormat.java
new file mode 100644
index 0000000000..b460071a97
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloMultiTableInputFormat.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import static java.util.Objects.requireNonNull;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * This class allows MapReduce jobs to use multiple Accumulo tables as the source of data. This
+ * {@link org.apache.hadoop.mapreduce.InputFormat} provides keys and values of type {@link Key} and
+ * {@link Value} to the Map function.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloMultiTableInputFormat#setClientInfo(Job, ClientInfo)}
+ * <li>{@link AccumuloMultiTableInputFormat#setScanAuthorizations(Job, Authorizations)}
+ * <li>{@link AccumuloMultiTableInputFormat#setInputTableConfigs(Job, Map)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloMultiTableInputFormat extends AbstractInputFormat<Key,Value> {
+
+ /**
+ * Sets the {@link InputTableConfig} objects on the given Hadoop configuration
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param configs
+ * the table query configs to be set on the configuration.
+ * @since 1.6.0
+ */
+ public static void setInputTableConfigs(Job job, Map<String,InputTableConfig> configs) {
+ requireNonNull(configs);
+ InputConfigurator.setInputTableConfigs(CLASS, job.getConfiguration(), configs);
+ }
+
+ @Override
+ public RecordReader<Key,Value> createRecordReader(InputSplit inputSplit,
+ TaskAttemptContext context) throws IOException, InterruptedException {
+ log.setLevel(getLogLevel(context));
+ return new AbstractRecordReader<Key,Value>() {
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ if (scannerIterator.hasNext()) {
+ ++numKeysRead;
+ Map.Entry<Key,Value> entry = scannerIterator.next();
+ currentK = currentKey = entry.getKey();
+ currentV = entry.getValue();
+ if (log.isTraceEnabled())
+ log.trace("Processing key/value pair: " + DefaultFormatter.formatEntry(entry, true));
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ protected List<IteratorSetting> contextIterators(TaskAttemptContext context,
+ String tableName) {
+ return getInputTableConfig(context, tableName).getIterators();
+ }
+ };
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloOutputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloOutputFormat.java
new file mode 100644
index 0000000000..62364240e6
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloOutputFormat.java
@@ -0,0 +1,595 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.admin.SecurityOperations;
+import org.apache.accumulo.core.client.impl.AuthenticationTokenIdentifier;
+import org.apache.accumulo.core.client.impl.DelegationTokenImpl;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.DelegationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.TabletId;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.ConfiguratorBase;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.OutputConfigurator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.security.token.Token;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the sink for data. This {@link OutputFormat}
+ * accepts keys and values of type {@link Text} (for a table name) and {@link Mutation} from the Map
+ * and Reduce functions.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloOutputFormat#setClientInfo(Job, ClientInfo)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloOutputFormat extends OutputFormat<Text,Mutation> {
+
+ private static final Class<?> CLASS = AccumuloOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * Set the connection information needed to communicate with Accumulo in this job.
+ *
+ * @param job
+ * Hadoop job to be configured
+ * @param info
+ * Accumulo connection information
+ * @since 2.0.0
+ */
+ public static void setClientInfo(Job job, ClientInfo info) {
+ OutputConfigurator.setClientInfo(CLASS, job.getConfiguration(), info);
+ }
+
+ /**
+ * Get connection information from this job
+ *
+ * @param context
+ * Hadoop job context
+ * @return {@link ClientInfo}
+ *
+ * @since 2.0.0
+ */
+ protected static ClientInfo getClientInfo(JobContext context) {
+ return OutputConfigurator.getClientInfo(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Set Accumulo client properties file used to connect to Accumulo
+ *
+ * @param job
+ * Hadoop job to be configured
+ * @param clientPropsFile
+ * URL to Accumulo client properties file
+ * @since 2.0.0
+ */
+ public static void setClientPropertiesFile(Job job, String clientPropsFile) {
+ OutputConfigurator.setClientPropertiesFile(CLASS, job.getConfiguration(), clientPropsFile);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> Some tokens, when serialized, divulge sensitive information in the
+ * configuration as a means to pass the token to MapReduce tasks. This information is BASE64
+ * encoded to provide a charset safe conversion to a string, but this conversion is not intended
+ * to be secure. {@link PasswordToken} is one example that is insecure in this way; however
+ * {@link DelegationToken}s, acquired using
+ * {@link SecurityOperations#getDelegationToken(DelegationTokenConfig)}, is not subject to this
+ * concern.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if
+ * {@link #setCreateTables(Job, boolean)} is set to true)
+ * @param token
+ * the user's password
+ * @since 1.5.0
+ * @deprecated since 2.0.0, replaced by {@link #setClientInfo(Job, ClientInfo)}
+ */
+ @Deprecated
+ public static void setConnectorInfo(Job job, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ // DelegationTokens can be passed securely from user to task without serializing insecurely in
+ // the configuration
+ if (token instanceof DelegationTokenImpl) {
+ DelegationTokenImpl delegationToken = (DelegationTokenImpl) token;
+
+ // Convert it into a Hadoop Token
+ AuthenticationTokenIdentifier identifier = delegationToken.getIdentifier();
+ Token<AuthenticationTokenIdentifier> hadoopToken = new Token<>(identifier.getBytes(),
+ delegationToken.getPassword(), identifier.getKind(), delegationToken.getServiceName());
+
+ // Add the Hadoop Token to the Job so it gets serialized and passed along.
+ job.getCredentials().addToken(hadoopToken.getService(), hadoopToken);
+ }
+
+ OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an attempt
+ * to be more secure than storing it in the Configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param principal
+ * a valid Accumulo user name (user must have Table.CREATE permission if
+ * {@link #setCreateTables(Job, boolean)} is set to true)
+ * @param tokenFile
+ * the path to the token file
+ * @since 1.6.0
+ * @deprecated since 2.0.0, replaced by {@link #setClientPropertiesFile(Job, String)}
+ */
+ @Deprecated
+ public static void setConnectorInfo(Job job, String principal, String tokenFile)
+ throws AccumuloSecurityException {
+ setClientPropertiesFile(job, tokenFile);
+ }
+
+ /**
+ * Determines if the connector has been configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the connector has been configured, false otherwise
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static Boolean isConnectorInfoSet(JobContext context) {
+ return OutputConfigurator.isConnectorInfoSet(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the user name
+ * @since 1.5.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ */
+ protected static String getPrincipal(JobContext context) {
+ return OutputConfigurator.getPrincipal(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the
+ * configuration, whichever was used when the job was configured.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Job, String, AuthenticationToken)
+ * @see #setConnectorInfo(Job, String, String)
+ */
+ protected static AuthenticationToken getAuthenticationToken(JobContext context) {
+ AuthenticationToken token = OutputConfigurator.getAuthenticationToken(CLASS,
+ context.getConfiguration());
+ return ConfiguratorBase.unwrapAuthenticationToken(context, token);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ *
+ * @param clientConfig
+ * client configuration for specifying connection timeouts, SSL connection options, etc.
+ * @since 1.6.0
+ * @deprecated since 2.0.0; Use {@link #setClientInfo(Job, ClientInfo)} instead.
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Job job,
+ org.apache.accumulo.core.client.ClientConfiguration clientConfig) {
+ OutputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), clientConfig);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the
+ * configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return an Accumulo instance
+ * @since 1.5.0
+ * @deprecated since 2.0.0; Use {@link #getClientInfo(JobContext)} instead.
+ */
+ @Deprecated
+ protected static org.apache.accumulo.core.client.Instance getInstance(JobContext context) {
+ return OutputConfigurator.getInstance(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param level
+ * the logging level
+ * @since 1.5.0
+ */
+ public static void setLogLevel(Job job, Level level) {
+ OutputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the log level
+ * @since 1.5.0
+ * @see #setLogLevel(Job, Level)
+ */
+ protected static Level getLogLevel(JobContext context) {
+ return OutputConfigurator.getLogLevel(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given
+ * mutation. Table names can only be alpha-numeric and underscores.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.5.0
+ */
+ public static void setDefaultTableName(Job job, String tableName) {
+ OutputConfigurator.setDefaultTableName(CLASS, job.getConfiguration(), tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the default table name
+ * @since 1.5.0
+ * @see #setDefaultTableName(Job, String)
+ */
+ protected static String getDefaultTableName(JobContext context) {
+ return OutputConfigurator.getDefaultTableName(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new
+ * {@link BatchWriterConfig}, with sensible built-in defaults is used. Setting the configuration
+ * multiple times overwrites any previous configuration.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.5.0
+ */
+ public static void setBatchWriterOptions(Job job, BatchWriterConfig bwConfig) {
+ OutputConfigurator.setBatchWriterOptions(CLASS, job.getConfiguration(), bwConfig);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return the configuration object
+ * @since 1.5.0
+ * @see #setBatchWriterOptions(Job, BatchWriterConfig)
+ */
+ protected static BatchWriterConfig getBatchWriterOptions(JobContext context) {
+ return OutputConfigurator.getBatchWriterOptions(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric
+ * and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setCreateTables(Job job, boolean enableFeature) {
+ OutputConfigurator.setCreateTables(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.5.0
+ * @see #setCreateTables(Job, boolean)
+ */
+ protected static Boolean canCreateTables(JobContext context) {
+ return OutputConfigurator.canCreateTables(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is
+ * produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.5.0
+ */
+ public static void setSimulationMode(Job job, boolean enableFeature) {
+ OutputConfigurator.setSimulationMode(CLASS, job.getConfiguration(), enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param context
+ * the Hadoop context for the configured job
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.5.0
+ * @see #setSimulationMode(Job, boolean)
+ */
+ protected static Boolean getSimulationMode(JobContext context) {
+ return OutputConfigurator.getSimulationMode(CLASS, context.getConfiguration());
+ }
+
+ /**
+ * A base class to be used to create {@link RecordWriter} instances that write to Accumulo.
+ */
+ protected static class AccumuloRecordWriter extends RecordWriter<Text,Mutation> {
+ private MultiTableBatchWriter mtbw = null;
+ private HashMap<Text,BatchWriter> bws = null;
+ private Text defaultTableName = null;
+
+ private boolean simulate = false;
+ private boolean createTables = false;
+
+ private long mutCount = 0;
+ private long valCount = 0;
+
+ private AccumuloClient client;
+
+ protected AccumuloRecordWriter(TaskAttemptContext context)
+ throws AccumuloException, AccumuloSecurityException, IOException {
+ Level l = getLogLevel(context);
+ if (l != null)
+ log.setLevel(getLogLevel(context));
+ this.simulate = getSimulationMode(context);
+ this.createTables = canCreateTables(context);
+
+ if (simulate)
+ log.info("Simulating output only. No writes to tables will occur");
+
+ this.bws = new HashMap<>();
+
+ String tname = getDefaultTableName(context);
+ this.defaultTableName = (tname == null) ? null : new Text(tname);
+
+ if (!simulate) {
+ this.client = Accumulo.newClient().usingClientInfo(getClientInfo(context)).build();
+ mtbw = client.createMultiTableBatchWriter(getBatchWriterOptions(context));
+ }
+ }
+
+ /**
+ * Push a mutation into a table. If table is null, the defaultTable will be used. If
+ * {@link AccumuloOutputFormat#canCreateTables(JobContext)} is set, the table will be created if
+ * it does not exist. The table name must only contain alphanumerics and underscore.
+ */
+ @Override
+ public void write(Text table, Mutation mutation) throws IOException {
+ if (table == null || table.toString().isEmpty())
+ table = this.defaultTableName;
+
+ if (!simulate && table == null)
+ throw new IOException("No table or default table specified. Try simulation mode next time");
+
+ ++mutCount;
+ valCount += mutation.size();
+ printMutation(table, mutation);
+
+ if (simulate)
+ return;
+
+ if (!bws.containsKey(table))
+ try {
+ addTable(table);
+ } catch (Exception e) {
+ log.error("Could not add table '" + table + "'", e);
+ throw new IOException(e);
+ }
+
+ try {
+ bws.get(table).addMutation(mutation);
+ } catch (MutationsRejectedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
+ if (simulate) {
+ log.info("Simulating adding table: " + tableName);
+ return;
+ }
+
+ log.debug("Adding table: " + tableName);
+ BatchWriter bw = null;
+ String table = tableName.toString();
+
+ if (createTables && !client.tableOperations().exists(table)) {
+ try {
+ client.tableOperations().create(table);
+ } catch (AccumuloSecurityException e) {
+ log.error("Accumulo security violation creating " + table, e);
+ throw e;
+ } catch (TableExistsException e) {
+ // Shouldn't happen
+ }
+ }
+
+ try {
+ bw = mtbw.getBatchWriter(table);
+ } catch (TableNotFoundException e) {
+ log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
+ throw new AccumuloException(e);
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw e;
+ }
+
+ if (bw != null)
+ bws.put(tableName, bw);
+ }
+
+ private int printMutation(Text table, Mutation m) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
+ for (ColumnUpdate cu : m.getUpdates()) {
+ log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()),
+ hexDump(cu.getColumnQualifier())));
+ log.trace(String.format("Table %s security: %s", table,
+ new ColumnVisibility(cu.getColumnVisibility()).toString()));
+ log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
+ }
+ }
+ return m.getUpdates().size();
+ }
+
+ private String hexDump(byte[] ba) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : ba) {
+ if ((b > 0x20) && (b < 0x7e))
+ sb.append((char) b);
+ else
+ sb.append(String.format("x%02x", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close(TaskAttemptContext attempt) throws IOException, InterruptedException {
+ log.debug("mutations written: " + mutCount + ", values written: " + valCount);
+ if (simulate)
+ return;
+
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ if (e.getSecurityErrorCodes().size() >= 0) {
+ HashMap<String,Set<SecurityErrorCode>> tables = new HashMap<>();
+ for (Entry<TabletId,Set<SecurityErrorCode>> ke : e.getSecurityErrorCodes().entrySet()) {
+ String tableId = ke.getKey().getTableId().toString();
+ Set<SecurityErrorCode> secCodes = tables.get(tableId);
+ if (secCodes == null) {
+ secCodes = new HashSet<>();
+ tables.put(tableId, secCodes);
+ }
+ secCodes.addAll(ke.getValue());
+ }
+
+ log.error("Not authorized to write to tables : " + tables);
+ }
+
+ if (e.getConstraintViolationSummaries().size() > 0) {
+ log.error("Constraint violations : " + e.getConstraintViolationSummaries().size());
+ }
+ throw new IOException(e);
+ }
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext job) throws IOException {
+ if (!isConnectorInfoSet(job))
+ throw new IOException("Connector info has not been set.");
+ try {
+ // if the instance isn't configured, it will complain here
+ String principal = getPrincipal(job);
+ AuthenticationToken token = getAuthenticationToken(job);
+ AccumuloClient c = Accumulo.newClient().usingClientInfo(getClientInfo(job)).build();
+ if (!c.securityOperations().authenticateUser(principal, token))
+ throw new IOException("Unable to authenticate user");
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
+ return new NullOutputFormat<Text,Mutation>().getOutputCommitter(context);
+ }
+
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(TaskAttemptContext attempt)
+ throws IOException {
+ try {
+ return new AccumuloRecordWriter(attempt);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java
new file mode 100644
index 0000000000..d330604a04
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/AccumuloRowInputFormat.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.PeekingIterator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * This class allows MapReduce jobs to use Accumulo as the source of data. This {@link InputFormat}
+ * provides row names as {@link Text} as keys, and a corresponding {@link PeekingIterator} as a
+ * value, which in turn makes the {@link Key}/{@link Value} pairs for that row available to the Map
+ * function.
+ *
+ * The user must specify the following via static configurator methods:
+ *
+ * <ul>
+ * <li>{@link AccumuloRowInputFormat#setClientInfo(Job, ClientInfo)}
+ * <li>{@link AccumuloRowInputFormat#setInputTableName(Job, String)}
+ * <li>{@link AccumuloRowInputFormat#setScanAuthorizations(Job, Authorizations)}
+ * </ul>
+ *
+ * Other static methods are optional.
+ */
+public class AccumuloRowInputFormat
+ extends InputFormatBase<Text,PeekingIterator<Entry<Key,Value>>> {
+ @Override
+ public RecordReader<Text,PeekingIterator<Entry<Key,Value>>> createRecordReader(InputSplit split,
+ TaskAttemptContext context) throws IOException, InterruptedException {
+ log.setLevel(getLogLevel(context));
+ return new RecordReaderBase<Text,PeekingIterator<Entry<Key,Value>>>() {
+ RowIterator rowIterator;
+
+ @Override
+ public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
+ super.initialize(inSplit, attempt);
+ rowIterator = new RowIterator(scannerIterator);
+ currentK = new Text();
+ currentV = null;
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ if (!rowIterator.hasNext())
+ return false;
+ currentV = new PeekingIterator<>(rowIterator.next());
+ numKeysRead = rowIterator.getKVCount();
+ currentKey = currentV.peek().getKey();
+ currentK = new Text(currentKey.getRow());
+ return true;
+ }
+ };
+ }
+}
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBase.java
similarity index 99%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
rename to hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBase.java
index 9d1d042f97..868b81e004 100644
--- a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBase.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.accumulo.core.client.mapreduce;
+package org.apache.accumulo.hadoop.mapreduce;
import java.io.IOException;
import java.util.Collection;
@@ -27,12 +27,12 @@
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.ScannerBase;
import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.client.mapreduce.lib.impl.InputConfigurator;
import org.apache.accumulo.core.client.sample.SamplerConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputTableConfig.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputTableConfig.java
new file mode 100644
index 0000000000..0875398ab0
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputTableConfig.java
@@ -0,0 +1,448 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class to holds a batch scan configuration for a table. It contains all the properties needed
+ * to specify how rows should be returned from the table.
+ */
+public class InputTableConfig implements Writable {
+
+ private List<IteratorSetting> iterators;
+ private List<Range> ranges;
+ private Collection<Pair<Text,Text>> columns;
+
+ private boolean autoAdjustRanges = true;
+ private boolean useLocalIterators = false;
+ private boolean useIsolatedScanners = false;
+ private boolean offlineScan = false;
+ private SamplerConfiguration samplerConfig = null;
+ private Map<String,String> executionHints = Collections.emptyMap();
+
+ public InputTableConfig() {}
+
+ /**
+ * Creates a batch scan config object out of a previously serialized batch scan config object.
+ *
+ * @param input
+ * the data input of the serialized batch scan config
+ */
+ public InputTableConfig(DataInput input) throws IOException {
+ readFields(input);
+ }
+
+ /**
+ * Sets the input ranges to scan for all tables associated with this job. This will be added to
+ * any per-table ranges that have been set using
+ *
+ * @param ranges
+ * the ranges that will be mapped over
+ * @since 1.6.0
+ */
+ public InputTableConfig setRanges(List<Range> ranges) {
+ this.ranges = ranges;
+ return this;
+ }
+
+ /**
+ * Returns the ranges to be queried in the configuration
+ */
+ public List<Range> getRanges() {
+ return ranges != null ? ranges : new ArrayList<>();
+ }
+
+ /**
+ * Restricts the columns that will be mapped over for this job for the default input table.
+ *
+ * @param columns
+ * a pair of {@link Text} objects corresponding to column family and column qualifier. If
+ * the column qualifier is null, the entire column family is selected. An empty set is
+ * the default and is equivalent to scanning the all columns.
+ * @since 1.6.0
+ */
+ public InputTableConfig fetchColumns(Collection<Pair<Text,Text>> columns) {
+ this.columns = columns;
+ return this;
+ }
+
+ /**
+ * Returns the columns to be fetched for this configuration
+ */
+ public Collection<Pair<Text,Text>> getFetchedColumns() {
+ return columns != null ? columns : new HashSet<>();
+ }
+
+ /**
+ * Set iterators on to be used in the query.
+ *
+ * @param iterators
+ * the configurations for the iterators
+ * @since 1.6.0
+ */
+ public InputTableConfig setIterators(List<IteratorSetting> iterators) {
+ this.iterators = iterators;
+ return this;
+ }
+
+ /**
+ * Returns the iterators to be set on this configuration
+ */
+ public List<IteratorSetting> getIterators() {
+ return iterators != null ? iterators : new ArrayList<>();
+ }
+
+ /**
+ * Controls the automatic adjustment of ranges for this job. This feature merges overlapping
+ * ranges, then splits them to align with tablet boundaries. Disabling this feature will cause
+ * exactly one Map task to be created for each specified range. The default setting is enabled. *
+ *
+ * <p>
+ * By default, this feature is <b>enabled</b>.
+ *
+ * @param autoAdjustRanges
+ * the feature is enabled if true, disabled otherwise
+ * @see #setRanges(java.util.List)
+ * @since 1.6.0
+ */
+ public InputTableConfig setAutoAdjustRanges(boolean autoAdjustRanges) {
+ this.autoAdjustRanges = autoAdjustRanges;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has auto-adjust ranges enabled.
+ *
+ * @return false if the feature is disabled, true otherwise
+ * @since 1.6.0
+ * @see #setAutoAdjustRanges(boolean)
+ */
+ public boolean shouldAutoAdjustRanges() {
+ return autoAdjustRanges;
+ }
+
+ /**
+ * Controls the use of the {@link org.apache.accumulo.core.client.ClientSideIteratorScanner} in
+ * this job. Enabling this feature will cause the iterator stack to be constructed within the Map
+ * task, rather than within the Accumulo TServer. To use this feature, all classes needed for
+ * those iterators must be available on the classpath for the task.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param useLocalIterators
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setUseLocalIterators(boolean useLocalIterators) {
+ this.useLocalIterators = useLocalIterators;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration uses local iterators.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setUseLocalIterators(boolean)
+ */
+ public boolean shouldUseLocalIterators() {
+ return useLocalIterators;
+ }
+
+ /**
+ * Enable reading offline tables. By default, this feature is disabled and only online tables are
+ * scanned. This will make the map reduce job directly read the table's files. If the table is not
+ * offline, then the job will fail. If the table comes online during the map reduce job, it is
+ * likely that the job will fail.
+ *
+ * <p>
+ * To use this option, the map reduce user will need access to read the Accumulo directory in
+ * HDFS.
+ *
+ * <p>
+ * Reading the offline table will create the scan time iterator stack in the map process. So any
+ * iterators that are configured for the table will need to be on the mapper's classpath. The
+ * accumulo.properties may need to be on the mapper's classpath if HDFS or the Accumulo directory
+ * in HDFS are non-standard.
+ *
+ * <p>
+ * One way to use this feature is to clone a table, take the clone offline, and use the clone as
+ * the input table for a map reduce job. If you plan to map reduce over the data many times, it
+ * may be better to the compact the table, clone it, take it offline, and use the clone for all
+ * map reduce jobs. The reason to do this is that compaction will reduce each tablet in the table
+ * to one file, and it is faster to read from one file.
+ *
+ * <p>
+ * There are two possible advantages to reading a tables file directly out of HDFS. First, you may
+ * see better read performance. Second, it will support speculative execution better. When reading
+ * an online table speculative execution can put more load on an already slow tablet server.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param offlineScan
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setOfflineScan(boolean offlineScan) {
+ this.offlineScan = offlineScan;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has the offline table scan feature enabled.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setOfflineScan(boolean)
+ */
+ public boolean isOfflineScan() {
+ return offlineScan;
+ }
+
+ /**
+ * Controls the use of the {@link org.apache.accumulo.core.client.IsolatedScanner} in this job.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param useIsolatedScanners
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public InputTableConfig setUseIsolatedScanners(boolean useIsolatedScanners) {
+ this.useIsolatedScanners = useIsolatedScanners;
+ return this;
+ }
+
+ /**
+ * Determines whether a configuration has isolation enabled.
+ *
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setUseIsolatedScanners(boolean)
+ */
+ public boolean shouldUseIsolatedScanners() {
+ return useIsolatedScanners;
+ }
+
+ /**
+ * Set the sampler configuration to use when reading from the data.
+ *
+ * @see ScannerBase#setSamplerConfiguration(SamplerConfiguration)
+ * @see InputFormatBase#setSamplerConfiguration(org.apache.hadoop.mapreduce.Job,
+ * SamplerConfiguration)
+ *
+ * @since 1.8.0
+ */
+ public void setSamplerConfiguration(SamplerConfiguration samplerConfiguration) {
+ this.samplerConfig = samplerConfiguration;
+ }
+
+ /**
+ *
+ * @since 1.8.0
+ */
+ public SamplerConfiguration getSamplerConfiguration() {
+ return samplerConfig;
+ }
+
+ /**
+ * The execution hints to set on created scanners. See {@link ScannerBase#setExecutionHints(Map)}
+ *
+ * @since 2.0.0
+ */
+ public void setExecutionHints(Map<String,String> executionHints) {
+ this.executionHints = executionHints;
+ }
+
+ /**
+ * @since 2.0.0
+ */
+ public Map<String,String> getExecutionHints() {
+ return executionHints;
+ }
+
+ @Override
+ public void write(DataOutput dataOutput) throws IOException {
+ if (iterators != null) {
+ dataOutput.writeInt(iterators.size());
+ for (IteratorSetting setting : iterators)
+ setting.write(dataOutput);
+ } else {
+ dataOutput.writeInt(0);
+ }
+ if (ranges != null) {
+ dataOutput.writeInt(ranges.size());
+ for (Range range : ranges)
+ range.write(dataOutput);
+ } else {
+ dataOutput.writeInt(0);
+ }
+ if (columns != null) {
+ dataOutput.writeInt(columns.size());
+ for (Pair<Text,Text> column : columns) {
+ if (column.getSecond() == null) {
+ dataOutput.writeInt(1);
+ column.getFirst().write(dataOutput);
+ } else {
+ dataOutput.writeInt(2);
+ column.getFirst().write(dataOutput);
+ column.getSecond().write(dataOutput);
+ }
+ }
+ } else {
+ dataOutput.writeInt(0);
+ }
+ dataOutput.writeBoolean(autoAdjustRanges);
+ dataOutput.writeBoolean(useLocalIterators);
+ dataOutput.writeBoolean(useIsolatedScanners);
+ dataOutput.writeBoolean(offlineScan);
+ if (samplerConfig == null) {
+ dataOutput.writeBoolean(false);
+ } else {
+ dataOutput.writeBoolean(true);
+ new SamplerConfigurationImpl(samplerConfig).write(dataOutput);
+ }
+
+ if (executionHints == null || executionHints.size() == 0) {
+ dataOutput.writeInt(0);
+ } else {
+ dataOutput.writeInt(executionHints.size());
+ for (Entry<String,String> entry : executionHints.entrySet()) {
+ dataOutput.writeUTF(entry.getKey());
+ dataOutput.writeUTF(entry.getValue());
+ }
+ }
+ }
+
+ @Override
+ public void readFields(DataInput dataInput) throws IOException {
+ // load iterators
+ long iterSize = dataInput.readInt();
+ if (iterSize > 0)
+ iterators = new ArrayList<>();
+ for (int i = 0; i < iterSize; i++)
+ iterators.add(new IteratorSetting(dataInput));
+ // load ranges
+ long rangeSize = dataInput.readInt();
+ if (rangeSize > 0)
+ ranges = new ArrayList<>();
+ for (int i = 0; i < rangeSize; i++) {
+ Range range = new Range();
+ range.readFields(dataInput);
+ ranges.add(range);
+ }
+ // load columns
+ long columnSize = dataInput.readInt();
+ if (columnSize > 0)
+ columns = new HashSet<>();
+ for (int i = 0; i < columnSize; i++) {
+ long numPairs = dataInput.readInt();
+ Text colFam = new Text();
+ colFam.readFields(dataInput);
+ if (numPairs == 1) {
+ columns.add(new Pair<>(colFam, null));
+ } else if (numPairs == 2) {
+ Text colQual = new Text();
+ colQual.readFields(dataInput);
+ columns.add(new Pair<>(colFam, colQual));
+ }
+ }
+ autoAdjustRanges = dataInput.readBoolean();
+ useLocalIterators = dataInput.readBoolean();
+ useIsolatedScanners = dataInput.readBoolean();
+ offlineScan = dataInput.readBoolean();
+
+ if (dataInput.readBoolean()) {
+ samplerConfig = new SamplerConfigurationImpl(dataInput).toSamplerConfiguration();
+ }
+
+ executionHints = new HashMap<>();
+ int numHints = dataInput.readInt();
+ for (int i = 0; i < numHints; i++) {
+ String k = dataInput.readUTF();
+ String v = dataInput.readUTF();
+ executionHints.put(k, v);
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ InputTableConfig that = (InputTableConfig) o;
+
+ if (autoAdjustRanges != that.autoAdjustRanges)
+ return false;
+ if (offlineScan != that.offlineScan)
+ return false;
+ if (useIsolatedScanners != that.useIsolatedScanners)
+ return false;
+ if (useLocalIterators != that.useLocalIterators)
+ return false;
+ if (columns != null ? !columns.equals(that.columns) : that.columns != null)
+ return false;
+ if (iterators != null ? !iterators.equals(that.iterators) : that.iterators != null)
+ return false;
+ if (ranges != null ? !ranges.equals(that.ranges) : that.ranges != null)
+ return false;
+ if (executionHints != null ? !executionHints.equals(that.executionHints)
+ : that.executionHints != null)
+ return false;
+ return samplerConfig != null ? samplerConfig.equals(that.samplerConfig)
+ : that.samplerConfig == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 31 * (iterators != null ? iterators.hashCode() : 0);
+ result = 31 * result + (ranges != null ? ranges.hashCode() : 0);
+ result = 31 * result + (columns != null ? columns.hashCode() : 0);
+ result = 31 * result + (autoAdjustRanges ? 1 : 0);
+ result = 31 * result + (useLocalIterators ? 1 : 0);
+ result = 31 * result + (useIsolatedScanners ? 1 : 0);
+ result = 31 * result + (offlineScan ? 1 : 0);
+ result = 31 * result + (samplerConfig == null ? 0 : samplerConfig.hashCode());
+ result = 31 * result + (executionHints == null ? 0 : executionHints.hashCode());
+ return result;
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/RangeInputSplit.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/RangeInputSplit.java
new file mode 100644
index 0000000000..a3f00106bb
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/RangeInputSplit.java
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.hadoopImpl.mapreduce.SplitUtils;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.InputConfigurator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.log4j.Level;
+
+/**
+ * The Class RangeInputSplit. Encapsulates an Accumulo range for use in Map Reduce jobs.
+ */
+public class RangeInputSplit extends InputSplit implements Writable {
+ private Range range;
+ private String[] locations;
+ private String tableId, tableName;
+ private Boolean offline, isolatedScan, localIterators;
+ private Set<Pair<Text,Text>> fetchedColumns;
+ private List<IteratorSetting> iterators;
+ private SamplerConfiguration samplerConfig;
+ private Level level;
+ private Map<String,String> executionHints;
+
+ public RangeInputSplit() {
+ range = new Range();
+ locations = new String[0];
+ tableName = "";
+ tableId = "";
+ }
+
+ public RangeInputSplit(RangeInputSplit split) throws IOException {
+ this.range = split.getRange();
+ this.setLocations(split.getLocations());
+ this.setTableName(split.getTableName());
+ this.setTableId(split.getTableId());
+ }
+
+ protected RangeInputSplit(String table, String tableId, Range range, String[] locations) {
+ this.range = range;
+ setLocations(locations);
+ this.tableName = table;
+ this.tableId = tableId;
+ }
+
+ public Range getRange() {
+ return range;
+ }
+
+ public static float getProgress(ByteSequence start, ByteSequence end, ByteSequence position) {
+ return SplitUtils.getProgress(start, end, position);
+ }
+
+ public float getProgress(Key currentKey) {
+ if (currentKey == null)
+ return 0f;
+ if (range.contains(currentKey)) {
+ if (range.getStartKey() != null && range.getEndKey() != null) {
+ if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW) != 0) {
+ // just look at the row progress
+ return getProgress(range.getStartKey().getRowData(), range.getEndKey().getRowData(),
+ currentKey.getRowData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM) != 0) {
+ // just look at the column family progress
+ return getProgress(range.getStartKey().getColumnFamilyData(),
+ range.getEndKey().getColumnFamilyData(), currentKey.getColumnFamilyData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(),
+ PartialKey.ROW_COLFAM_COLQUAL) != 0) {
+ // just look at the column qualifier progress
+ return getProgress(range.getStartKey().getColumnQualifierData(),
+ range.getEndKey().getColumnQualifierData(), currentKey.getColumnQualifierData());
+ }
+ }
+ }
+ // if we can't figure it out, then claim no progress
+ return 0f;
+ }
+
+ /**
+ * This implementation of length is only an estimate, it does not provide exact values. Do not
+ * have your code rely on this return value.
+ */
+ @Override
+ public long getLength() throws IOException {
+ return SplitUtils.getRangeLength(range);
+ }
+
+ @Override
+ public String[] getLocations() throws IOException {
+ return Arrays.copyOf(locations, locations.length);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ range.readFields(in);
+ tableName = in.readUTF();
+ tableId = in.readUTF();
+ int numLocs = in.readInt();
+ locations = new String[numLocs];
+ for (int i = 0; i < numLocs; ++i)
+ locations[i] = in.readUTF();
+
+ if (in.readBoolean()) {
+ isolatedScan = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ offline = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ localIterators = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ int numColumns = in.readInt();
+ List<String> columns = new ArrayList<>(numColumns);
+ for (int i = 0; i < numColumns; i++) {
+ columns.add(in.readUTF());
+ }
+
+ fetchedColumns = InputConfigurator.deserializeFetchedColumns(columns);
+ }
+
+ if (in.readBoolean()) {
+ int numIterators = in.readInt();
+ iterators = new ArrayList<>(numIterators);
+ for (int i = 0; i < numIterators; i++) {
+ iterators.add(new IteratorSetting(in));
+ }
+ }
+
+ if (in.readBoolean()) {
+ level = Level.toLevel(in.readInt());
+ }
+
+ if (in.readBoolean()) {
+ samplerConfig = new SamplerConfigurationImpl(in).toSamplerConfiguration();
+ }
+
+ executionHints = new HashMap<>();
+ int numHints = in.readInt();
+ for (int i = 0; i < numHints; i++) {
+ String k = in.readUTF();
+ String v = in.readUTF();
+ executionHints.put(k, v);
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ range.write(out);
+ out.writeUTF(tableName);
+ out.writeUTF(tableId);
+ out.writeInt(locations.length);
+ for (String location : locations)
+ out.writeUTF(location);
+
+ out.writeBoolean(null != isolatedScan);
+ if (null != isolatedScan) {
+ out.writeBoolean(isolatedScan);
+ }
+
+ out.writeBoolean(null != offline);
+ if (null != offline) {
+ out.writeBoolean(offline);
+ }
+
+ out.writeBoolean(null != localIterators);
+ if (null != localIterators) {
+ out.writeBoolean(localIterators);
+ }
+
+ out.writeBoolean(null != fetchedColumns);
+ if (null != fetchedColumns) {
+ String[] cols = InputConfigurator.serializeColumns(fetchedColumns);
+ out.writeInt(cols.length);
+ for (String col : cols) {
+ out.writeUTF(col);
+ }
+ }
+
+ out.writeBoolean(null != iterators);
+ if (null != iterators) {
+ out.writeInt(iterators.size());
+ for (IteratorSetting iterator : iterators) {
+ iterator.write(out);
+ }
+ }
+
+ out.writeBoolean(null != level);
+ if (null != level) {
+ out.writeInt(level.toInt());
+ }
+
+ out.writeBoolean(null != samplerConfig);
+ if (null != samplerConfig) {
+ new SamplerConfigurationImpl(samplerConfig).write(out);
+ }
+
+ if (executionHints == null || executionHints.size() == 0) {
+ out.writeInt(0);
+ } else {
+ out.writeInt(executionHints.size());
+ for (Entry<String,String> entry : executionHints.entrySet()) {
+ out.writeUTF(entry.getKey());
+ out.writeUTF(entry.getValue());
+ }
+ }
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String table) {
+ this.tableName = table;
+ }
+
+ public void setTableId(String tableId) {
+ this.tableId = tableId;
+ }
+
+ public String getTableId() {
+ return tableId;
+ }
+
+ public Boolean isOffline() {
+ return offline;
+ }
+
+ public void setOffline(Boolean offline) {
+ this.offline = offline;
+ }
+
+ public void setLocations(String[] locations) {
+ this.locations = Arrays.copyOf(locations, locations.length);
+ }
+
+ public Boolean isIsolatedScan() {
+ return isolatedScan;
+ }
+
+ public void setIsolatedScan(Boolean isolatedScan) {
+ this.isolatedScan = isolatedScan;
+ }
+
+ public void setRange(Range range) {
+ this.range = range;
+ }
+
+ public Boolean usesLocalIterators() {
+ return localIterators;
+ }
+
+ public void setUsesLocalIterators(Boolean localIterators) {
+ this.localIterators = localIterators;
+ }
+
+ public Set<Pair<Text,Text>> getFetchedColumns() {
+ return fetchedColumns;
+ }
+
+ public void setFetchedColumns(Collection<Pair<Text,Text>> fetchedColumns) {
+ this.fetchedColumns = new HashSet<>();
+ for (Pair<Text,Text> columns : fetchedColumns) {
+ this.fetchedColumns.add(columns);
+ }
+ }
+
+ public void setFetchedColumns(Set<Pair<Text,Text>> fetchedColumns) {
+ this.fetchedColumns = fetchedColumns;
+ }
+
+ public List<IteratorSetting> getIterators() {
+ return iterators;
+ }
+
+ public void setIterators(List<IteratorSetting> iterators) {
+ this.iterators = iterators;
+ }
+
+ public Level getLogLevel() {
+ return level;
+ }
+
+ public void setLogLevel(Level level) {
+ this.level = level;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(256);
+ sb.append("Range: ").append(range);
+ sb.append(" Locations: ").append(Arrays.asList(locations));
+ sb.append(" Table: ").append(tableName);
+ sb.append(" TableID: ").append(tableId);
+ sb.append(" offlineScan: ").append(offline);
+ sb.append(" isolatedScan: ").append(isolatedScan);
+ sb.append(" localIterators: ").append(localIterators);
+ sb.append(" fetchColumns: ").append(fetchedColumns);
+ sb.append(" iterators: ").append(iterators);
+ sb.append(" logLevel: ").append(level);
+ sb.append(" samplerConfig: ").append(samplerConfig);
+ sb.append(" executionHints: ").append(executionHints);
+ return sb.toString();
+ }
+
+ public void setSamplerConfiguration(SamplerConfiguration samplerConfiguration) {
+ this.samplerConfig = samplerConfiguration;
+ }
+
+ public SamplerConfiguration getSamplerConfiguration() {
+ return samplerConfig;
+ }
+
+ public void setExecutionHints(Map<String,String> executionHints) {
+ this.executionHints = executionHints;
+ }
+
+ public Map<String,String> getExecutionHints() {
+ return executionHints;
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/KeyRangePartitioner.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/KeyRangePartitioner.java
new file mode 100644
index 0000000000..07398826d4
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/KeyRangePartitioner.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce.lib.partition;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Partitioner;
+
+/**
+ * Hadoop partitioner that uses ranges based on row keys, and optionally sub-bins based on hashing.
+ */
+public class KeyRangePartitioner extends Partitioner<Key,Writable> implements Configurable {
+ private RangePartitioner rp = new RangePartitioner();
+
+ @Override
+ public int getPartition(Key key, Writable value, int numPartitions) {
+ return rp.getPartition(key.getRow(), value, numPartitions);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return rp.getConf();
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ rp.setConf(conf);
+ }
+
+ /**
+ * Sets the hdfs file name to use, containing a newline separated list of Base64 encoded split
+ * points that represent ranges for partitioning
+ */
+ public static void setSplitFile(Job job, String file) {
+ RangePartitioner.setSplitFile(job, file);
+ }
+
+ /**
+ * Sets the number of random sub-bins per range
+ */
+ public static void setNumSubBins(Job job, int num) {
+ RangePartitioner.setNumSubBins(job, num);
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/RangePartitioner.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/RangePartitioner.java
new file mode 100644
index 0000000000..9cb744fb32
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/RangePartitioner.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce.lib.partition;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Scanner;
+import java.util.TreeSet;
+
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.DistributedCacheHelper;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Partitioner;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+/**
+ * Hadoop partitioner that uses ranges, and optionally sub-bins based on hashing.
+ */
+public class RangePartitioner extends Partitioner<Text,Writable> implements Configurable {
+ private static final String PREFIX = RangePartitioner.class.getName();
+ private static final String CUTFILE_KEY = PREFIX + ".cutFile";
+ private static final String NUM_SUBBINS = PREFIX + ".subBins";
+
+ private Configuration conf;
+
+ @Override
+ public int getPartition(Text key, Writable value, int numPartitions) {
+ try {
+ return findPartition(key, getCutPoints(), getNumSubBins());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ int findPartition(Text key, Text[] array, int numSubBins) {
+ // find the bin for the range, and guarantee it is positive
+ int index = Arrays.binarySearch(array, key);
+ index = index < 0 ? (index + 1) * -1 : index;
+
+ // both conditions work with numSubBins == 1, but this check is to avoid
+ // hashing, when we don't need to, for speed
+ if (numSubBins < 2)
+ return index;
+ return (key.toString().hashCode() & Integer.MAX_VALUE) % numSubBins + index * numSubBins;
+ }
+
+ private int _numSubBins = 0;
+
+ private synchronized int getNumSubBins() {
+ if (_numSubBins < 1) {
+ // get number of sub-bins and guarantee it is positive
+ _numSubBins = Math.max(1, getConf().getInt(NUM_SUBBINS, 1));
+ }
+ return _numSubBins;
+ }
+
+ private Text cutPointArray[] = null;
+
+ @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN",
+ justification = "path provided by distributed cache framework, not user input")
+ private synchronized Text[] getCutPoints() throws IOException {
+ if (cutPointArray == null) {
+ String cutFileName = conf.get(CUTFILE_KEY);
+ Path[] cf = DistributedCacheHelper.getLocalCacheFiles(conf);
+
+ if (cf != null) {
+ for (Path path : cf) {
+ if (path.toUri().getPath()
+ .endsWith(cutFileName.substring(cutFileName.lastIndexOf('/')))) {
+ TreeSet<Text> cutPoints = new TreeSet<>();
+ try (Scanner in = new Scanner(new BufferedReader(
+ new InputStreamReader(new FileInputStream(path.toString()), UTF_8)))) {
+ while (in.hasNextLine())
+ cutPoints.add(new Text(Base64.getDecoder().decode(in.nextLine())));
+ }
+ cutPointArray = cutPoints.toArray(new Text[cutPoints.size()]);
+ break;
+ }
+ }
+ }
+ if (cutPointArray == null)
+ throw new FileNotFoundException(cutFileName + " not found in distributed cache");
+ }
+ return cutPointArray;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Sets the hdfs file name to use, containing a newline separated list of Base64 encoded split
+ * points that represent ranges for partitioning
+ */
+ public static void setSplitFile(Job job, String file) {
+ URI uri = new Path(file).toUri();
+ DistributedCacheHelper.addCacheFile(uri, job.getConfiguration());
+ job.getConfiguration().set(CUTFILE_KEY, uri.getPath());
+ }
+
+ /**
+ * Sets the number of random sub-bins per range
+ */
+ public static void setNumSubBins(Job job, int num) {
+ job.getConfiguration().setInt(NUM_SUBBINS, num);
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/BatchInputSplit.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/BatchInputSplit.java
new file mode 100644
index 0000000000..711f7822f2
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/BatchInputSplit.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapred;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.data.Range;
+import org.apache.hadoop.mapred.InputSplit;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+/**
+ * The Class BatchInputSplit. Encapsulates Accumulo ranges for use in Map Reduce jobs. Can contain
+ * several Ranges per InputSplit.
+ */
+@SuppressFBWarnings(value = "NM_SAME_SIMPLE_NAME_AS_SUPERCLASS",
+ justification = "Intended to share code between mapred and mapreduce")
+public class BatchInputSplit extends org.apache.accumulo.hadoopImpl.mapreduce.BatchInputSplit
+ implements InputSplit {
+
+ public BatchInputSplit() {
+ super();
+ }
+
+ public BatchInputSplit(BatchInputSplit split) throws IOException {
+ super(split);
+ }
+
+ public BatchInputSplit(String table, Table.ID tableId, Collection<Range> ranges,
+ String[] location) {
+ super(table, tableId, ranges, location);
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/BatchInputSplit.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/BatchInputSplit.java
new file mode 100644
index 0000000000..40b6326f7a
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/BatchInputSplit.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.hadoop.mapreduce.RangeInputSplit;
+
+/**
+ * The Class BatchInputSplit. Encapsulates a set of Accumulo ranges on a single tablet for use in
+ * Map Reduce jobs. Can contain several Ranges per split.
+ */
+public class BatchInputSplit extends RangeInputSplit {
+ private Collection<Range> ranges;
+ private float[] rangeProgress = null;
+
+ public BatchInputSplit() {
+ ranges = Collections.emptyList();
+ }
+
+ public BatchInputSplit(BatchInputSplit split) throws IOException {
+ super(split);
+ this.setRanges(split.getRanges());
+ }
+
+ public BatchInputSplit(String table, Table.ID tableId, Collection<Range> ranges,
+ String[] locations) {
+ super(table, tableId.canonicalID(), new Range(), locations);
+ this.ranges = ranges;
+ }
+
+ /**
+ * Save progress on each call to this function, implied by value of currentKey, and return average
+ * ranges in the split
+ */
+ @Override
+ public float getProgress(Key currentKey) {
+ if (null == rangeProgress)
+ rangeProgress = new float[ranges.size()];
+
+ float total = 0; // progress per range could be on different scales, this number is "fuzzy"
+
+ if (currentKey == null) {
+ for (float progress : rangeProgress)
+ total += progress;
+ } else {
+ int i = 0;
+ for (Range range : ranges) {
+ if (range.contains(currentKey)) {
+ // find the current range and report as if that is the single range
+ if (range.getStartKey() != null && range.getEndKey() != null) {
+ if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW) != 0) {
+ // just look at the row progress
+ rangeProgress[i] = SplitUtils.getProgress(range.getStartKey().getRowData(),
+ range.getEndKey().getRowData(), currentKey.getRowData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(),
+ PartialKey.ROW_COLFAM) != 0) {
+ // just look at the column family progress
+ rangeProgress[i] = SplitUtils.getProgress(range.getStartKey().getColumnFamilyData(),
+ range.getEndKey().getColumnFamilyData(), currentKey.getColumnFamilyData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(),
+ PartialKey.ROW_COLFAM_COLQUAL) != 0) {
+ // just look at the column qualifier progress
+ rangeProgress[i] = SplitUtils.getProgress(
+ range.getStartKey().getColumnQualifierData(),
+ range.getEndKey().getColumnQualifierData(), currentKey.getColumnQualifierData());
+ }
+ }
+ total += rangeProgress[i];
+ }
+ i++;
+ }
+ }
+
+ return total / ranges.size();
+ }
+
+ /**
+ * This implementation of length is only an estimate, it does not provide exact values. Do not
+ * have your code rely on this return value.
+ */
+ @Override
+ public long getLength() throws IOException {
+ long sum = 0;
+ for (Range range : ranges)
+ sum += SplitUtils.getRangeLength(range);
+ return sum;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+
+ int numRanges = in.readInt();
+ ranges = new ArrayList<>(numRanges);
+ for (int i = 0; i < numRanges; ++i) {
+ Range r = new Range();
+ r.readFields(in);
+ ranges.add(r);
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+
+ out.writeInt(ranges.size());
+ for (Range r : ranges)
+ r.write(out);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(256);
+ sb.append("BatchInputSplit:");
+ sb.append(" Ranges: ").append(Arrays.asList(ranges));
+ sb.append(super.toString());
+ return sb.toString();
+ }
+
+ public void setRanges(Collection<Range> ranges) {
+ this.ranges = ranges;
+ }
+
+ public Collection<Range> getRanges() {
+ return ranges;
+ }
+
+ @Override
+ public Range getRange() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setRange(Range range) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Boolean isIsolatedScan() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setIsolatedScan(Boolean isolatedScan) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Boolean isOffline() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setOffline(Boolean offline) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Boolean usesLocalIterators() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setUsesLocalIterators(Boolean localIterators) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/DelegationTokenStub.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/DelegationTokenStub.java
new file mode 100644
index 0000000000..62529f962e
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/DelegationTokenStub.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce;
+
+import static java.util.Objects.requireNonNull;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Set;
+
+import javax.security.auth.DestroyFailedException;
+
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+
+/**
+ * An internal stub class for passing DelegationToken information out of the Configuration back up
+ * to the appropriate implementation for mapreduce or mapred.
+ */
+public class DelegationTokenStub implements AuthenticationToken {
+
+ private String serviceName;
+
+ public DelegationTokenStub(String serviceName) {
+ requireNonNull(serviceName);
+ this.serviceName = serviceName;
+ }
+
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void destroy() throws DestroyFailedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isDestroyed() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void init(Properties properties) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Set<TokenProperty> getProperties() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public AuthenticationToken clone() {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/SplitUtils.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/SplitUtils.java
new file mode 100644
index 0000000000..6f0c8a055e
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/SplitUtils.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.hadoopImpl.mapreduce;
+
+import java.io.IOException;
+import java.math.BigInteger;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.hadoop.mapreduce.InputTableConfig;
+import org.apache.accumulo.hadoop.mapreduce.RangeInputSplit;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+
+public class SplitUtils {
+
+ /**
+ * Central place to set common split configuration not handled by split constructors. The
+ * intention is to make it harder to miss optional setters in future refactor.
+ */
+ public static void updateSplit(RangeInputSplit split, InputTableConfig tableConfig,
+ Level logLevel) {
+ split.setFetchedColumns(tableConfig.getFetchedColumns());
+ split.setIterators(tableConfig.getIterators());
+ split.setLogLevel(logLevel);
+ split.setSamplerConfiguration(tableConfig.getSamplerConfiguration());
+ split.setExecutionHints(tableConfig.getExecutionHints());
+ }
+
+ public static float getProgress(ByteSequence start, ByteSequence end, ByteSequence position) {
+ int maxDepth = Math.min(Math.max(end.length(), start.length()), position.length());
+ BigInteger startBI = new BigInteger(SplitUtils.extractBytes(start, maxDepth));
+ BigInteger endBI = new BigInteger(SplitUtils.extractBytes(end, maxDepth));
+ BigInteger positionBI = new BigInteger(SplitUtils.extractBytes(position, maxDepth));
+ return (float) (positionBI.subtract(startBI).doubleValue()
+ / endBI.subtract(startBI).doubleValue());
+ }
+
+ public static long getRangeLength(Range range) throws IOException {
+ Text startRow = range.isInfiniteStartKey() ? new Text(new byte[] {Byte.MIN_VALUE})
+ : range.getStartKey().getRow();
+ Text stopRow = range.isInfiniteStopKey() ? new Text(new byte[] {Byte.MAX_VALUE})
+ : range.getEndKey().getRow();
+ int maxCommon = Math.min(7, Math.min(startRow.getLength(), stopRow.getLength()));
+ long diff = 0;
+
+ byte[] start = startRow.getBytes();
+ byte[] stop = stopRow.getBytes();
+ for (int i = 0; i < maxCommon; ++i) {
+ diff |= 0xff & (start[i] ^ stop[i]);
+ diff <<= Byte.SIZE;
+ }
+
+ if (startRow.getLength() != stopRow.getLength())
+ diff |= 0xff;
+
+ return diff + 1;
+ }
+
+ static byte[] extractBytes(ByteSequence seq, int numBytes) {
+ byte[] bytes = new byte[numBytes + 1];
+ bytes[0] = 0;
+ for (int i = 0; i < numBytes; i++) {
+ if (i >= seq.length())
+ bytes[i + 1] = 0;
+ else
+ bytes[i + 1] = seq.byteAt(i);
+ }
+ return bytes;
+ }
+
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/ConfiguratorBase.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/ConfiguratorBase.java
new file mode 100644
index 0000000000..c26dbff9fa
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/ConfiguratorBase.java
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Properties;
+import java.util.Scanner;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.impl.AuthenticationTokenIdentifier;
+import org.apache.accumulo.core.client.impl.ClientConfConverter;
+import org.apache.accumulo.core.client.impl.ClientInfoImpl;
+import org.apache.accumulo.core.client.impl.DelegationTokenImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.conf.ClientProperty;
+import org.apache.accumulo.hadoopImpl.mapreduce.DelegationTokenStub;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * @since 1.6.0
+ */
+public class ConfiguratorBase {
+
+ protected static final Logger log = Logger.getLogger(ConfiguratorBase.class);
+
+ /**
+ * Specifies that connection info was configured
+ *
+ * @since 1.6.0
+ */
+ public enum ConnectorInfo {
+ IS_CONFIGURED
+ }
+
+ public enum ClientOpts {
+ CLIENT_PROPS, CLIENT_PROPS_FILE
+ }
+
+ /**
+ * Configuration keys for general configuration options.
+ *
+ * @since 1.6.0
+ */
+ public enum GeneralOpts {
+ LOG_LEVEL, VISIBILITY_CACHE_SIZE
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum, prefixed by the implementingClass
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ * @since 1.6.0
+ */
+ protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
+ return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "."
+ + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum.
+ *
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ */
+ protected static String enumToConfKey(Enum<?> e) {
+ return e.getDeclaringClass().getSimpleName() + "."
+ + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ public static ClientInfo updateToken(org.apache.hadoop.security.Credentials credentials,
+ ClientInfo info) {
+ ClientInfo result = info;
+ if (info.getAuthenticationToken() instanceof KerberosToken) {
+ log.info("Received KerberosToken, attempting to fetch DelegationToken");
+ try {
+ AccumuloClient client = Accumulo.newClient().usingClientInfo(info).build();
+ AuthenticationToken token = client.securityOperations()
+ .getDelegationToken(new DelegationTokenConfig());
+ result = Accumulo.newClient().usingClientInfo(info).usingToken(info.getPrincipal(), token)
+ .info();
+ } catch (Exception e) {
+ log.warn("Failed to automatically obtain DelegationToken, "
+ + "Mappers/Reducers will likely fail to communicate with Accumulo", e);
+ }
+ }
+ // DelegationTokens can be passed securely from user to task without serializing insecurely in
+ // the configuration
+ if (info.getAuthenticationToken() instanceof DelegationTokenImpl) {
+ DelegationTokenImpl delegationToken = (DelegationTokenImpl) info.getAuthenticationToken();
+
+ // Convert it into a Hadoop Token
+ AuthenticationTokenIdentifier identifier = delegationToken.getIdentifier();
+ Token<AuthenticationTokenIdentifier> hadoopToken = new Token<>(identifier.getBytes(),
+ delegationToken.getPassword(), identifier.getKind(), delegationToken.getServiceName());
+
+ // Add the Hadoop Token to the Job so it gets serialized and passed along.
+ credentials.addToken(hadoopToken.getService(), hadoopToken);
+ }
+ return result;
+ }
+
+ public static void setClientInfo(Class<?> implementingClass, Configuration conf,
+ ClientInfo info) {
+ setClientProperties(implementingClass, conf, info.getProperties());
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ }
+
+ public static ClientInfo getClientInfo(Class<?> implementingClass, Configuration conf) {
+ Properties props = getClientProperties(implementingClass, conf);
+ return new ClientInfoImpl(props);
+ }
+
+ public static void setClientPropertiesFile(Class<?> implementingClass, Configuration conf,
+ String clientPropertiesFile) {
+ try {
+ DistributedCacheHelper.addCacheFile(new URI(clientPropertiesFile), conf);
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Unable to add client properties file \""
+ + clientPropertiesFile + "\" to distributed cache.");
+ }
+ conf.set(enumToConfKey(implementingClass, ClientOpts.CLIENT_PROPS_FILE), clientPropertiesFile);
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ }
+
+ public static void setClientProperties(Class<?> implementingClass, Configuration conf,
+ Properties props) {
+ StringWriter writer = new StringWriter();
+ try {
+ props.store(writer, "client properties");
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ conf.set(enumToConfKey(implementingClass, ClientOpts.CLIENT_PROPS), writer.toString());
+ }
+
+ public static Properties getClientProperties(Class<?> implementingClass, Configuration conf) {
+ String propString;
+ String clientPropsFile = conf
+ .get(enumToConfKey(implementingClass, ClientOpts.CLIENT_PROPS_FILE), "");
+ if (!clientPropsFile.isEmpty()) {
+ try {
+ URI[] uris = DistributedCacheHelper.getCacheFiles(conf);
+ Path path = null;
+ for (URI u : uris) {
+ if (u.toString().equals(clientPropsFile)) {
+ path = new Path(u);
+ }
+ }
+ FileSystem fs = FileSystem.get(conf);
+ FSDataInputStream inputStream = fs.open(path);
+ StringBuilder sb = new StringBuilder();
+ try (Scanner scanner = new Scanner(inputStream)) {
+ while (scanner.hasNextLine()) {
+ sb.append(scanner.nextLine() + "\n");
+ }
+ }
+ propString = sb.toString();
+ } catch (IOException e) {
+ throw new IllegalStateException(
+ "Failed to read client properties from distributed cache: " + clientPropsFile);
+ }
+ } else {
+ propString = conf.get(enumToConfKey(implementingClass, ClientOpts.CLIENT_PROPS), "");
+ }
+ Properties props = new Properties();
+ if (!propString.isEmpty()) {
+ try {
+ props.load(new StringReader(propString));
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ return props;
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all
+ * MapReduce tasks. It is BASE64 encoded to provide a charset safe conversion to a string, and is
+ * not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param token
+ * the user's password
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf,
+ String principal, AuthenticationToken token) {
+ checkArgument(principal != null, "principal is null");
+ checkArgument(token != null, "token is null");
+ Properties props = getClientProperties(implementingClass, conf);
+ props.setProperty(ClientProperty.AUTH_PRINCIPAL.getKey(), principal);
+ ClientProperty.setAuthenticationToken(props, token);
+ setClientProperties(implementingClass, conf, props);
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ }
+
+ /**
+ * Determines if the connector info has already been set for this instance.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the connector info has already been set, false otherwise
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
+ Properties props = getClientProperties(implementingClass, conf);
+ return props.getProperty(ClientProperty.AUTH_PRINCIPAL.getKey());
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the
+ * configuration, whichever was used when the job was configured.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static AuthenticationToken getAuthenticationToken(Class<?> implementingClass,
+ Configuration conf) {
+ Properties props = getClientProperties(implementingClass, conf);
+ return ClientProperty.getAuthenticationToken(props);
+ }
+
+ /**
+ * Configures a {@link org.apache.accumulo.core.client.ZooKeeperInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param clientConfig
+ * client configuration for specifying connection timeouts, SSL connection options, etc.
+ * @since 1.6.0
+ * @deprecated since 2.0.0; use {@link #setClientInfo(Class, Configuration, ClientInfo)} instead
+ */
+ @Deprecated
+ public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf,
+ org.apache.accumulo.core.client.ClientConfiguration clientConfig) {
+ Properties props = getClientProperties(implementingClass, conf);
+ Properties newProps = ClientConfConverter.toProperties(clientConfig);
+ for (Object keyObj : newProps.keySet()) {
+ String propKey = (String) keyObj;
+ String val = newProps.getProperty(propKey);
+ props.setProperty(propKey, val);
+ }
+ setClientProperties(implementingClass, conf, props);
+ }
+
+ /**
+ * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the
+ * configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return an Accumulo instance
+ * @since 1.6.0
+ * @deprecated since 2.0.0, replaced by {@link #getClientInfo(Class, Configuration)}
+ */
+ @Deprecated
+ public static org.apache.accumulo.core.client.Instance getInstance(Class<?> implementingClass,
+ Configuration conf) {
+ return org.apache.accumulo.core.client.Connector.from(getClient(implementingClass, conf))
+ .getInstance();
+ }
+
+ /**
+ * Creates an Accumulo {@link AccumuloClient} based on the configuration
+ *
+ * @param implementingClass
+ * class whose name will be used as a prefix for the property configuration
+ * @param conf
+ * Hadoop configuration object
+ * @return Accumulo connector
+ * @since 2.0.0
+ */
+ public static AccumuloClient getClient(Class<?> implementingClass, Configuration conf) {
+ try {
+ return Accumulo.newClient().usingClientInfo(getClientInfo(implementingClass, conf)).build();
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Obtain a ClientConfiguration based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ *
+ * @return A ClientConfiguration
+ * @since 1.7.0
+ * @deprecated since 2.0.0; use {@link #getClientInfo(Class, Configuration)} instead
+ */
+ @Deprecated
+ public static org.apache.accumulo.core.client.ClientConfiguration getClientConfiguration(
+ Class<?> implementingClass, Configuration conf) {
+ return ClientConfConverter.toClientConf(getClientInfo(implementingClass, conf).getProperties());
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param level
+ * the logging level
+ * @since 1.6.0
+ */
+ public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
+ checkArgument(level != null, "level is null");
+ Logger.getLogger(implementingClass).setLevel(level);
+ conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the log level
+ * @since 1.6.0
+ * @see #setLogLevel(Class, Configuration, Level)
+ */
+ public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
+ return Level.toLevel(
+ conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
+ }
+
+ /**
+ * Sets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param visibilityCacheSize
+ * the LRU cache size
+ */
+ public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
+ conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
+ }
+
+ /**
+ * Gets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the valid visibility count
+ */
+ public static int getVisibilityCacheSize(Configuration conf) {
+ return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),
+ Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
+ }
+
+ /**
+ * Unwraps the provided {@link AuthenticationToken} if it is an instance of
+ * {@link DelegationTokenStub}, reconstituting it from the provided {@link JobConf}.
+ *
+ * @param job
+ * The job
+ * @param token
+ * The authentication token
+ */
+ public static AuthenticationToken unwrapAuthenticationToken(JobConf job,
+ AuthenticationToken token) {
+ requireNonNull(job);
+ requireNonNull(token);
+ if (token instanceof DelegationTokenStub) {
+ DelegationTokenStub delTokenStub = (DelegationTokenStub) token;
+ Token<? extends TokenIdentifier> hadoopToken = job.getCredentials()
+ .getToken(new Text(delTokenStub.getServiceName()));
+ AuthenticationTokenIdentifier identifier = new AuthenticationTokenIdentifier();
+ try {
+ identifier
+ .readFields(new DataInputStream(new ByteArrayInputStream(hadoopToken.getIdentifier())));
+ return new DelegationTokenImpl(hadoopToken.getPassword(), identifier);
+ } catch (IOException e) {
+ throw new RuntimeException("Could not construct DelegationToken from JobConf Credentials",
+ e);
+ }
+ }
+ return token;
+ }
+
+ /**
+ * Unwraps the provided {@link AuthenticationToken} if it is an instance of
+ * {@link DelegationTokenStub}, reconstituting it from the provided {@link JobConf}.
+ *
+ * @param job
+ * The job
+ * @param token
+ * The authentication token
+ */
+ public static AuthenticationToken unwrapAuthenticationToken(JobContext job,
+ AuthenticationToken token) {
+ requireNonNull(job);
+ requireNonNull(token);
+ if (token instanceof DelegationTokenStub) {
+ DelegationTokenStub delTokenStub = (DelegationTokenStub) token;
+ Token<? extends TokenIdentifier> hadoopToken = job.getCredentials()
+ .getToken(new Text(delTokenStub.getServiceName()));
+ AuthenticationTokenIdentifier identifier = new AuthenticationTokenIdentifier();
+ try {
+ identifier
+ .readFields(new DataInputStream(new ByteArrayInputStream(hadoopToken.getIdentifier())));
+ return new DelegationTokenImpl(hadoopToken.getPassword(), identifier);
+ } catch (IOException e) {
+ throw new RuntimeException("Could not construct DelegationToken from JobConf Credentials",
+ e);
+ }
+ }
+ return token;
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/DistributedCacheHelper.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/DistributedCacheHelper.java
new file mode 100644
index 0000000000..eb1e650b09
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/DistributedCacheHelper.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * @since 1.6.0
+ */
+@SuppressWarnings("deprecation")
+public class DistributedCacheHelper {
+
+ /**
+ * @since 1.6.0
+ */
+ public static void addCacheFile(URI uri, Configuration conf) {
+ org.apache.hadoop.filecache.DistributedCache.addCacheFile(uri, conf);
+ }
+
+ /**
+ * @since 1.6.0
+ */
+ public static URI[] getCacheFiles(Configuration conf) throws IOException {
+ return org.apache.hadoop.filecache.DistributedCache.getCacheFiles(conf);
+ }
+
+ /**
+ * @since 1.6.0
+ */
+ public static Path[] getLocalCacheFiles(Configuration conf) throws IOException {
+ return org.apache.hadoop.filecache.DistributedCache.getLocalCacheFiles(conf);
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/FileOutputConfigurator.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/FileOutputConfigurator.java
new file mode 100644
index 0000000000..ee26f9a3b6
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/FileOutputConfigurator.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * @since 1.6.0
+ */
+public class FileOutputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link AccumuloConfiguration}.
+ *
+ * @since 1.6.0
+ */
+ public static enum Opts {
+ ACCUMULO_PROPERTIES;
+ }
+
+ /**
+ * The supported Accumulo properties we set in this OutputFormat, that change the behavior of the
+ * RecordWriter.<br>
+ * These properties correspond to the supported public static setter methods available to this
+ * class.
+ *
+ * @param property
+ * the Accumulo property to check
+ * @since 1.6.0
+ */
+ protected static Boolean isSupportedAccumuloProperty(Property property) {
+ switch (property) {
+ case TABLE_FILE_COMPRESSION_TYPE:
+ case TABLE_FILE_COMPRESSED_BLOCK_SIZE:
+ case TABLE_FILE_BLOCK_SIZE:
+ case TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX:
+ case TABLE_FILE_REPLICATION:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Helper for transforming Accumulo configuration properties into something that can be stored
+ * safely inside the Hadoop Job configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param property
+ * the supported Accumulo property
+ * @param value
+ * the value of the property to set
+ * @since 1.6.0
+ */
+ private static <T> void setAccumuloProperty(Class<?> implementingClass, Configuration conf,
+ Property property, T value) {
+ if (isSupportedAccumuloProperty(property)) {
+ String val = String.valueOf(value);
+ if (property.getType().isValidFormat(val))
+ conf.set(
+ enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + "." + property.getKey(),
+ val);
+ else
+ throw new IllegalArgumentException(
+ "Value is not appropriate for property type '" + property.getType() + "'");
+ } else
+ throw new IllegalArgumentException("Unsupported configuration property " + property.getKey());
+ }
+
+ /**
+ * This helper method provides an AccumuloConfiguration object constructed from the Accumulo
+ * defaults, and overridden with Accumulo properties that have been stored in the Job's
+ * configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @since 1.6.0
+ */
+ public static AccumuloConfiguration getAccumuloConfiguration(Class<?> implementingClass,
+ Configuration conf) {
+ String prefix = enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + ".";
+ ConfigurationCopy acuConf = new ConfigurationCopy(DefaultConfiguration.getInstance());
+ for (Entry<String,String> entry : conf)
+ if (entry.getKey().startsWith(prefix)) {
+ String propString = entry.getKey().substring(prefix.length());
+ Property prop = Property.getPropertyByKey(propString);
+ if (prop != null) {
+ acuConf.set(prop, entry.getValue());
+ } else if (Property.isValidTablePropertyKey(propString)) {
+ acuConf.set(propString, entry.getValue());
+ } else {
+ throw new IllegalArgumentException("Unknown accumulo file property " + propString);
+ }
+ }
+ return acuConf;
+ }
+
+ /**
+ * Sets the compression type to use for data blocks. Specifying a compression may require
+ * additional libraries to be available to your Job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param compressionType
+ * one of "none", "gz", "lzo", "snappy", or "zstd"
+ * @since 1.6.0
+ */
+ public static void setCompressionType(Class<?> implementingClass, Configuration conf,
+ String compressionType) {
+ if (compressionType == null
+ || !Arrays.asList("none", "gz", "lzo", "snappy", "zstd").contains(compressionType))
+ throw new IllegalArgumentException(
+ "Compression type must be one of: none, gz, lzo, snappy, zstd");
+ setAccumuloProperty(implementingClass, conf, Property.TABLE_FILE_COMPRESSION_TYPE,
+ compressionType);
+ }
+
+ /**
+ * Sets the size for data blocks within each file.<br>
+ * Data blocks are a span of key/value pairs stored in the file that are compressed and indexed as
+ * a group.
+ *
+ * <p>
+ * Making this value smaller may increase seek performance, but at the cost of increasing the size
+ * of the indexes (which can also affect seek performance).
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param dataBlockSize
+ * the block size, in bytes
+ * @since 1.6.0
+ */
+ public static void setDataBlockSize(Class<?> implementingClass, Configuration conf,
+ long dataBlockSize) {
+ setAccumuloProperty(implementingClass, conf, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE,
+ dataBlockSize);
+ }
+
+ /**
+ * Sets the size for file blocks in the file system; file blocks are managed, and replicated, by
+ * the underlying file system.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param fileBlockSize
+ * the block size, in bytes
+ * @since 1.6.0
+ */
+ public static void setFileBlockSize(Class<?> implementingClass, Configuration conf,
+ long fileBlockSize) {
+ setAccumuloProperty(implementingClass, conf, Property.TABLE_FILE_BLOCK_SIZE, fileBlockSize);
+ }
+
+ /**
+ * Sets the size for index blocks within each file; smaller blocks means a deeper index hierarchy
+ * within the file, while larger blocks mean a more shallow index hierarchy within the file. This
+ * can affect the performance of queries.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param indexBlockSize
+ * the block size, in bytes
+ * @since 1.6.0
+ */
+ public static void setIndexBlockSize(Class<?> implementingClass, Configuration conf,
+ long indexBlockSize) {
+ setAccumuloProperty(implementingClass, conf, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX,
+ indexBlockSize);
+ }
+
+ /**
+ * Sets the file system replication factor for the resulting file, overriding the file system
+ * default.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param replication
+ * the number of replicas for produced files
+ * @since 1.6.0
+ */
+ public static void setReplication(Class<?> implementingClass, Configuration conf,
+ int replication) {
+ setAccumuloProperty(implementingClass, conf, Property.TABLE_FILE_REPLICATION, replication);
+ }
+
+ /**
+ * @since 1.8.0
+ */
+ public static void setSampler(Class<?> implementingClass, Configuration conf,
+ SamplerConfiguration samplerConfig) {
+ Map<String,String> props = new SamplerConfigurationImpl(samplerConfig).toTablePropertiesMap();
+
+ Set<Entry<String,String>> es = props.entrySet();
+ for (Entry<String,String> entry : es) {
+ conf.set(enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + "." + entry.getKey(),
+ entry.getValue());
+ }
+ }
+
+ public static void setSummarizers(Class<?> implementingClass, Configuration conf,
+ SummarizerConfiguration[] sumarizerConfigs) {
+ Map<String,String> props = SummarizerConfiguration.toTableProperties(sumarizerConfigs);
+
+ for (Entry<String,String> entry : props.entrySet()) {
+ conf.set(enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + "." + entry.getKey(),
+ entry.getValue());
+ }
+ }
+
+}
diff --git a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/InputConfigurator.java
similarity index 99%
rename from client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
rename to hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/InputConfigurator.java
index 37046152bd..7f84729527 100644
--- a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/InputConfigurator.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.accumulo.core.client.mapreduce.lib.impl;
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
import static com.google.common.base.Preconditions.checkArgument;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -52,7 +52,6 @@
import org.apache.accumulo.core.client.impl.Table;
import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.client.impl.TabletLocator;
-import org.apache.accumulo.core.client.mapreduce.InputTableConfig;
import org.apache.accumulo.core.client.sample.SamplerConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -68,6 +67,7 @@
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.hadoop.mapreduce.InputTableConfig;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
@@ -793,8 +793,8 @@ public static void validatePermissions(Class<?> implementingClass, Configuration
}
/**
- * Returns the {@link org.apache.accumulo.core.client.mapreduce.InputTableConfig} for the
- * configuration based on the properties set using the single-table input methods.
+ * Returns the {@link InputTableConfig} for the configuration based on the properties set using
+ * the single-table input methods.
*
* @param implementingClass
* the class whose name will be used as a prefix for the property configuration key
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
new file mode 100644
index 0000000000..02c669b9c9
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
+
+import com.beust.jcommander.Parameter;
+
+public class MapReduceClientOnDefaultTable extends MapReduceClientOpts {
+ @Parameter(names = "--table", description = "table to use")
+ public String tableName;
+
+ public MapReduceClientOnDefaultTable(String table) {
+ this.tableName = table;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ @Override
+ public void setAccumuloConfigs(Job job) throws AccumuloSecurityException {
+ super.setAccumuloConfigs(job);
+ final String tableName = getTableName();
+ final ClientInfo info = getClientInfo();
+ AccumuloInputFormat.setClientInfo(job, info);
+ AccumuloInputFormat.setInputTableName(job, tableName);
+ AccumuloInputFormat.setScanAuthorizations(job, auths);
+ AccumuloOutputFormat.setClientInfo(job, info);
+ AccumuloOutputFormat.setCreateTables(job, true);
+ AccumuloOutputFormat.setDefaultTableName(job, tableName);
+ }
+
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
new file mode 100644
index 0000000000..b061ab84df
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
+
+import com.beust.jcommander.Parameter;
+
+public class MapReduceClientOnRequiredTable extends MapReduceClientOpts {
+
+ @Parameter(names = {"-t", "--table"}, required = true, description = "table to use")
+ private String tableName;
+
+ @Override
+ public void setAccumuloConfigs(Job job) throws AccumuloSecurityException {
+ super.setAccumuloConfigs(job);
+ AccumuloInputFormat.setInputTableName(job, getTableName());
+ AccumuloInputFormat.setScanAuthorizations(job, auths);
+ AccumuloOutputFormat.setCreateTables(job, true);
+ AccumuloOutputFormat.setDefaultTableName(job, getTableName());
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
new file mode 100644
index 0000000000..b1a76e030a
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
+
+import org.apache.accumulo.core.cli.ClientOpts;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Adds some MR awareness to the ClientOpts
+ */
+public class MapReduceClientOpts extends ClientOpts {
+ private static final Logger log = LoggerFactory.getLogger(MapReduceClientOpts.class);
+
+ public void setAccumuloConfigs(Job job) throws AccumuloSecurityException {
+ AccumuloInputFormat.setClientInfo(job, this.getClientInfo());
+ AccumuloOutputFormat.setClientInfo(job, this.getClientInfo());
+ }
+
+ @Override
+ public AuthenticationToken getToken() {
+ AuthenticationToken authToken = super.getToken();
+ // For MapReduce, Kerberos credentials don't make it to the Mappers and Reducers,
+ // so we need to request a delegation token and use that instead.
+ if (authToken instanceof KerberosToken) {
+ log.info("Received KerberosToken, fetching DelegationToken for MapReduce");
+ final KerberosToken krbToken = (KerberosToken) authToken;
+
+ try {
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+ if (!user.hasKerberosCredentials()) {
+ throw new IllegalStateException("Expected current user to have Kerberos credentials");
+ }
+
+ String newPrincipal = user.getUserName();
+ log.info("Obtaining delegation token for {}", newPrincipal);
+
+ setPrincipal(newPrincipal);
+ AccumuloClient client = Accumulo.newClient().usingClientInfo(getClientInfo())
+ .usingToken(newPrincipal, krbToken).build();
+
+ // Do the explicit check to see if the user has the permission to get a delegation token
+ if (!client.securityOperations().hasSystemPermission(client.whoami(),
+ SystemPermission.OBTAIN_DELEGATION_TOKEN)) {
+ log.error(
+ "{} doesn't have the {} SystemPermission neccesary to obtain a delegation"
+ + " token. MapReduce tasks cannot automatically use the client's"
+ + " credentials on remote servers. Delegation tokens provide a means to run"
+ + " MapReduce without distributing the user's credentials.",
+ user.getUserName(), SystemPermission.OBTAIN_DELEGATION_TOKEN.name());
+ throw new IllegalStateException(
+ client.whoami() + " does not have permission to obtain a delegation token");
+ }
+
+ // Get the delegation token from Accumulo
+ return client.securityOperations().getDelegationToken(new DelegationTokenConfig());
+ } catch (Exception e) {
+ final String msg = "Failed to acquire DelegationToken for use with MapReduce";
+ log.error(msg, e);
+ throw new RuntimeException(msg, e);
+ }
+ }
+ return authToken;
+ }
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/OutputConfigurator.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/OutputConfigurator.java
new file mode 100644
index 0000000000..12ce5726dd
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/OutputConfigurator.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * @since 1.6.0
+ */
+public class OutputConfigurator extends ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link BatchWriter}.
+ *
+ * @since 1.6.0
+ */
+ public static enum WriteOpts {
+ DEFAULT_TABLE_NAME, BATCH_WRITER_CONFIG
+ }
+
+ /**
+ * Configuration keys for various features.
+ *
+ * @since 1.6.0
+ */
+ public static enum Features {
+ CAN_CREATE_TABLES, SIMULATION_MODE
+ }
+
+ /**
+ * Sets the default table name to use if one emits a null in place of a table name for a given
+ * mutation. Table names can only be alpha-numeric and underscores.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param tableName
+ * the table to use when the tablename is null in the write call
+ * @since 1.6.0
+ */
+ public static void setDefaultTableName(Class<?> implementingClass, Configuration conf,
+ String tableName) {
+ if (tableName != null)
+ conf.set(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME), tableName);
+ }
+
+ /**
+ * Gets the default table name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the default table name
+ * @since 1.6.0
+ * @see #setDefaultTableName(Class, Configuration, String)
+ */
+ public static String getDefaultTableName(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, WriteOpts.DEFAULT_TABLE_NAME));
+ }
+
+ /**
+ * Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new
+ * {@link BatchWriterConfig}, with sensible built-in defaults is used. Setting the configuration
+ * multiple times overwrites any previous configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param bwConfig
+ * the configuration for the {@link BatchWriter}
+ * @since 1.6.0
+ */
+ public static void setBatchWriterOptions(Class<?> implementingClass, Configuration conf,
+ BatchWriterConfig bwConfig) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ String serialized;
+ try {
+ bwConfig.write(new DataOutputStream(baos));
+ serialized = new String(baos.toByteArray(), UTF_8);
+ baos.close();
+ } catch (IOException e) {
+ throw new IllegalArgumentException(
+ "unable to serialize " + BatchWriterConfig.class.getName());
+ }
+ conf.set(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG), serialized);
+ }
+
+ /**
+ * Gets the {@link BatchWriterConfig} settings.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the configuration object
+ * @since 1.6.0
+ * @see #setBatchWriterOptions(Class, Configuration, BatchWriterConfig)
+ */
+ public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass,
+ Configuration conf) {
+ String serialized = conf.get(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG));
+ BatchWriterConfig bwConfig = new BatchWriterConfig();
+ if (serialized == null || serialized.isEmpty()) {
+ return bwConfig;
+ } else {
+ try {
+ ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(UTF_8));
+ bwConfig.readFields(new DataInputStream(bais));
+ bais.close();
+ return bwConfig;
+ } catch (IOException e) {
+ throw new IllegalArgumentException(
+ "unable to serialize " + BatchWriterConfig.class.getName());
+ }
+ }
+ }
+
+ /**
+ * Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric
+ * and underscores.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setCreateTables(Class<?> implementingClass, Configuration conf,
+ boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), enableFeature);
+ }
+
+ /**
+ * Determines whether tables are permitted to be created as needed.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is disabled, false otherwise
+ * @since 1.6.0
+ * @see #setCreateTables(Class, Configuration, boolean)
+ */
+ public static Boolean canCreateTables(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.CAN_CREATE_TABLES), false);
+ }
+
+ /**
+ * Sets the directive to use simulation mode for this job. In simulation mode, no output is
+ * produced. This is useful for testing.
+ *
+ * <p>
+ * By default, this feature is <b>disabled</b>.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param enableFeature
+ * the feature is enabled if true, disabled otherwise
+ * @since 1.6.0
+ */
+ public static void setSimulationMode(Class<?> implementingClass, Configuration conf,
+ boolean enableFeature) {
+ conf.setBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), enableFeature);
+ }
+
+ /**
+ * Determines whether this feature is enabled.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the feature is enabled, false otherwise
+ * @since 1.6.0
+ * @see #setSimulationMode(Class, Configuration, boolean)
+ */
+ public static Boolean getSimulationMode(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, Features.SIMULATION_MODE), false);
+ }
+
+}
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/package-info.java b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/package-info.java
new file mode 100644
index 0000000000..504087565b
--- /dev/null
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/package-info.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package exists to store common helpers for configuring MapReduce jobs in a single location.
+ * It contains static configurator methods, stored in classes separate from the things they
+ * configure (typically, {@link org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat}/
+ * {@link org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat} and related classes in
+ * compatible frameworks), rather than storing them in those InputFormats/OutputFormats, so as not
+ * to clutter their API with methods that don't match the conventions for that framework. These
+ * classes may be useful to input/output plugins for other frameworks, so they can reuse the same
+ * configuration options and/or serialize them into a {@link org.apache.hadoop.conf.Configuration}
+ * instance in a standard way.
+ *
+ * <p>
+ * It is not expected these will change much (except when new features are added), but end users
+ * should not use these classes. They should use the static configurators on the
+ * {@link org.apache.hadoop.mapreduce.InputFormat} or
+ * {@link org.apache.hadoop.mapreduce.OutputFormat} they are configuring, which in turn may use
+ * these classes to implement their own static configurators. Once again, these classes are intended
+ * for internal use, but may be useful to developers of plugins for other frameworks that read/write
+ * to Accumulo.
+ *
+ * @since 1.6.0
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloFileOutputFormatTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloFileOutputFormatTest.java
new file mode 100644
index 0000000000..16d2d8f101
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloFileOutputFormatTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.accumulo.core.client.sample.RowSampler;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.client.summary.CountingSummarizer;
+import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
+import org.apache.accumulo.core.client.summary.summarizers.FamilySummarizer;
+import org.apache.accumulo.core.client.summary.summarizers.VisibilitySummarizer;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.FileOutputConfigurator;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.Test;
+
+public class AccumuloFileOutputFormatTest {
+
+ @Test
+ public void validateConfiguration() throws IOException, InterruptedException {
+
+ int a = 7;
+ long b = 300L;
+ long c = 50L;
+ long d = 10L;
+ String e = "snappy";
+ SamplerConfiguration samplerConfig = new SamplerConfiguration(RowSampler.class.getName());
+ samplerConfig.addOption("hasher", "murmur3_32");
+ samplerConfig.addOption("modulus", "109");
+
+ SummarizerConfiguration sc1 = SummarizerConfiguration.builder(VisibilitySummarizer.class)
+ .addOption(CountingSummarizer.MAX_COUNTERS_OPT, 2048).build();
+ SummarizerConfiguration sc2 = SummarizerConfiguration.builder(FamilySummarizer.class)
+ .addOption(CountingSummarizer.MAX_COUNTERS_OPT, 256).build();
+
+ JobConf job = new JobConf();
+ AccumuloFileOutputFormat.setReplication(job, a);
+ AccumuloFileOutputFormat.setFileBlockSize(job, b);
+ AccumuloFileOutputFormat.setDataBlockSize(job, c);
+ AccumuloFileOutputFormat.setIndexBlockSize(job, d);
+ AccumuloFileOutputFormat.setCompressionType(job, e);
+ AccumuloFileOutputFormat.setSampler(job, samplerConfig);
+ AccumuloFileOutputFormat.setSummarizers(job, sc1, sc2);
+
+ AccumuloConfiguration acuconf = FileOutputConfigurator
+ .getAccumuloConfiguration(AccumuloFileOutputFormat.class, job);
+
+ assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+ assertEquals(300L, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE));
+ assertEquals(50L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+ assertEquals(10L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+ assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+ assertEquals(new SamplerConfigurationImpl(samplerConfig),
+ SamplerConfigurationImpl.newSamplerConfig(acuconf));
+
+ Collection<SummarizerConfiguration> summarizerConfigs = SummarizerConfiguration
+ .fromTableProperties(acuconf);
+ assertEquals(2, summarizerConfigs.size());
+ assertTrue(summarizerConfigs.contains(sc1));
+ assertTrue(summarizerConfigs.contains(sc2));
+
+ a = 17;
+ b = 1300L;
+ c = 150L;
+ d = 110L;
+ e = "lzo";
+ samplerConfig = new SamplerConfiguration(RowSampler.class.getName());
+ samplerConfig.addOption("hasher", "md5");
+ samplerConfig.addOption("modulus", "100003");
+
+ job = new JobConf();
+ AccumuloFileOutputFormat.setReplication(job, a);
+ AccumuloFileOutputFormat.setFileBlockSize(job, b);
+ AccumuloFileOutputFormat.setDataBlockSize(job, c);
+ AccumuloFileOutputFormat.setIndexBlockSize(job, d);
+ AccumuloFileOutputFormat.setCompressionType(job, e);
+ AccumuloFileOutputFormat.setSampler(job, samplerConfig);
+
+ acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job);
+
+ assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+ assertEquals(1300L, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE));
+ assertEquals(150L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+ assertEquals(110L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+ assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+ assertEquals(new SamplerConfigurationImpl(samplerConfig),
+ SamplerConfigurationImpl.newSamplerConfig(acuconf));
+
+ summarizerConfigs = SummarizerConfiguration.fromTableProperties(acuconf);
+ assertEquals(0, summarizerConfigs.size());
+ }
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormatTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormatTest.java
new file mode 100644
index 0000000000..7b0a3daa24
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormatTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.List;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class AccumuloInputFormatTest {
+
+ private JobConf job;
+
+ @Rule
+ public TestName test = new TestName();
+
+ @Before
+ public void createJob() {
+ job = new JobConf();
+ }
+
+ /**
+ * Check that the iterator configuration is getting stored in the Job conf correctly.
+ */
+ @Test
+ public void testSetIterator() throws IOException {
+ IteratorSetting is = new IteratorSetting(1, "WholeRow",
+ "org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job, is);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
+ String iterators = job.get("AccumuloInputFormat.ScanOpts.Iterators");
+ assertEquals(Base64.getEncoder().encodeToString(baos.toByteArray()), iterators);
+ }
+
+ @Test
+ public void testAddIterator() throws IOException {
+ AccumuloInputFormat.addIterator(job,
+ new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions",
+ "org.apache.accumulo.core.iterators.VersioningIterator"));
+ IteratorSetting iter = new IteratorSetting(3, "Count",
+ "org.apache.accumulo.core.iterators.CountingIterator");
+ iter.addOption("v1", "1");
+ iter.addOption("junk", "\0omg:!\\xyzzy");
+ AccumuloInputFormat.addIterator(job, iter);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertEquals(3, list.size());
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator",
+ setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator",
+ setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+ }
+
+ /**
+ * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR
+ * character (':') and ITERATOR_SEPARATOR (',') characters. There should be no exceptions thrown
+ * when trying to parse these types of option entries.
+ *
+ * This test makes sure that the expected raw values, as appears in the Job, are equal to what's
+ * expected.
+ */
+ @Test
+ public void testIteratorOptionEncoding() throws Throwable {
+ String key = "colon:delimited:key";
+ String value = "comma,delimited,value";
+ IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+ someSetting.addOption(key, value);
+ AccumuloInputFormat.addIterator(job, someSetting);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+
+ someSetting.addOption(key + "2", value);
+ someSetting.setPriority(2);
+ someSetting.setName("it2");
+ AccumuloInputFormat.addIterator(job, someSetting);
+ list = AccumuloInputFormat.getIterators(job);
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
+ }
+
+ /**
+ * Test getting iterator settings for multiple iterators set
+ */
+ @Test
+ public void testGetIteratorSettings() throws IOException {
+ AccumuloInputFormat.addIterator(job,
+ new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions",
+ "org.apache.accumulo.core.iterators.VersioningIterator"));
+ AccumuloInputFormat.addIterator(job,
+ new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertEquals(3, list.size());
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator",
+ setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+
+ }
+
+ @Test
+ public void testSetRegex() throws IOException {
+ String regex = ">\"*%<>\'\\";
+
+ IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ AccumuloInputFormat.addIterator(job, is);
+
+ assertEquals(regex, AccumuloInputFormat.getIterators(job).get(0).getName());
+ }
+
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloMultiTableInputFormatTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloMultiTableInputFormatTest.java
new file mode 100644
index 0000000000..cc925c1844
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloMultiTableInputFormatTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.hadoop.mapreduce.InputTableConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class AccumuloMultiTableInputFormatTest {
+
+ @Rule
+ public TestName testName = new TestName();
+
+ /**
+ * Verify {@link org.apache.accumulo.hadoop.mapreduce.InputTableConfig} objects get correctly
+ * serialized in the JobContext.
+ */
+ @Test
+ public void testTableQueryConfigSerialization() throws IOException {
+ String table1Name = testName.getMethodName() + "1";
+ String table2Name = testName.getMethodName() + "2";
+ JobConf job = new JobConf();
+
+ InputTableConfig table1 = new InputTableConfig()
+ .setRanges(Collections.singletonList(new Range("a", "b")))
+ .fetchColumns(Collections.singleton(new Pair<>(new Text("CF1"), new Text("CQ1"))))
+ .setIterators(Collections.singletonList(new IteratorSetting(50, "iter1", "iterclass1")));
+
+ InputTableConfig table2 = new InputTableConfig()
+ .setRanges(Collections.singletonList(new Range("a", "b")))
+ .fetchColumns(Collections.singleton(new Pair<>(new Text("CF1"), new Text("CQ1"))))
+ .setIterators(Collections.singletonList(new IteratorSetting(50, "iter1", "iterclass1")));
+
+ Map<String,InputTableConfig> configMap = new HashMap<>();
+ configMap.put(table1Name, table1);
+ configMap.put(table2Name, table2);
+ AccumuloMultiTableInputFormat.setInputTableConfigs(job, configMap);
+
+ assertEquals(table1, AccumuloMultiTableInputFormat.getInputTableConfig(job, table1Name));
+ assertEquals(table2, AccumuloMultiTableInputFormat.getInputTableConfig(job, table2Name));
+ }
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloOutputFormatTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloOutputFormatTest.java
new file mode 100644
index 0000000000..716f4e16c6
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloOutputFormatTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.Test;
+
+public class AccumuloOutputFormatTest {
+
+ @Test
+ public void testBWSettings() throws IOException {
+ JobConf job = new JobConf();
+
+ // make sure we aren't testing defaults
+ final BatchWriterConfig bwDefaults = new BatchWriterConfig();
+ assertNotEquals(7654321L, bwDefaults.getMaxLatency(TimeUnit.MILLISECONDS));
+ assertNotEquals(9898989L, bwDefaults.getTimeout(TimeUnit.MILLISECONDS));
+ assertNotEquals(42, bwDefaults.getMaxWriteThreads());
+ assertNotEquals(1123581321L, bwDefaults.getMaxMemory());
+
+ final BatchWriterConfig bwConfig = new BatchWriterConfig();
+ bwConfig.setMaxLatency(7654321L, TimeUnit.MILLISECONDS);
+ bwConfig.setTimeout(9898989L, TimeUnit.MILLISECONDS);
+ bwConfig.setMaxWriteThreads(42);
+ bwConfig.setMaxMemory(1123581321L);
+ AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig);
+
+ AccumuloOutputFormat myAOF = new AccumuloOutputFormat() {
+ @Override
+ public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
+ BatchWriterConfig bwOpts = getBatchWriterOptions(job);
+
+ // passive check
+ assertEquals(bwConfig.getMaxLatency(TimeUnit.MILLISECONDS),
+ bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
+ assertEquals(bwConfig.getTimeout(TimeUnit.MILLISECONDS),
+ bwOpts.getTimeout(TimeUnit.MILLISECONDS));
+ assertEquals(bwConfig.getMaxWriteThreads(), bwOpts.getMaxWriteThreads());
+ assertEquals(bwConfig.getMaxMemory(), bwOpts.getMaxMemory());
+
+ // explicit check
+ assertEquals(7654321L, bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
+ assertEquals(9898989L, bwOpts.getTimeout(TimeUnit.MILLISECONDS));
+ assertEquals(42, bwOpts.getMaxWriteThreads());
+ assertEquals(1123581321L, bwOpts.getMaxMemory());
+
+ }
+ };
+ myAOF.checkOutputSpecs(null, job);
+ }
+
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/RangeInputSplitTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/RangeInputSplitTest.java
new file mode 100644
index 0000000000..e7988d9c84
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/RangeInputSplitTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapred;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+
+public class RangeInputSplitTest {
+
+ @Test
+ public void testSimpleWritable() throws IOException {
+ RangeInputSplit split = new RangeInputSplit("table", "1", new Range(new Key("a"), new Key("b")),
+ new String[] {"localhost"});
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ split.write(dos);
+
+ RangeInputSplit newSplit = new RangeInputSplit();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ newSplit.readFields(dis);
+
+ assertEquals(split.getRange(), newSplit.getRange());
+ assertTrue(Arrays.equals(split.getLocations(), newSplit.getLocations()));
+ }
+
+ @Test
+ public void testAllFieldsWritable() throws IOException {
+ RangeInputSplit split = new RangeInputSplit("table", "1", new Range(new Key("a"), new Key("b")),
+ new String[] {"localhost"});
+
+ Set<Pair<Text,Text>> fetchedColumns = new HashSet<>();
+
+ fetchedColumns.add(new Pair<>(new Text("colf1"), new Text("colq1")));
+ fetchedColumns.add(new Pair<>(new Text("colf2"), new Text("colq2")));
+
+ // Fake some iterators
+ ArrayList<IteratorSetting> iterators = new ArrayList<>();
+ IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
+ setting.addOption("foo", "bar");
+ iterators.add(setting);
+
+ setting = new IteratorSetting(100, WholeRowIterator.class);
+ setting.addOption("bar", "foo");
+ iterators.add(setting);
+
+ split.setOffline(true);
+ split.setIsolatedScan(true);
+ split.setUsesLocalIterators(true);
+ split.setFetchedColumns(fetchedColumns);
+ split.setIterators(iterators);
+ split.setLogLevel(Level.WARN);
+ split.setExecutionHints(ImmutableMap.of("priority", "9"));
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ split.write(dos);
+
+ RangeInputSplit newSplit = new RangeInputSplit();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ newSplit.readFields(dis);
+
+ assertEquals(split.getRange(), newSplit.getRange());
+ assertArrayEquals(split.getLocations(), newSplit.getLocations());
+
+ assertEquals(split.isOffline(), newSplit.isOffline());
+ assertEquals(split.isIsolatedScan(), newSplit.isOffline());
+ assertEquals(split.usesLocalIterators(), newSplit.usesLocalIterators());
+ assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
+ assertEquals(split.getIterators(), newSplit.getIterators());
+ assertEquals(split.getLogLevel(), newSplit.getLogLevel());
+ assertEquals(split.getExecutionHints(), newSplit.getExecutionHints());
+ }
+
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloFileOutputFormatTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloFileOutputFormatTest.java
new file mode 100644
index 0000000000..9eb3e381de
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloFileOutputFormatTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.accumulo.core.client.sample.RowSampler;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.client.summary.CountingSummarizer;
+import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
+import org.apache.accumulo.core.client.summary.summarizers.FamilySummarizer;
+import org.apache.accumulo.core.client.summary.summarizers.VisibilitySummarizer;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.accumulo.hadoopImpl.mapreduce.lib.FileOutputConfigurator;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class AccumuloFileOutputFormatTest {
+
+ @Test
+ public void validateConfiguration() throws IOException, InterruptedException {
+
+ int a = 7;
+ long b = 300L;
+ long c = 50L;
+ long d = 10L;
+ String e = "snappy";
+ SamplerConfiguration samplerConfig = new SamplerConfiguration(RowSampler.class.getName());
+ samplerConfig.addOption("hasher", "murmur3_32");
+ samplerConfig.addOption("modulus", "109");
+
+ SummarizerConfiguration sc1 = SummarizerConfiguration.builder(VisibilitySummarizer.class)
+ .addOption(CountingSummarizer.MAX_COUNTERS_OPT, 2048).build();
+ SummarizerConfiguration sc2 = SummarizerConfiguration.builder(FamilySummarizer.class)
+ .addOption(CountingSummarizer.MAX_COUNTERS_OPT, 256).build();
+
+ Job job1 = Job.getInstance();
+ AccumuloFileOutputFormat.setReplication(job1, a);
+ AccumuloFileOutputFormat.setFileBlockSize(job1, b);
+ AccumuloFileOutputFormat.setDataBlockSize(job1, c);
+ AccumuloFileOutputFormat.setIndexBlockSize(job1, d);
+ AccumuloFileOutputFormat.setCompressionType(job1, e);
+ AccumuloFileOutputFormat.setSampler(job1, samplerConfig);
+ AccumuloFileOutputFormat.setSummarizers(job1, sc1, sc2);
+
+ AccumuloConfiguration acuconf = FileOutputConfigurator
+ .getAccumuloConfiguration(AccumuloFileOutputFormat.class, job1.getConfiguration());
+
+ assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+ assertEquals(300L, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE));
+ assertEquals(50L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+ assertEquals(10L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+ assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+ assertEquals(new SamplerConfigurationImpl(samplerConfig),
+ SamplerConfigurationImpl.newSamplerConfig(acuconf));
+
+ Collection<SummarizerConfiguration> summarizerConfigs = SummarizerConfiguration
+ .fromTableProperties(acuconf);
+ assertEquals(2, summarizerConfigs.size());
+ assertTrue(summarizerConfigs.contains(sc1));
+ assertTrue(summarizerConfigs.contains(sc2));
+
+ a = 17;
+ b = 1300L;
+ c = 150L;
+ d = 110L;
+ e = "lzo";
+ samplerConfig = new SamplerConfiguration(RowSampler.class.getName());
+ samplerConfig.addOption("hasher", "md5");
+ samplerConfig.addOption("modulus", "100003");
+
+ Job job2 = Job.getInstance();
+ AccumuloFileOutputFormat.setReplication(job2, a);
+ AccumuloFileOutputFormat.setFileBlockSize(job2, b);
+ AccumuloFileOutputFormat.setDataBlockSize(job2, c);
+ AccumuloFileOutputFormat.setIndexBlockSize(job2, d);
+ AccumuloFileOutputFormat.setCompressionType(job2, e);
+ AccumuloFileOutputFormat.setSampler(job2, samplerConfig);
+
+ acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class,
+ job2.getConfiguration());
+
+ assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+ assertEquals(1300L, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE));
+ assertEquals(150L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+ assertEquals(110L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+ assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+ assertEquals(new SamplerConfigurationImpl(samplerConfig),
+ SamplerConfigurationImpl.newSamplerConfig(acuconf));
+
+ summarizerConfigs = SummarizerConfiguration.fromTableProperties(acuconf);
+ assertEquals(0, summarizerConfigs.size());
+
+ }
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormatTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormatTest.java
new file mode 100644
index 0000000000..18fc46e6b6
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormatTest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class AccumuloInputFormatTest {
+
+ /**
+ * Check that the iterator configuration is getting stored in the Job conf correctly.
+ */
+ @Test
+ public void testSetIterator() throws IOException {
+ Job job = Job.getInstance();
+
+ IteratorSetting is = new IteratorSetting(1, "WholeRow",
+ "org.apache.accumulo.core.iterators.WholeRowIterator");
+ AccumuloInputFormat.addIterator(job, is);
+ Configuration conf = job.getConfiguration();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ is.write(new DataOutputStream(baos));
+ String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
+ assertEquals(Base64.getEncoder().encodeToString(baos.toByteArray()), iterators);
+ }
+
+ @Test
+ public void testAddIterator() throws IOException {
+ Job job = Job.getInstance();
+
+ AccumuloInputFormat.addIterator(job,
+ new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions",
+ "org.apache.accumulo.core.iterators.VersioningIterator"));
+ IteratorSetting iter = new IteratorSetting(3, "Count",
+ "org.apache.accumulo.core.iterators.CountingIterator");
+ iter.addOption("v1", "1");
+ iter.addOption("junk", "\0omg:!\\xyzzy");
+ AccumuloInputFormat.addIterator(job, iter);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertEquals(3, list.size());
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator",
+ setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator",
+ setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+ assertEquals(0, setting.getOptions().size());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+ assertEquals(2, setting.getOptions().size());
+ assertEquals("1", setting.getOptions().get("v1"));
+ assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+ }
+
+ /**
+ * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR
+ * character (':') and ITERATOR_SEPARATOR (',') characters. There should be no exceptions thrown
+ * when trying to parse these types of option entries.
+ *
+ * This test makes sure that the expected raw values, as appears in the Job, are equal to what's
+ * expected.
+ */
+ @Test
+ public void testIteratorOptionEncoding() throws Throwable {
+ String key = "colon:delimited:key";
+ String value = "comma,delimited,value";
+ IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+ someSetting.addOption(key, value);
+ Job job = Job.getInstance();
+ AccumuloInputFormat.addIterator(job, someSetting);
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+ assertEquals(1, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+
+ someSetting.addOption(key + "2", value);
+ someSetting.setPriority(2);
+ someSetting.setName("it2");
+ AccumuloInputFormat.addIterator(job, someSetting);
+ list = AccumuloInputFormat.getIterators(job);
+ assertEquals(2, list.size());
+ assertEquals(1, list.get(0).getOptions().size());
+ assertEquals(list.get(0).getOptions().get(key), value);
+ assertEquals(2, list.get(1).getOptions().size());
+ assertEquals(list.get(1).getOptions().get(key), value);
+ assertEquals(list.get(1).getOptions().get(key + "2"), value);
+ }
+
+ /**
+ * Test getting iterator settings for multiple iterators set
+ */
+ @Test
+ public void testGetIteratorSettings() throws IOException {
+ Job job = Job.getInstance();
+
+ AccumuloInputFormat.addIterator(job,
+ new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+ AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions",
+ "org.apache.accumulo.core.iterators.VersioningIterator"));
+ AccumuloInputFormat.addIterator(job,
+ new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+
+ List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+
+ // Check the list size
+ assertEquals(3, list.size());
+
+ // Walk the list and make sure our settings are correct
+ IteratorSetting setting = list.get(0);
+ assertEquals(1, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+ assertEquals("WholeRow", setting.getName());
+
+ setting = list.get(1);
+ assertEquals(2, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.VersioningIterator",
+ setting.getIteratorClass());
+ assertEquals("Versions", setting.getName());
+
+ setting = list.get(2);
+ assertEquals(3, setting.getPriority());
+ assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+ assertEquals("Count", setting.getName());
+
+ }
+
+ @Test
+ public void testSetRegex() throws IOException {
+ Job job = Job.getInstance();
+
+ String regex = ">\"*%<>\'\\";
+
+ IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ AccumuloInputFormat.addIterator(job, is);
+
+ assertEquals(regex, AccumuloInputFormat.getIterators(job).get(0).getName());
+ }
+
+ @Test
+ public void testEmptyColumnFamily() throws IOException {
+ Job job = Job.getInstance();
+ Set<Pair<Text,Text>> cols = new HashSet<>();
+ cols.add(new Pair<>(new Text(""), null));
+ cols.add(new Pair<>(new Text("foo"), new Text("bar")));
+ cols.add(new Pair<>(new Text(""), new Text("bar")));
+ cols.add(new Pair<>(new Text(""), new Text("")));
+ cols.add(new Pair<>(new Text("foo"), new Text("")));
+ AccumuloInputFormat.fetchColumns(job, cols);
+ Set<Pair<Text,Text>> setCols = AccumuloInputFormat.getFetchedColumns(job);
+ assertEquals(cols, setCols);
+ }
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloMultiTableInputFormatTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloMultiTableInputFormatTest.java
new file mode 100644
index 0000000000..18b1b6b83b
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloMultiTableInputFormatTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class AccumuloMultiTableInputFormatTest {
+
+ @Rule
+ public TestName testName = new TestName();
+
+ /**
+ * Verify {@link InputTableConfig} objects get correctly serialized in the JobContext.
+ */
+ @Test
+ public void testInputTableConfigSerialization() throws IOException {
+ String table1 = testName.getMethodName() + "1";
+ String table2 = testName.getMethodName() + "2";
+ Job job = Job.getInstance();
+
+ InputTableConfig tableConfig = new InputTableConfig()
+ .setRanges(Collections.singletonList(new Range("a", "b")))
+ .fetchColumns(Collections.singleton(new Pair<>(new Text("CF1"), new Text("CQ1"))))
+ .setIterators(Collections.singletonList(new IteratorSetting(50, "iter1", "iterclass1")));
+
+ Map<String,InputTableConfig> configMap = new HashMap<>();
+ configMap.put(table1, tableConfig);
+ configMap.put(table2, tableConfig);
+
+ AccumuloMultiTableInputFormat.setInputTableConfigs(job, configMap);
+
+ assertEquals(tableConfig, AccumuloMultiTableInputFormat.getInputTableConfig(job, table1));
+ assertEquals(tableConfig, AccumuloMultiTableInputFormat.getInputTableConfig(job, table2));
+ }
+
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloOutputFormatTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloOutputFormatTest.java
new file mode 100644
index 0000000000..687d276e0b
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloOutputFormatTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Test;
+
+public class AccumuloOutputFormatTest {
+
+ @Test
+ public void testBWSettings() throws IOException {
+ Job job = Job.getInstance();
+
+ // make sure we aren't testing defaults
+ final BatchWriterConfig bwDefaults = new BatchWriterConfig();
+ assertNotEquals(7654321L, bwDefaults.getMaxLatency(TimeUnit.MILLISECONDS));
+ assertNotEquals(9898989L, bwDefaults.getTimeout(TimeUnit.MILLISECONDS));
+ assertNotEquals(42, bwDefaults.getMaxWriteThreads());
+ assertNotEquals(1123581321L, bwDefaults.getMaxMemory());
+
+ final BatchWriterConfig bwConfig = new BatchWriterConfig();
+ bwConfig.setMaxLatency(7654321L, TimeUnit.MILLISECONDS);
+ bwConfig.setTimeout(9898989L, TimeUnit.MILLISECONDS);
+ bwConfig.setMaxWriteThreads(42);
+ bwConfig.setMaxMemory(1123581321L);
+ AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig);
+
+ AccumuloOutputFormat myAOF = new AccumuloOutputFormat() {
+ @Override
+ public void checkOutputSpecs(JobContext job) throws IOException {
+ BatchWriterConfig bwOpts = getBatchWriterOptions(job);
+
+ // passive check
+ assertEquals(bwConfig.getMaxLatency(TimeUnit.MILLISECONDS),
+ bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
+ assertEquals(bwConfig.getTimeout(TimeUnit.MILLISECONDS),
+ bwOpts.getTimeout(TimeUnit.MILLISECONDS));
+ assertEquals(bwConfig.getMaxWriteThreads(), bwOpts.getMaxWriteThreads());
+ assertEquals(bwConfig.getMaxMemory(), bwOpts.getMaxMemory());
+
+ // explicit check
+ assertEquals(7654321L, bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
+ assertEquals(9898989L, bwOpts.getTimeout(TimeUnit.MILLISECONDS));
+ assertEquals(42, bwOpts.getMaxWriteThreads());
+ assertEquals(1123581321L, bwOpts.getMaxMemory());
+
+ }
+ };
+ myAOF.checkOutputSpecs(job);
+ }
+
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/InputTableConfigTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/InputTableConfigTest.java
new file mode 100644
index 0000000000..847981f327
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/InputTableConfigTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+
+public class InputTableConfigTest {
+
+ private InputTableConfig tableQueryConfig;
+
+ @Before
+ public void setUp() {
+ tableQueryConfig = new InputTableConfig();
+ }
+
+ @Test
+ public void testSerialization_OnlyTable() throws IOException {
+ byte[] serialized = serialize(tableQueryConfig);
+ InputTableConfig actualConfig = deserialize(serialized);
+
+ assertEquals(tableQueryConfig, actualConfig);
+ }
+
+ @Test
+ public void testSerialization_allBooleans() throws IOException {
+ tableQueryConfig.setAutoAdjustRanges(false);
+ tableQueryConfig.setOfflineScan(true);
+ tableQueryConfig.setUseIsolatedScanners(true);
+ tableQueryConfig.setUseLocalIterators(true);
+ byte[] serialized = serialize(tableQueryConfig);
+ InputTableConfig actualConfig = deserialize(serialized);
+
+ assertEquals(tableQueryConfig, actualConfig);
+ }
+
+ @Test
+ public void testSerialization_ranges() throws IOException {
+ List<Range> ranges = new ArrayList<>();
+ ranges.add(new Range("a", "b"));
+ ranges.add(new Range("c", "d"));
+ tableQueryConfig.setRanges(ranges);
+
+ byte[] serialized = serialize(tableQueryConfig);
+ InputTableConfig actualConfig = deserialize(serialized);
+
+ assertEquals(ranges, actualConfig.getRanges());
+ }
+
+ @Test
+ public void testSerialization_columns() throws IOException {
+ Set<Pair<Text,Text>> columns = new HashSet<>();
+ columns.add(new Pair<>(new Text("cf1"), new Text("cq1")));
+ columns.add(new Pair<>(new Text("cf2"), null));
+ tableQueryConfig.fetchColumns(columns);
+
+ byte[] serialized = serialize(tableQueryConfig);
+ InputTableConfig actualConfig = deserialize(serialized);
+
+ assertEquals(actualConfig.getFetchedColumns(), columns);
+ }
+
+ @Test
+ public void testSerialization_iterators() throws IOException {
+ List<IteratorSetting> settings = new ArrayList<>();
+ settings.add(new IteratorSetting(50, "iter", "iterclass"));
+ settings.add(new IteratorSetting(55, "iter2", "iterclass2"));
+ tableQueryConfig.setIterators(settings);
+ byte[] serialized = serialize(tableQueryConfig);
+ InputTableConfig actualConfig = deserialize(serialized);
+ assertEquals(actualConfig.getIterators(), settings);
+
+ }
+
+ @Test
+ public void testSamplerConfig() throws IOException {
+ SamplerConfiguration sc = new SamplerConfiguration("com.foo.S1").addOption("k1", "v1");
+ tableQueryConfig.setSamplerConfiguration(sc);
+ InputTableConfig actualConfig = deserialize(serialize(tableQueryConfig));
+ assertEquals(sc, actualConfig.getSamplerConfiguration());
+ }
+
+ @Test
+ public void testExecutionHints() throws IOException {
+ tableQueryConfig.setExecutionHints(ImmutableMap.of("priority", "9"));
+ InputTableConfig actualConfig = deserialize(serialize(tableQueryConfig));
+ assertEquals(ImmutableMap.of("priority", "9"), actualConfig.getExecutionHints());
+ }
+
+ private byte[] serialize(InputTableConfig tableQueryConfig) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ tableQueryConfig.write(new DataOutputStream(baos));
+ baos.close();
+ return baos.toByteArray();
+ }
+
+ private InputTableConfig deserialize(byte[] bytes) throws IOException {
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ InputTableConfig actualConfig = new InputTableConfig(new DataInputStream(bais));
+ bais.close();
+ return actualConfig;
+ }
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/RangeInputSplitTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/RangeInputSplitTest.java
new file mode 100644
index 0000000000..8be9c35c7d
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/RangeInputSplitTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+
+public class RangeInputSplitTest {
+
+ @Test
+ public void testSimpleWritable() throws IOException {
+ RangeInputSplit split = new RangeInputSplit("table", "1", new Range(new Key("a"), new Key("b")),
+ new String[] {"localhost"});
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ split.write(dos);
+
+ RangeInputSplit newSplit = new RangeInputSplit();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ newSplit.readFields(dis);
+
+ assertEquals(split.getTableName(), newSplit.getTableName());
+ assertEquals(split.getTableId(), newSplit.getTableId());
+ assertEquals(split.getRange(), newSplit.getRange());
+ assertTrue(Arrays.equals(split.getLocations(), newSplit.getLocations()));
+ }
+
+ @Test
+ public void testAllFieldsWritable() throws IOException {
+ RangeInputSplit split = new RangeInputSplit("table", "1", new Range(new Key("a"), new Key("b")),
+ new String[] {"localhost"});
+
+ Set<Pair<Text,Text>> fetchedColumns = new HashSet<>();
+
+ fetchedColumns.add(new Pair<>(new Text("colf1"), new Text("colq1")));
+ fetchedColumns.add(new Pair<>(new Text("colf2"), new Text("colq2")));
+
+ // Fake some iterators
+ ArrayList<IteratorSetting> iterators = new ArrayList<>();
+ IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
+ setting.addOption("foo", "bar");
+ iterators.add(setting);
+
+ setting = new IteratorSetting(100, WholeRowIterator.class);
+ setting.addOption("bar", "foo");
+ iterators.add(setting);
+
+ split.setTableName("table");
+ split.setOffline(true);
+ split.setIsolatedScan(true);
+ split.setUsesLocalIterators(true);
+ split.setFetchedColumns(fetchedColumns);
+ split.setIterators(iterators);
+ split.setLogLevel(Level.WARN);
+ split.setExecutionHints(ImmutableMap.of("priority", "9"));
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ split.write(dos);
+
+ RangeInputSplit newSplit = new RangeInputSplit();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ newSplit.readFields(dis);
+
+ assertEquals(split.getRange(), newSplit.getRange());
+ assertArrayEquals(split.getLocations(), newSplit.getLocations());
+
+ assertEquals(split.getTableName(), newSplit.getTableName());
+ assertEquals(split.isOffline(), newSplit.isOffline());
+ assertEquals(split.isIsolatedScan(), newSplit.isOffline());
+ assertEquals(split.usesLocalIterators(), newSplit.usesLocalIterators());
+ assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
+ assertEquals(split.getIterators(), newSplit.getIterators());
+ assertEquals(split.getLogLevel(), newSplit.getLogLevel());
+ assertEquals(split.getExecutionHints(), newSplit.getExecutionHints());
+ }
+
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/RangePartitionerTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/RangePartitionerTest.java
new file mode 100644
index 0000000000..95451f3a4f
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/lib/partition/RangePartitionerTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoop.mapreduce.lib.partition;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class RangePartitionerTest {
+
+ private static Text[] cutArray = {new Text("A"), new Text("B"), new Text("C")};
+
+ @Test
+ public void testNoSubBins() throws IOException {
+ for (int i = -2; i < 2; ++i) {
+ checkExpectedBins(i, new String[] {"A", "B", "C"}, new int[] {0, 1, 2});
+ checkExpectedBins(i, new String[] {"C", "A", "B"}, new int[] {2, 0, 1});
+ checkExpectedBins(i, new String[] {"", "AA", "BB", "CC"}, new int[] {0, 1, 2, 3});
+ }
+ }
+
+ @Test
+ public void testSubBins() throws IOException {
+ checkExpectedRangeBins(2, new String[] {"A", "B", "C"}, new int[] {1, 3, 5});
+ checkExpectedRangeBins(2, new String[] {"C", "A", "B"}, new int[] {5, 1, 3});
+ checkExpectedRangeBins(2, new String[] {"", "AA", "BB", "CC"}, new int[] {1, 3, 5, 7});
+
+ checkExpectedRangeBins(3, new String[] {"A", "B", "C"}, new int[] {2, 5, 8});
+ checkExpectedRangeBins(3, new String[] {"C", "A", "B"}, new int[] {8, 2, 5});
+ checkExpectedRangeBins(3, new String[] {"", "AA", "BB", "CC"}, new int[] {2, 5, 8, 11});
+
+ checkExpectedRangeBins(10, new String[] {"A", "B", "C"}, new int[] {9, 19, 29});
+ checkExpectedRangeBins(10, new String[] {"C", "A", "B"}, new int[] {29, 9, 19});
+ checkExpectedRangeBins(10, new String[] {"", "AA", "BB", "CC"}, new int[] {9, 19, 29, 39});
+ }
+
+ private RangePartitioner prepPartitioner(int numSubBins) throws IOException {
+ Job job = Job.getInstance();
+ RangePartitioner.setNumSubBins(job, numSubBins);
+ RangePartitioner rp = new RangePartitioner();
+ rp.setConf(job.getConfiguration());
+ return rp;
+ }
+
+ private void checkExpectedRangeBins(int numSubBins, String[] strings, int[] rangeEnds)
+ throws IOException {
+ assertEquals(strings.length, rangeEnds.length);
+ for (int i = 0; i < strings.length; ++i) {
+ int endRange = rangeEnds[i];
+ int startRange = endRange + 1 - numSubBins;
+ int part = prepPartitioner(numSubBins).findPartition(new Text(strings[i]), cutArray,
+ numSubBins);
+ assertTrue(part >= startRange);
+ assertTrue(part <= endRange);
+ }
+ }
+
+ private void checkExpectedBins(int numSubBins, String[] strings, int[] bins) throws IOException {
+ assertEquals(strings.length, bins.length);
+ for (int i = 0; i < strings.length; ++i) {
+ int bin = bins[i], part = prepPartitioner(numSubBins).findPartition(new Text(strings[i]),
+ cutArray, numSubBins);
+ assertEquals(bin, part);
+ }
+ }
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoopImpl/mapreduce/BatchInputSplitTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoopImpl/mapreduce/BatchInputSplitTest.java
new file mode 100644
index 0000000000..27509aa32c
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoopImpl/mapreduce/BatchInputSplitTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.junit.Test;
+
+public class BatchInputSplitTest {
+
+ @Test
+ public void testSimpleWritable() throws IOException {
+ List<Range> ranges = Collections.singletonList(new Range(new Key("a"), new Key("b")));
+ BatchInputSplit split = new BatchInputSplit("table", Table.ID.of("1"), ranges,
+ new String[] {"localhost"});
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ split.write(dos);
+
+ BatchInputSplit newSplit = new BatchInputSplit();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ newSplit.readFields(dis);
+
+ assertEquals(split.getTableName(), newSplit.getTableName());
+ assertEquals(split.getTableId(), newSplit.getTableId());
+ assertEquals(split.getRanges(), newSplit.getRanges());
+ assertTrue(Arrays.equals(split.getLocations(), newSplit.getLocations()));
+ }
+
+ @Test
+ public void testAllFieldsWritable() throws IOException {
+ List<Range> ranges = Collections.singletonList(new Range(new Key("a"), new Key("b")));
+ BatchInputSplit split = new BatchInputSplit("table", Table.ID.of("1"), ranges,
+ new String[] {"localhost"});
+
+ Set<Pair<Text,Text>> fetchedColumns = new HashSet<>();
+
+ fetchedColumns.add(new Pair<>(new Text("colf1"), new Text("colq1")));
+ fetchedColumns.add(new Pair<>(new Text("colf2"), new Text("colq2")));
+
+ // Fake some iterators
+ ArrayList<IteratorSetting> iterators = new ArrayList<>();
+ IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
+ setting.addOption("foo", "bar");
+ iterators.add(setting);
+
+ setting = new IteratorSetting(100, WholeRowIterator.class);
+ setting.addOption("bar", "foo");
+ iterators.add(setting);
+
+ split.setTableName("table");
+ split.setFetchedColumns(fetchedColumns);
+ split.setIterators(iterators);
+ split.setLogLevel(Level.WARN);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ split.write(dos);
+
+ BatchInputSplit newSplit = new BatchInputSplit();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ newSplit.readFields(dis);
+
+ assertEquals(split.getRanges(), newSplit.getRanges());
+ assertArrayEquals(split.getLocations(), newSplit.getLocations());
+
+ assertEquals(split.getTableName(), newSplit.getTableName());
+ assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
+ assertEquals(split.getIterators(), newSplit.getIterators());
+ assertEquals(split.getLogLevel(), newSplit.getLogLevel());
+ }
+}
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/ConfiguratorBaseTest.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/ConfiguratorBaseTest.java
new file mode 100644
index 0000000000..3ccae56744
--- /dev/null
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/ConfiguratorBaseTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.hadoopImpl.mapreduce.lib;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientInfo;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.ClientProperty;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+
+public class ConfiguratorBaseTest {
+
+ private enum PrivateTestingEnum {
+ SOMETHING, SOMETHING_ELSE
+ }
+
+ @Test
+ public void testEnumToConfKey() {
+ assertEquals(this.getClass().getSimpleName() + ".PrivateTestingEnum.Something",
+ ConfiguratorBase.enumToConfKey(this.getClass(), PrivateTestingEnum.SOMETHING));
+ assertEquals(this.getClass().getSimpleName() + ".PrivateTestingEnum.SomethingElse",
+ ConfiguratorBase.enumToConfKey(this.getClass(), PrivateTestingEnum.SOMETHING_ELSE));
+ }
+
+ @Test
+ public void testSetConnectorInfoClassOfQConfigurationStringAuthenticationToken() {
+ Configuration conf = new Configuration();
+ assertFalse(ConfiguratorBase.isConnectorInfoSet(this.getClass(), conf));
+ ConfiguratorBase.setConnectorInfo(this.getClass(), conf, "testUser",
+ new PasswordToken("testPassword"));
+ assertTrue(ConfiguratorBase.isConnectorInfoSet(this.getClass(), conf));
+ assertEquals("testUser", ConfiguratorBase.getPrincipal(this.getClass(), conf));
+ AuthenticationToken token = ConfiguratorBase.getAuthenticationToken(this.getClass(), conf);
+ assertNotNull(token);
+ assertEquals(PasswordToken.class, token.getClass());
+ assertEquals(new PasswordToken("testPassword"), token);
+ }
+
+ @Test
+ public void testSetConnectorInfoClassOfQConfigurationStringString()
+ throws AccumuloSecurityException {
+ Configuration conf = new Configuration();
+ assertFalse(ConfiguratorBase.isConnectorInfoSet(this.getClass(), conf));
+ ConfiguratorBase.setConnectorInfo(this.getClass(), conf, "testUser",
+ new PasswordToken("testPass"));
+ assertTrue(ConfiguratorBase.isConnectorInfoSet(this.getClass(), conf));
+ assertEquals("testUser", ConfiguratorBase.getPrincipal(this.getClass(), conf));
+ assertEquals("testPass", new String(((PasswordToken) ConfiguratorBase
+ .getClientInfo(this.getClass(), conf).getAuthenticationToken()).getPassword()));
+ }
+
+ @Test
+ public void testSetClientInfo() {
+ Configuration conf = new Configuration();
+ ClientInfo info = Accumulo.newClient().forInstance("myinstance", "myzookeepers")
+ .usingPassword("user", "pass").info();
+ ConfiguratorBase.setClientInfo(this.getClass(), conf, info);
+ ClientInfo info2 = ConfiguratorBase.getClientInfo(this.getClass(), conf);
+ assertEquals("myinstance", info2.getInstanceName());
+ assertEquals("myzookeepers", info2.getZooKeepers());
+ assertEquals("user", info2.getPrincipal());
+ assertTrue(info2.getAuthenticationToken() instanceof PasswordToken);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testSetZooKeeperInstance() {
+ Configuration conf = new Configuration();
+ ConfiguratorBase.setZooKeeperInstance(this.getClass(), conf,
+ org.apache.accumulo.core.client.ClientConfiguration.create()
+ .withInstance("testInstanceName").withZkHosts("testZooKeepers").withSsl(true)
+ .withZkTimeout(15000));
+
+ org.apache.accumulo.core.client.ClientConfiguration clientConf = ConfiguratorBase
+ .getClientConfiguration(this.getClass(), conf);
+ assertEquals("testInstanceName", clientConf
+ .get(org.apache.accumulo.core.client.ClientConfiguration.ClientProperty.INSTANCE_NAME));
+
+ Properties props = ConfiguratorBase.getClientInfo(this.getClass(), conf).getProperties();
+ assertEquals("testInstanceName", props.getProperty(ClientProperty.INSTANCE_NAME.getKey()));
+ assertEquals("testZooKeepers", props.getProperty(ClientProperty.INSTANCE_ZOOKEEPERS.getKey()));
+ assertEquals("true", props.getProperty(ClientProperty.SSL_ENABLED.getKey()));
+ assertEquals("15000", props.getProperty(ClientProperty.INSTANCE_ZOOKEEPERS_TIMEOUT.getKey()));
+ }
+
+ @Test
+ public void testSetLogLevel() {
+ Configuration conf = new Configuration();
+ Level currentLevel = Logger.getLogger(this.getClass()).getLevel();
+
+ ConfiguratorBase.setLogLevel(this.getClass(), conf, Level.DEBUG);
+ Logger.getLogger(this.getClass()).setLevel(currentLevel);
+ assertEquals(Level.DEBUG, ConfiguratorBase.getLogLevel(this.getClass(), conf));
+
+ ConfiguratorBase.setLogLevel(this.getClass(), conf, Level.INFO);
+ Logger.getLogger(this.getClass()).setLevel(currentLevel);
+ assertEquals(Level.INFO, ConfiguratorBase.getLogLevel(this.getClass(), conf));
+
+ ConfiguratorBase.setLogLevel(this.getClass(), conf, Level.FATAL);
+ Logger.getLogger(this.getClass()).setLevel(currentLevel);
+ assertEquals(Level.FATAL, ConfiguratorBase.getLogLevel(this.getClass(), conf));
+ }
+
+ @Test
+ public void testSetVisibilityCacheSize() {
+ Configuration conf = new Configuration();
+ assertEquals(Constants.DEFAULT_VISIBILITY_CACHE_SIZE,
+ ConfiguratorBase.getVisibilityCacheSize(conf));
+ ConfiguratorBase.setVisibilityCacheSize(conf, 2000);
+ assertEquals(2000, ConfiguratorBase.getVisibilityCacheSize(conf));
+ }
+}
diff --git a/pom.xml b/pom.xml
index 8f74e52e5e..49cce61692 100644
--- a/pom.xml
+++ b/pom.xml
@@ -78,8 +78,8 @@
</mailingLists>
<modules>
<module>assemble</module>
- <module>client/mapreduce</module>
<module>core</module>
+ <module>hadoop-mapreduce</module>
<module>iterator-test-harness</module>
<module>minicluster</module>
<module>proxy</module>
@@ -311,17 +311,17 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-client-mapreduce</artifactId>
+ <artifactId>accumulo-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
+ <artifactId>accumulo-gc</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-gc</artifactId>
+ <artifactId>accumulo-hadoop-mapreduce</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
diff --git a/test/pom.xml b/test/pom.xml
index f06ca97452..3ad8bd43d7 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -78,10 +78,6 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-client-mapreduce</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services