You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/06/09 16:42:51 UTC
[1/6] git commit: ACCUMULO-2803 Agitator: Independent Tablet Server
and Datanode Min/Max Kill Values
Repository: accumulo
Updated Branches:
refs/heads/1.6.1-SNAPSHOT 73e0f9ced -> ae1682d4a
refs/heads/master 0e097ac59 -> b06799a3d
ACCUMULO-2803 Agitator: Independent Tablet Server and Datanode Min/Max Kill Values
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c839e2ff
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c839e2ff
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c839e2ff
Branch: refs/heads/master
Commit: c839e2ff7491685be07c3ffff461f9b5ead347ba
Parents: affb974
Author: Drew Farris <dr...@apache.org>
Authored: Fri Jun 6 18:17:13 2014 -0700
Committer: Drew Farris <dr...@apache.org>
Committed: Fri Jun 6 18:17:13 2014 -0700
----------------------------------------------------------------------
.../system/continuous/continuous-env.sh.example | 30 ++++++++++++++------
test/system/continuous/start-agitator.sh | 12 ++++----
2 files changed, 27 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c839e2ff/test/system/continuous/continuous-env.sh.example
----------------------------------------------------------------------
diff --git a/test/system/continuous/continuous-env.sh.example b/test/system/continuous/continuous-env.sh.example
index bf4e99e..d3fdd62 100644
--- a/test/system/continuous/continuous-env.sh.example
+++ b/test/system/continuous/continuous-env.sh.example
@@ -74,18 +74,30 @@ SLEEP_TIME=10
#set auths.
AUTHS=''
-#time amount of time (in minutes) the agitator should sleep before killing
-KILL_SLEEP_TIME=20
-
-#time amount of time (in minutes) the agitator should sleep after killing before running tup
-TUP_SLEEP_TIME=10
-
#determines if checksum are generated, may want to turn of when performance testing
CHECKSUM=true
-#the minimum and maximum server the agitator will kill at once
-MIN_KILL=1
-MAX_KILL=1
+#the amount of time (in minutes) the agitator should sleep before killing tservers
+TSERVER_KILL_SLEEP_TIME=20
+
+#the amount of time (in minutes) the agitator should sleep after killing
+# before restarting tservers
+TSERVER_RESTART_SLEEP_TIME=10
+
+#the minimum and maximum number of tservers the agitator will kill at once
+TSERVER_MIN_KILL=1
+TSERVER_MAX_KILL=1
+
+#the amount of time (in minutes) the agitator should sleep before killing datanodes
+DATANODE_KILL_SLEEP_TIME=20
+
+#the amount of time (in minutes) the agitator should sleep after killing
+# before restarting datanodes
+DATANODE_RESTART_SLEEP_TIME=10
+
+#the minimum and maximum number of datanodes the agitator will kill at once
+DATANODE_MIN_KILL=1
+DATANODE_MAX_KILL=1
#time in minutes between killing masters
MASTER_KILL_SLEEP_TIME=60
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c839e2ff/test/system/continuous/start-agitator.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/start-agitator.sh b/test/system/continuous/start-agitator.sh
index 70d5c53..e82e5ab 100755
--- a/test/system/continuous/start-agitator.sh
+++ b/test/system/continuous/start-agitator.sh
@@ -40,18 +40,18 @@ if [[ "`whoami`" == "root" ]]; then
# Change to the correct user if started as root
su -c "nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >${LOG_BASE}_master-agitator.out 2>${LOG_BASE}_master-agitator.err &" -m - $ACCUMULO_USER
- su -c "nohup ${bin}/tserver-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $MIN_KILL $MAX_KILL >${LOG_BASE}_tserver-agitator.out 2>${LOG_BASE}_tserver-agitator.err &" -m - $ACCUMULO_USER
+ su -c "nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME $TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL >${LOG_BASE}_tserver-agitator.out 2>${LOG_BASE}_tserver-agitator.err &" -m - $ACCUMULO_USER
- su -c "nohup ${bin}/datanode-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $HADOOP_PREFIX $MIN_KILL $MAX_KILL >${LOG_BASE}_datanode-agitator.out 2>${LOG_BASE}_datanode-agitator.err &" -m - $HDFS_USER
+ su -c "nohup ${bin}/datanode-agitator.pl $DATANODE_KILL_SLEEP_TIME $DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $DATANODE_MIN_KILL $DATANODE_MAX_KILL >${LOG_BASE}_datanode-agitator.out 2>${LOG_BASE}_datanode-agitator.err &" -m - $HDFS_USER
elif [[ "`whoami`" == $ACCUMULO_USER ]]; then
echo "Running master-agitator and tserver-agitator as `whoami`. Running datanode-agitator as $HDFS_USER using sudo."
# Just run the master-agitator if we're the accumulo user
nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >${LOG_BASE}_master-agitator.out 2>${LOG_BASE}_master-agitator.err &
- nohup ${bin}/tserver-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $MIN_KILL $MAX_KILL >${LOG_BASE}_tserver-agitator.out 2>${LOG_BASE}_tserver-agitator.err &
+ nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME $TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL >${LOG_BASE}_tserver-agitator.out 2>${LOG_BASE}_tserver-agitator.err &
- sudo -u $HDFS_USER nohup ${bin}/datanode-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $HADOOP_PREFIX $MIN_KILL $MAX_KILL >${LOG_BASE}_datanode-agitator.out 2>${LOG_BASE}_datanode-agitator.err &
+ sudo -u $HDFS_USER nohup ${bin}/datanode-agitator.pl $DATANODE_KILL_SLEEP_TIME $DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $DATANODE_MIN_KILL $DATANODE_MAX_KILL >${LOG_BASE}_datanode-agitator.out 2>${LOG_BASE}_datanode-agitator.err &
else
echo "Running master-agitator and tserver-agitator as $ACCUMULO_USER using sudo. Running datanode-agitator as $HDFS_USER using sudo."
@@ -59,9 +59,9 @@ else
# Not root, and not the accumulo user, hope you can sudo to it
sudo -u $ACCUMULO_USER "nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >${LOG_BASE}_master-agitator.out 2>${LOG_BASE}_master-agitator.err &"
- sudo -u $ACCUMULO_USER "nohup ${bin}/tserver-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $MIN_KILL $MAX_KILL >${LOG_BASE}_tserver-agitator.out 2>${LOG_BASE}_tserver-agitator.err &"
+ sudo -u $ACCUMULO_USER "nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME $TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL >${LOG_BASE}_tserver-agitator.out 2>${LOG_BASE}_tserver-agitator.err &"
- sudo -u $HDFS_USER "nohup ${bin}/datanode-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $HADOOP_PREFIX $MIN_KILL $MAX_KILL >${LOG_BASE}_datanode-agitator.out 2>${LOG_BASE}_datanode-agitator.err &" -m - $HDFS_USER
+ sudo -u $HDFS_USER "nohup ${bin}/datanode-agitator.pl $DATANODE_KILL_SLEEP_TIME $DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $DATANODE_MIN_KILL $DATANODE_MAX_KILL >${LOG_BASE}_datanode-agitator.out 2>${LOG_BASE}_datanode-agitator.err &" -m - $HDFS_USER
fi
[4/6] git commit: ACCUMULO-1919 Size of visibility cache in
AccumuloFileOutputFormat should be configurable.
Posted by el...@apache.org.
ACCUMULO-1919 Size of visibility cache in AccumuloFileOutputFormat should be configurable.
Signed-off-by: Josh Elser <el...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ae1682d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ae1682d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ae1682d4
Branch: refs/heads/master
Commit: ae1682d4ae7a3031598748f676c90829d0b2a978
Parents: 73e0f9c
Author: dallaybatta <vi...@gmail.com>
Authored: Mon Jun 9 11:29:54 2014 +0530
Committer: Josh Elser <el...@apache.org>
Committed: Mon Jun 9 10:27:51 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 2 ++
.../client/mapred/AccumuloFileOutputFormat.java | 3 +-
.../mapreduce/lib/impl/ConfiguratorBase.java | 37 +++++++++++++++++++-
.../mapred/AccumuloFileOutputFormatTest.java | 3 ++
.../lib/impl/ConfiguratorBaseTest.java | 8 +++++
5 files changed, 51 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java
index e0e88eb..c9a51a9 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -115,4 +115,6 @@ public class Constants {
public static final String[] PATH_PROPERTY_ENV_VARS = new String[] {"ACCUMULO_HOME", "ACCUMULO_CONF_DIR"};
public static final String HDFS_TABLES_DIR = "/tables";
+
+ public static final int DEFAULT_VISIBILITY_CACHE_SIZE = 1000;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
index 8a1d6df..cfaaa58 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.mapred;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
import org.apache.accumulo.core.client.mapreduce.lib.impl.FileOutputConfigurator;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -145,7 +146,7 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension);
- final LRUMap validVisibilities = new LRUMap(1000);
+ final LRUMap validVisibilities = new LRUMap(ConfiguratorBase.getVisibilityCacheSize(conf));
return new RecordWriter<Key,Value>() {
FileSKVWriter out = null;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index e87d43b..cf1f0ea 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@ -82,7 +82,8 @@ public class ConfiguratorBase {
* @since 1.6.0
*/
public static enum GeneralOpts {
- LOG_LEVEL
+ LOG_LEVEL,
+ VISIBILITY_CACHE_SIZE
}
/**
@@ -100,6 +101,17 @@ public class ConfiguratorBase {
}
/**
+ * Provides a configuration key for a given feature enum.
+ *
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ */
+ protected static String enumToConfKey(Enum<?> e) {
+ return e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
* Sets the connector information needed to communicate with Accumulo in this job.
*
* <p>
@@ -369,4 +381,27 @@ public class ConfiguratorBase {
return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
}
+ /**
+ * Sets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param visibilityCacheSize
+ * the LRU cache size
+ */
+ public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
+ conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
+ }
+
+ /**
+ * Gets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the valid visibility count
+ */
+ public static int getVisibilityCacheSize(Configuration conf) {
+ return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
index aad544b..3a4d641 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
@@ -28,6 +28,7 @@ import java.io.IOException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -53,6 +54,7 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
public class AccumuloFileOutputFormatTest {
+ private static final int JOB_VISIBILITY_CACHE_SIZE = 3000;
private static final String PREFIX = AccumuloFileOutputFormatTest.class.getSimpleName();
private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
private static final String BAD_TABLE = PREFIX + "_mapred_bad_table";
@@ -145,6 +147,7 @@ public class AccumuloFileOutputFormatTest {
JobConf job = new JobConf(getConf());
job.setJarByClass(this.getClass());
+ ConfiguratorBase.setVisibilityCacheSize(job, JOB_VISIBILITY_CACHE_SIZE);
job.setInputFormat(AccumuloInputFormat.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
index d5ebb22..7c1f98b 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
@@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.ClientConfiguration;
import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
@@ -126,4 +127,11 @@ public class ConfiguratorBaseTest {
assertEquals(Level.FATAL, ConfiguratorBase.getLogLevel(this.getClass(), conf));
}
+ @Test
+ public void testSetVisibiltyCacheSize(){
+ Configuration conf = new Configuration();
+ assertEquals(Constants.DEFAULT_VISIBILITY_CACHE_SIZE,ConfiguratorBase.getVisibilityCacheSize(conf));
+ ConfiguratorBase.setVisibilityCacheSize(conf, 2000);
+ assertEquals(2000,ConfiguratorBase.getVisibilityCacheSize(conf));
+ }
}
[6/6] git commit: Merge branch '1.6.1-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.1-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b06799a3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b06799a3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b06799a3
Branch: refs/heads/master
Commit: b06799a3d11ab23b5187b2d3bdc69cb8b37a5da2
Parents: 0e097ac ae1682d
Author: Josh Elser <el...@apache.org>
Authored: Mon Jun 9 10:42:14 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Jun 9 10:42:14 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 2 ++
.../client/mapred/AccumuloFileOutputFormat.java | 3 +-
.../mapreduce/lib/impl/ConfiguratorBase.java | 37 +++++++++++++++++++-
.../mapred/AccumuloFileOutputFormatTest.java | 3 ++
.../lib/impl/ConfiguratorBaseTest.java | 8 +++++
.../system/continuous/continuous-env.sh.example | 30 +++++++++++-----
test/system/continuous/start-agitator.sh | 12 +++----
7 files changed, 78 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b06799a3/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b06799a3/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
index 8a1d6df,0000000..cfaaa58
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
@@@ -1,178 -1,0 +1,179 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import java.io.IOException;
+import java.util.Arrays;
+
++import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.FileOutputConfigurator;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Progressable;
+import org.apache.log4j.Logger;
+
+/**
+ * This class allows MapReduce jobs to write output in the Accumulo data file format.<br />
+ * Care should be taken to write only sorted data (sorted by {@link Key}), as this is an important requirement of Accumulo data files.
+ *
+ * <p>
+ * The output path to be created must be specified via {@link AccumuloFileOutputFormat#setOutputPath(JobConf, Path)}. This is inherited from
+ * {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Other methods from {@link FileOutputFormat} are not supported and may be ignored or cause failures.
+ * Using other Hadoop configuration options that affect the behavior of the underlying files directly in the Job's configuration may work, but are not directly
+ * supported at this time.
+ */
+public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
+
+ private static final Class<?> CLASS = AccumuloFileOutputFormat.class;
+ protected static final Logger log = Logger.getLogger(CLASS);
+
+ /**
+ * This helper method provides an AccumuloConfiguration object constructed from the Accumulo defaults, and overridden with Accumulo properties that have been
+ * stored in the Job's configuration.
+ *
+ * @param job
+ * the Hadoop context for the configured job
+ * @since 1.5.0
+ */
+ protected static AccumuloConfiguration getAccumuloConfiguration(JobConf job) {
+ return FileOutputConfigurator.getAccumuloConfiguration(CLASS, job);
+ }
+
+ /**
+ * Sets the compression type to use for data blocks. Specifying a compression may require additional libraries to be available to your Job.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param compressionType
+ * one of "none", "gz", "lzo", or "snappy"
+ * @since 1.5.0
+ */
+ public static void setCompressionType(JobConf job, String compressionType) {
+ FileOutputConfigurator.setCompressionType(CLASS, job, compressionType);
+ }
+
+ /**
+ * Sets the size for data blocks within each file.<br />
+ * Data blocks are a span of key/value pairs stored in the file that are compressed and indexed as a group.
+ *
+ * <p>
+ * Making this value smaller may increase seek performance, but at the cost of increasing the size of the indexes (which can also affect seek performance).
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param dataBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setDataBlockSize(JobConf job, long dataBlockSize) {
+ FileOutputConfigurator.setDataBlockSize(CLASS, job, dataBlockSize);
+ }
+
+ /**
+ * Sets the size for file blocks in the file system; file blocks are managed, and replicated, by the underlying file system.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param fileBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setFileBlockSize(JobConf job, long fileBlockSize) {
+ FileOutputConfigurator.setFileBlockSize(CLASS, job, fileBlockSize);
+ }
+
+ /**
+ * Sets the size for index blocks within each file; smaller blocks means a deeper index hierarchy within the file, while larger blocks mean a more shallow
+ * index hierarchy within the file. This can affect the performance of queries.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param indexBlockSize
+ * the block size, in bytes
+ * @since 1.5.0
+ */
+ public static void setIndexBlockSize(JobConf job, long indexBlockSize) {
+ FileOutputConfigurator.setIndexBlockSize(CLASS, job, indexBlockSize);
+ }
+
+ /**
+ * Sets the file system replication factor for the resulting file, overriding the file system default.
+ *
+ * @param job
+ * the Hadoop job instance to be configured
+ * @param replication
+ * the number of replicas for produced files
+ * @since 1.5.0
+ */
+ public static void setReplication(JobConf job, int replication) {
+ FileOutputConfigurator.setReplication(CLASS, job, replication);
+ }
+
+ @Override
+ public RecordWriter<Key,Value> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
+ // get the path of the temporary output file
+ final Configuration conf = job;
+ final AccumuloConfiguration acuConf = getAccumuloConfiguration(job);
+
+ final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
+ final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension);
+
- final LRUMap validVisibilities = new LRUMap(1000);
++ final LRUMap validVisibilities = new LRUMap(ConfiguratorBase.getVisibilityCacheSize(conf));
+
+ return new RecordWriter<Key,Value>() {
+ FileSKVWriter out = null;
+
+ @Override
+ public void close(Reporter reporter) throws IOException {
+ if (out != null)
+ out.close();
+ }
+
+ @Override
+ public void write(Key key, Value value) throws IOException {
+
+ Boolean wasChecked = (Boolean) validVisibilities.get(key.getColumnVisibilityData());
+ if (wasChecked == null) {
+ byte[] cv = key.getColumnVisibilityData().toArray();
+ new ColumnVisibility(cv);
+ validVisibilities.put(new ArrayByteSequence(Arrays.copyOf(cv, cv.length)), Boolean.TRUE);
+ }
+
+ if (out == null) {
+ out = FileOperations.getInstance().openWriter(file.toString(), file.getFileSystem(conf), conf, acuConf);
+ out.startDefaultLocalityGroup();
+ }
+ out.append(key, value);
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b06799a3/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --cc mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index cf131bd,0000000..bd7c6bf
mode 100644,000000..100644
--- a/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@@ -1,369 -1,0 +1,404 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * @since 1.6.0
+ */
+public class ConfiguratorBase {
+
+ /**
+ * Configuration keys for {@link Instance#getConnector(String, AuthenticationToken)}.
+ *
+ * @since 1.6.0
+ */
+ public static enum ConnectorInfo {
+ IS_CONFIGURED, PRINCIPAL, TOKEN,
+ }
+
+ public static enum TokenSource {
+ FILE, INLINE;
+
+ private String prefix;
+
+ private TokenSource() {
+ prefix = name().toLowerCase() + ":";
+ }
+
+ public String prefix() {
+ return prefix;
+ }
+ }
+
+ /**
+ * Configuration keys for {@link Instance}, {@link ZooKeeperInstance}, and {@link MockInstance}.
+ *
+ * @since 1.6.0
+ */
+ public static enum InstanceOpts {
+ TYPE, NAME, ZOO_KEEPERS, CLIENT_CONFIG;
+ }
+
+ /**
+ * Configuration keys for general configuration options.
+ *
+ * @since 1.6.0
+ */
+ public static enum GeneralOpts {
- LOG_LEVEL
++ LOG_LEVEL,
++ VISIBILITY_CACHE_SIZE
+ }
+
+ /**
+ * Provides a configuration key for a given feature enum, prefixed by the implementingClass
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ * @since 1.6.0
+ */
+ protected static String enumToConfKey(Class<?> implementingClass, Enum<?> e) {
+ return implementingClass.getSimpleName() + "." + e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
++ * Provides a configuration key for a given feature enum.
++ *
++ * @param e
++ * the enum used to provide the unique part of the configuration key
++ * @return the configuration key
++ */
++ protected static String enumToConfKey(Enum<?> e) {
++ return e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
++ }
++
++ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * <b>WARNING:</b> The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe
+ * conversion to a string, and is not intended to be secure.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param token
+ * the user's password
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, AuthenticationToken token)
+ throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ checkArgument(principal != null, "principal is null");
+ checkArgument(token != null, "token is null");
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
+ TokenSource.INLINE.prefix() + token.getClass().getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(token)));
+ }
+
+ /**
+ * Sets the connector information needed to communicate with Accumulo in this job.
+ *
+ * <p>
+ * Pulls a token file into the Distributed Cache that contains the authentication token in an attempt to be more secure than storing the password in the
+ * Configuration. Token file created with "bin/accumulo create-token".
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param principal
+ * a valid Accumulo user name
+ * @param tokenFile
+ * the path to the token file in DFS
+ * @since 1.6.0
+ */
+ public static void setConnectorInfo(Class<?> implementingClass, Configuration conf, String principal, String tokenFile) throws AccumuloSecurityException {
+ if (isConnectorInfoSet(implementingClass, conf))
+ throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName() + " can only be set once per job");
+
+ checkArgument(principal != null, "principal is null");
+ checkArgument(tokenFile != null, "tokenFile is null");
+
+ try {
+ DistributedCacheHelper.addCacheFile(new URI(tokenFile), conf);
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed cache.");
+ }
+
+ conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+ conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN), TokenSource.FILE.prefix() + tokenFile);
+ }
+
+ /**
+ * Determines if the connector info has already been set for this instance.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return true if the connector info has already been set, false otherwise
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static Boolean isConnectorInfoSet(Class<?> implementingClass, Configuration conf) {
+ return conf.getBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), false);
+ }
+
+ /**
+ * Gets the user name from the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static String getPrincipal(Class<?> implementingClass, Configuration conf) {
+ return conf.get(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL));
+ }
+
+ /**
+ * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the principal's authentication token
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ * @see #setConnectorInfo(Class, Configuration, String, String)
+ */
+ public static AuthenticationToken getAuthenticationToken(Class<?> implementingClass, Configuration conf) {
+ String token = conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN));
+ if (token == null || token.isEmpty())
+ return null;
+ if (token.startsWith(TokenSource.INLINE.prefix())) {
+ String[] args = token.substring(TokenSource.INLINE.prefix().length()).split(":", 2);
+ if (args.length == 2)
+ return AuthenticationTokenSerializer.deserialize(args[0], Base64.decodeBase64(args[1].getBytes(StandardCharsets.UTF_8)));
+ } else if (token.startsWith(TokenSource.FILE.prefix())) {
+ String tokenFileName = token.substring(TokenSource.FILE.prefix().length());
+ return getTokenFromFile(conf, getPrincipal(implementingClass, conf), tokenFileName);
+ }
+
+ throw new IllegalStateException("Token was not properly serialized into the configuration");
+ }
+
+ /**
+ * Reads from the token file in distributed cache. Currently, the token file stores data separated by colons e.g. principal:token_class:token
+ *
+ * @param conf
+ * the Hadoop context for the configured job
+ * @return path to the token file as a String
+ * @since 1.6.0
+ * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
+ */
+ public static AuthenticationToken getTokenFromFile(Configuration conf, String principal, String tokenFile) {
+ FSDataInputStream in = null;
+ try {
+ URI[] uris = DistributedCacheHelper.getCacheFiles(conf);
+ Path path = null;
+ for (URI u : uris) {
+ if (u.toString().equals(tokenFile)) {
+ path = new Path(u);
+ }
+ }
+ if (path == null) {
+ throw new IllegalArgumentException("Couldn't find password file called \"" + tokenFile + "\" in cache.");
+ }
+ FileSystem fs = FileSystem.get(conf);
+ in = fs.open(path);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Couldn't open password file called \"" + tokenFile + "\".");
+ }
+ try (java.util.Scanner fileScanner = new java.util.Scanner(in)) {
+ while (fileScanner.hasNextLine()) {
+ Credentials creds = Credentials.deserialize(fileScanner.nextLine());
+ if (principal.equals(creds.getPrincipal())) {
+ return creds.getToken();
+ }
+ }
+ throw new IllegalArgumentException("Couldn't find token for user \"" + principal + "\" in file \"" + tokenFile + "\"");
+ }
+ }
+
+ /**
+ * Configures a {@link ZooKeeperInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param clientConfig
+ * client configuration for specifying connection timeouts, SSL connection options, etc.
+ * @since 1.6.0
+ */
+ public static void setZooKeeperInstance(Class<?> implementingClass, Configuration conf, ClientConfiguration clientConfig) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "ZooKeeperInstance");
+ if (clientConfig != null) {
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG), clientConfig.serialize());
+ }
+ }
+
+ /**
+ * Configures a {@link MockInstance} for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param instanceName
+ * the Accumulo instance name
+ * @since 1.6.0
+ */
+ public static void setMockInstance(Class<?> implementingClass, Configuration conf, String instanceName) {
+ String key = enumToConfKey(implementingClass, InstanceOpts.TYPE);
+ if (!conf.get(key, "").isEmpty())
+ throw new IllegalStateException("Instance info can only be set once per job; it has already been configured with " + conf.get(key));
+ conf.set(key, "MockInstance");
+
+ checkArgument(instanceName != null, "instanceName is null");
+ conf.set(enumToConfKey(implementingClass, InstanceOpts.NAME), instanceName);
+ }
+
+ /**
+ * Initializes an Accumulo {@link Instance} based on the configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return an Accumulo instance
+ * @since 1.6.0
+ * @see #setZooKeeperInstance(Class, Configuration, ClientConfiguration)
+ * @see #setMockInstance(Class, Configuration, String)
+ */
+ public static Instance getInstance(Class<?> implementingClass, Configuration conf) {
+ String instanceType = conf.get(enumToConfKey(implementingClass, InstanceOpts.TYPE), "");
+ if ("MockInstance".equals(instanceType))
+ return new MockInstance(conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME)));
+ else if ("ZooKeeperInstance".equals(instanceType)) {
+ String clientConfigString = conf.get(enumToConfKey(implementingClass, InstanceOpts.CLIENT_CONFIG));
+ if (clientConfigString == null) {
+ String instanceName = conf.get(enumToConfKey(implementingClass, InstanceOpts.NAME));
+ String zookeepers = conf.get(enumToConfKey(implementingClass, InstanceOpts.ZOO_KEEPERS));
+ return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
+ } else {
+ return new ZooKeeperInstance(ClientConfiguration.deserialize(clientConfigString));
+ }
+ } else if (instanceType.isEmpty())
+ throw new IllegalStateException("Instance has not been configured for " + implementingClass.getSimpleName());
+ else
+ throw new IllegalStateException("Unrecognized instance type " + instanceType);
+ }
+
+ /**
+ * Sets the log level for this job.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param level
+ * the logging level
+ * @since 1.6.0
+ */
+ public static void setLogLevel(Class<?> implementingClass, Configuration conf, Level level) {
+ checkArgument(level != null, "level is null");
+ Logger.getLogger(implementingClass).setLevel(level);
+ conf.setInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), level.toInt());
+ }
+
+ /**
+ * Gets the log level from this configuration.
+ *
+ * @param implementingClass
+ * the class whose name will be used as a prefix for the property configuration key
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the log level
+ * @since 1.6.0
+ * @see #setLogLevel(Class, Configuration, Level)
+ */
+ public static Level getLogLevel(Class<?> implementingClass, Configuration conf) {
+ return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
+ }
+
++ /**
++ * Sets the valid visibility count for this job.
++ *
++ * @param conf
++ * the Hadoop configuration object to configure
++ * @param visibilityCacheSize
++ * the LRU cache size
++ */
++ public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
++ conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
++ }
++
++ /**
++ * Gets the valid visibility count for this job.
++ *
++ * @param conf
++ * the Hadoop configuration object to configure
++ * @return the valid visibility count
++ */
++ public static int getVisibilityCacheSize(Configuration conf) {
++ return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
++ }
++
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b06799a3/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
----------------------------------------------------------------------
diff --cc mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
index aad544b,0000000..3a4d641
mode 100644,000000..100644
--- a/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
+++ b/mapreduce/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
@@@ -1,247 -1,0 +1,250 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
++import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.IdentityMapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class AccumuloFileOutputFormatTest {
++ private static final int JOB_VISIBILITY_CACHE_SIZE = 3000;
+ private static final String PREFIX = AccumuloFileOutputFormatTest.class.getSimpleName();
+ private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+ private static final String BAD_TABLE = PREFIX + "_mapred_bad_table";
+ private static final String TEST_TABLE = PREFIX + "_mapred_test_table";
+ private static final String EMPTY_TABLE = PREFIX + "_mapred_empty_table";
+
+ private static AssertionError e1 = null;
+ private static AssertionError e2 = null;
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+ Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+ c.tableOperations().create(EMPTY_TABLE);
+ c.tableOperations().create(TEST_TABLE);
+ c.tableOperations().create(BAD_TABLE);
+ BatchWriter bw = c.createBatchWriter(TEST_TABLE, new BatchWriterConfig());
+ Mutation m = new Mutation("Key");
+ m.put("", "", "");
+ bw.addMutation(m);
+ bw.close();
+ bw = c.createBatchWriter(BAD_TABLE, new BatchWriterConfig());
+ m = new Mutation("r1");
+ m.put("cf1", "cq1", "A&B");
+ m.put("cf1", "cq1", "A&B");
+ m.put("cf1", "cq2", "A&");
+ bw.addMutation(m);
+ bw.close();
+ }
+
+ @Test
+ public void testEmptyWrite() throws Exception {
+ handleWriteTests(false);
+ }
+
+ @Test
+ public void testRealWrite() throws Exception {
+ handleWriteTests(true);
+ }
+
+ private static class MRTester extends Configured implements Tool {
+ private static class BadKeyMapper implements Mapper<Key,Value,Key,Value> {
+
+ int index = 0;
+
+ @Override
+ public void map(Key key, Value value, OutputCollector<Key,Value> output, Reporter reporter) throws IOException {
+ try {
+ try {
+ output.collect(key, value);
+ if (index == 2)
+ fail();
+ } catch (Exception e) {
+ Logger.getLogger(this.getClass()).error(e, e);
+ assertEquals(2, index);
+ }
+ } catch (AssertionError e) {
+ e1 = e;
+ }
+ index++;
+ }
+
+ @Override
+ public void configure(JobConf job) {}
+
+ @Override
+ public void close() throws IOException {
+ try {
+ assertEquals(2, index);
+ } catch (AssertionError e) {
+ e2 = e;
+ }
+ }
+
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 4) {
+ throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table> <outputfile>");
+ }
+
+ String user = args[0];
+ String pass = args[1];
+ String table = args[2];
+
+ JobConf job = new JobConf(getConf());
+ job.setJarByClass(this.getClass());
++ ConfiguratorBase.setVisibilityCacheSize(job, JOB_VISIBILITY_CACHE_SIZE);
+
+ job.setInputFormat(AccumuloInputFormat.class);
+
+ AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
+ AccumuloInputFormat.setInputTableName(job, table);
+ AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+ AccumuloFileOutputFormat.setOutputPath(job, new Path(args[3]));
+
+ job.setMapperClass(BAD_TABLE.equals(table) ? BadKeyMapper.class : IdentityMapper.class);
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+ job.setOutputFormat(AccumuloFileOutputFormat.class);
+
+ job.setNumReduceTasks(0);
+
+ return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+ }
+ }
+
+ public void handleWriteTests(boolean content) throws Exception {
+ File f = folder.newFile("handleWriteTests");
+ f.delete();
+ MRTester.main(new String[] {"root", "", content ? TEST_TABLE : EMPTY_TABLE, f.getAbsolutePath()});
+
+ assertTrue(f.exists());
+ File[] files = f.listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File file) {
+ return file.getName().startsWith("part-m-");
+ }
+ });
+ if (content) {
+ assertEquals(1, files.length);
+ assertTrue(files[0].exists());
+ } else {
+ assertEquals(0, files.length);
+ }
+ }
+
+ @Test
+ public void writeBadVisibility() throws Exception {
+ File f = folder.newFile("writeBadVisibility");
+ f.delete();
+ MRTester.main(new String[] {"root", "", BAD_TABLE, f.getAbsolutePath()});
+ Logger.getLogger(this.getClass()).error(e1, e1);
+ assertNull(e1);
+ assertNull(e2);
+ }
+
+ @Test
+ public void validateConfiguration() throws IOException, InterruptedException {
+
+ int a = 7;
+ long b = 300l;
+ long c = 50l;
+ long d = 10l;
+ String e = "snappy";
+
+ JobConf job = new JobConf();
+ AccumuloFileOutputFormat.setReplication(job, a);
+ AccumuloFileOutputFormat.setFileBlockSize(job, b);
+ AccumuloFileOutputFormat.setDataBlockSize(job, c);
+ AccumuloFileOutputFormat.setIndexBlockSize(job, d);
+ AccumuloFileOutputFormat.setCompressionType(job, e);
+
+ AccumuloConfiguration acuconf = AccumuloFileOutputFormat.getAccumuloConfiguration(job);
+
+ assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+ assertEquals(300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
+ assertEquals(50l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+ assertEquals(10l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+ assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+
+ a = 17;
+ b = 1300l;
+ c = 150l;
+ d = 110l;
+ e = "lzo";
+
+ job = new JobConf();
+ AccumuloFileOutputFormat.setReplication(job, a);
+ AccumuloFileOutputFormat.setFileBlockSize(job, b);
+ AccumuloFileOutputFormat.setDataBlockSize(job, c);
+ AccumuloFileOutputFormat.setIndexBlockSize(job, d);
+ AccumuloFileOutputFormat.setCompressionType(job, e);
+
+ acuconf = AccumuloFileOutputFormat.getAccumuloConfiguration(job);
+
+ assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+ assertEquals(1300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
+ assertEquals(150l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+ assertEquals(110l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+ assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b06799a3/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
----------------------------------------------------------------------
diff --cc mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
index d5ebb22,0000000..7c1f98b
mode 100644,000000..100644
--- a/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
+++ b/mapreduce/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
@@@ -1,129 -1,0 +1,137 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce.lib.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
++import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class ConfiguratorBaseTest {
+
+ private static enum PrivateTestingEnum {
+ SOMETHING, SOMETHING_ELSE
+ }
+
+ @Test
+ public void testEnumToConfKey() {
+ assertEquals(this.getClass().getSimpleName() + ".PrivateTestingEnum.Something",
+ ConfiguratorBase.enumToConfKey(this.getClass(), PrivateTestingEnum.SOMETHING));
+ assertEquals(this.getClass().getSimpleName() + ".PrivateTestingEnum.SomethingElse",
+ ConfiguratorBase.enumToConfKey(this.getClass(), PrivateTestingEnum.SOMETHING_ELSE));
+ }
+
+ @Test
+ public void testSetConnectorInfoClassOfQConfigurationStringAuthenticationToken() throws AccumuloSecurityException {
+ Configuration conf = new Configuration();
+ assertFalse(ConfiguratorBase.isConnectorInfoSet(this.getClass(), conf));
+ ConfiguratorBase.setConnectorInfo(this.getClass(), conf, "testUser", new PasswordToken("testPassword"));
+ assertTrue(ConfiguratorBase.isConnectorInfoSet(this.getClass(), conf));
+ assertEquals("testUser", ConfiguratorBase.getPrincipal(this.getClass(), conf));
+ AuthenticationToken token = ConfiguratorBase.getAuthenticationToken(this.getClass(), conf);
+ assertEquals(PasswordToken.class, token.getClass());
+ assertEquals(new PasswordToken("testPassword"), token);
+ assertEquals(
+ "inline:" + PasswordToken.class.getName() + ":" + Base64.encodeBase64String(AuthenticationTokenSerializer.serialize(new PasswordToken("testPassword"))),
+ conf.get(ConfiguratorBase.enumToConfKey(this.getClass(), ConfiguratorBase.ConnectorInfo.TOKEN)));
+ }
+
+ @Test
+ public void testSetConnectorInfoClassOfQConfigurationStringString() throws AccumuloSecurityException {
+ Configuration conf = new Configuration();
+ assertFalse(ConfiguratorBase.isConnectorInfoSet(this.getClass(), conf));
+ ConfiguratorBase.setConnectorInfo(this.getClass(), conf, "testUser", "testFile");
+ assertTrue(ConfiguratorBase.isConnectorInfoSet(this.getClass(), conf));
+ assertEquals("testUser", ConfiguratorBase.getPrincipal(this.getClass(), conf));
+ assertEquals("file:testFile", conf.get(ConfiguratorBase.enumToConfKey(this.getClass(), ConfiguratorBase.ConnectorInfo.TOKEN)));
+ }
+
+ @Test
+ public void testSetZooKeeperInstance() {
+ Configuration conf = new Configuration();
+ ConfiguratorBase.setZooKeeperInstance(this.getClass(), conf, new ClientConfiguration().withInstance("testInstanceName").withZkHosts("testZooKeepers")
+ .withSsl(true).withZkTimeout(1234));
+ ClientConfiguration clientConf = ClientConfiguration.deserialize(conf.get(ConfiguratorBase.enumToConfKey(this.getClass(),
+ ConfiguratorBase.InstanceOpts.CLIENT_CONFIG)));
+ assertEquals("testInstanceName", clientConf.get(ClientProperty.INSTANCE_NAME));
+ assertEquals("testZooKeepers", clientConf.get(ClientProperty.INSTANCE_ZK_HOST));
+ assertEquals("true", clientConf.get(ClientProperty.INSTANCE_RPC_SSL_ENABLED));
+ assertEquals("1234", clientConf.get(ClientProperty.INSTANCE_ZK_TIMEOUT));
+ assertEquals(ZooKeeperInstance.class.getSimpleName(), conf.get(ConfiguratorBase.enumToConfKey(this.getClass(), ConfiguratorBase.InstanceOpts.TYPE)));
+
+ Instance instance = ConfiguratorBase.getInstance(this.getClass(), conf);
+ assertEquals(ZooKeeperInstance.class.getName(), instance.getClass().getName());
+ assertEquals("testInstanceName", ((ZooKeeperInstance) instance).getInstanceName());
+ assertEquals("testZooKeepers", ((ZooKeeperInstance) instance).getZooKeepers());
+ assertEquals(1234000, ((ZooKeeperInstance) instance).getZooKeepersSessionTimeOut());
+ }
+
+ @Test
+ public void testSetMockInstance() {
+ Configuration conf = new Configuration();
+ ConfiguratorBase.setMockInstance(this.getClass(), conf, "testInstanceName");
+ assertEquals("testInstanceName", conf.get(ConfiguratorBase.enumToConfKey(this.getClass(), ConfiguratorBase.InstanceOpts.NAME)));
+ assertEquals(null, conf.get(ConfiguratorBase.enumToConfKey(this.getClass(), ConfiguratorBase.InstanceOpts.ZOO_KEEPERS)));
+ assertEquals(MockInstance.class.getSimpleName(), conf.get(ConfiguratorBase.enumToConfKey(this.getClass(), ConfiguratorBase.InstanceOpts.TYPE)));
+ Instance instance = ConfiguratorBase.getInstance(this.getClass(), conf);
+ assertEquals(MockInstance.class.getName(), instance.getClass().getName());
+ }
+
+ @Test
+ public void testSetLogLevel() {
+ Configuration conf = new Configuration();
+ Level currentLevel = Logger.getLogger(this.getClass()).getLevel();
+
+ ConfiguratorBase.setLogLevel(this.getClass(), conf, Level.DEBUG);
+ Logger.getLogger(this.getClass()).setLevel(currentLevel);
+ assertEquals(Level.DEBUG, ConfiguratorBase.getLogLevel(this.getClass(), conf));
+
+ ConfiguratorBase.setLogLevel(this.getClass(), conf, Level.INFO);
+ Logger.getLogger(this.getClass()).setLevel(currentLevel);
+ assertEquals(Level.INFO, ConfiguratorBase.getLogLevel(this.getClass(), conf));
+
+ ConfiguratorBase.setLogLevel(this.getClass(), conf, Level.FATAL);
+ Logger.getLogger(this.getClass()).setLevel(currentLevel);
+ assertEquals(Level.FATAL, ConfiguratorBase.getLogLevel(this.getClass(), conf));
+ }
+
++ @Test
++ public void testSetVisibiltyCacheSize(){
++ Configuration conf = new Configuration();
++ assertEquals(Constants.DEFAULT_VISIBILITY_CACHE_SIZE,ConfiguratorBase.getVisibilityCacheSize(conf));
++ ConfiguratorBase.setVisibilityCacheSize(conf, 2000);
++ assertEquals(2000,ConfiguratorBase.getVisibilityCacheSize(conf));
++ }
+}
[5/6] git commit: ACCUMULO-1919 Size of visibility cache in
AccumuloFileOutputFormat should be configurable.
Posted by el...@apache.org.
ACCUMULO-1919 Size of visibility cache in AccumuloFileOutputFormat should be configurable.
Signed-off-by: Josh Elser <el...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ae1682d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ae1682d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ae1682d4
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: ae1682d4ae7a3031598748f676c90829d0b2a978
Parents: 73e0f9c
Author: dallaybatta <vi...@gmail.com>
Authored: Mon Jun 9 11:29:54 2014 +0530
Committer: Josh Elser <el...@apache.org>
Committed: Mon Jun 9 10:27:51 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 2 ++
.../client/mapred/AccumuloFileOutputFormat.java | 3 +-
.../mapreduce/lib/impl/ConfiguratorBase.java | 37 +++++++++++++++++++-
.../mapred/AccumuloFileOutputFormatTest.java | 3 ++
.../lib/impl/ConfiguratorBaseTest.java | 8 +++++
5 files changed, 51 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java
index e0e88eb..c9a51a9 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -115,4 +115,6 @@ public class Constants {
public static final String[] PATH_PROPERTY_ENV_VARS = new String[] {"ACCUMULO_HOME", "ACCUMULO_CONF_DIR"};
public static final String HDFS_TABLES_DIR = "/tables";
+
+ public static final int DEFAULT_VISIBILITY_CACHE_SIZE = 1000;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
index 8a1d6df..cfaaa58 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.mapred;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
import org.apache.accumulo.core.client.mapreduce.lib.impl.FileOutputConfigurator;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -145,7 +146,7 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension);
- final LRUMap validVisibilities = new LRUMap(1000);
+ final LRUMap validVisibilities = new LRUMap(ConfiguratorBase.getVisibilityCacheSize(conf));
return new RecordWriter<Key,Value>() {
FileSKVWriter out = null;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index e87d43b..cf1f0ea 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@ -82,7 +82,8 @@ public class ConfiguratorBase {
* @since 1.6.0
*/
public static enum GeneralOpts {
- LOG_LEVEL
+ LOG_LEVEL,
+ VISIBILITY_CACHE_SIZE
}
/**
@@ -100,6 +101,17 @@ public class ConfiguratorBase {
}
/**
+ * Provides a configuration key for a given feature enum.
+ *
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ */
+ protected static String enumToConfKey(Enum<?> e) {
+ return e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
* Sets the connector information needed to communicate with Accumulo in this job.
*
* <p>
@@ -369,4 +381,27 @@ public class ConfiguratorBase {
return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
}
+ /**
+ * Sets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param visibilityCacheSize
+ * the LRU cache size
+ */
+ public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
+ conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
+ }
+
+ /**
+ * Gets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the valid visibility count
+ */
+ public static int getVisibilityCacheSize(Configuration conf) {
+ return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
index aad544b..3a4d641 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
@@ -28,6 +28,7 @@ import java.io.IOException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -53,6 +54,7 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
public class AccumuloFileOutputFormatTest {
+ private static final int JOB_VISIBILITY_CACHE_SIZE = 3000;
private static final String PREFIX = AccumuloFileOutputFormatTest.class.getSimpleName();
private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
private static final String BAD_TABLE = PREFIX + "_mapred_bad_table";
@@ -145,6 +147,7 @@ public class AccumuloFileOutputFormatTest {
JobConf job = new JobConf(getConf());
job.setJarByClass(this.getClass());
+ ConfiguratorBase.setVisibilityCacheSize(job, JOB_VISIBILITY_CACHE_SIZE);
job.setInputFormat(AccumuloInputFormat.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
index d5ebb22..7c1f98b 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
@@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.ClientConfiguration;
import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
@@ -126,4 +127,11 @@ public class ConfiguratorBaseTest {
assertEquals(Level.FATAL, ConfiguratorBase.getLogLevel(this.getClass(), conf));
}
+ @Test
+ public void testSetVisibiltyCacheSize(){
+ Configuration conf = new Configuration();
+ assertEquals(Constants.DEFAULT_VISIBILITY_CACHE_SIZE,ConfiguratorBase.getVisibilityCacheSize(conf));
+ ConfiguratorBase.setVisibilityCacheSize(conf, 2000);
+ assertEquals(2000,ConfiguratorBase.getVisibilityCacheSize(conf));
+ }
}
[3/6] git commit: Merge branch '1.6.1-SNAPSHOT' of
https://git-wip-us.apache.org/repos/asf/accumulo into 1.6.1-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.6.1-SNAPSHOT' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.6.1-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/73e0f9ce
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/73e0f9ce
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/73e0f9ce
Branch: refs/heads/master
Commit: 73e0f9ced78d7ced44b2286dfe8e7bdf3f6ee3da
Parents: 7a8ec73 e7e050a
Author: Josh Elser <el...@apache.org>
Authored: Sun Jun 8 02:40:12 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Sun Jun 8 02:40:12 2014 -0400
----------------------------------------------------------------------
.../accumulo/fate/zookeeper/ZooCache.java | 85 ++++-
.../accumulo/fate/zookeeper/ZooCacheTest.java | 345 +++++++++++++++++++
.../system/continuous/continuous-env.sh.example | 30 +-
test/system/continuous/start-agitator.sh | 12 +-
4 files changed, 450 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
[2/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e7e050a6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e7e050a6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e7e050a6
Branch: refs/heads/master
Commit: e7e050a6b05b88c90a93405577e7a6eb0751f3c3
Parents: 4950870 c839e2f
Author: Drew Farris <dr...@apache.org>
Authored: Sat Jun 7 10:11:52 2014 -0400
Committer: Drew Farris <dr...@apache.org>
Committed: Sat Jun 7 10:11:52 2014 -0400
----------------------------------------------------------------------
.../system/continuous/continuous-env.sh.example | 30 ++++++++++++++------
test/system/continuous/start-agitator.sh | 12 ++++----
2 files changed, 27 insertions(+), 15 deletions(-)
----------------------------------------------------------------------