You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/06/09 16:42:54 UTC
[4/6] git commit: ACCUMULO-1919 Size of visibility cache in
AccumuloFileOutputFormat should be configurable.
ACCUMULO-1919 Size of visibility cache in AccumuloFileOutputFormat should be configurable.
Signed-off-by: Josh Elser <el...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ae1682d4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ae1682d4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ae1682d4
Branch: refs/heads/master
Commit: ae1682d4ae7a3031598748f676c90829d0b2a978
Parents: 73e0f9c
Author: dallaybatta <vi...@gmail.com>
Authored: Mon Jun 9 11:29:54 2014 +0530
Committer: Josh Elser <el...@apache.org>
Committed: Mon Jun 9 10:27:51 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 2 ++
.../client/mapred/AccumuloFileOutputFormat.java | 3 +-
.../mapreduce/lib/impl/ConfiguratorBase.java | 37 +++++++++++++++++++-
.../mapred/AccumuloFileOutputFormatTest.java | 3 ++
.../lib/impl/ConfiguratorBaseTest.java | 8 +++++
5 files changed, 51 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java
index e0e88eb..c9a51a9 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -115,4 +115,6 @@ public class Constants {
public static final String[] PATH_PROPERTY_ENV_VARS = new String[] {"ACCUMULO_HOME", "ACCUMULO_CONF_DIR"};
public static final String HDFS_TABLES_DIR = "/tables";
+
+ public static final int DEFAULT_VISIBILITY_CACHE_SIZE = 1000;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
index 8a1d6df..cfaaa58 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormat.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.mapred;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
import org.apache.accumulo.core.client.mapreduce.lib.impl.FileOutputConfigurator;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -145,7 +146,7 @@ public class AccumuloFileOutputFormat extends FileOutputFormat<Key,Value> {
final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension);
- final LRUMap validVisibilities = new LRUMap(1000);
+ final LRUMap validVisibilities = new LRUMap(ConfiguratorBase.getVisibilityCacheSize(conf));
return new RecordWriter<Key,Value>() {
FileSKVWriter out = null;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
index e87d43b..cf1f0ea 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBase.java
@@ -82,7 +82,8 @@ public class ConfiguratorBase {
* @since 1.6.0
*/
public static enum GeneralOpts {
- LOG_LEVEL
+ LOG_LEVEL,
+ VISIBILITY_CACHE_SIZE
}
/**
@@ -100,6 +101,17 @@ public class ConfiguratorBase {
}
/**
+ * Provides a configuration key for a given feature enum.
+ *
+ * @param e
+ * the enum used to provide the unique part of the configuration key
+ * @return the configuration key
+ */
+ protected static String enumToConfKey(Enum<?> e) {
+ return e.getDeclaringClass().getSimpleName() + "." + StringUtils.camelize(e.name().toLowerCase());
+ }
+
+ /**
* Sets the connector information needed to communicate with Accumulo in this job.
*
* <p>
@@ -369,4 +381,27 @@ public class ConfiguratorBase {
return Level.toLevel(conf.getInt(enumToConfKey(implementingClass, GeneralOpts.LOG_LEVEL), Level.INFO.toInt()));
}
+ /**
+ * Sets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @param visibilityCacheSize
+ * the LRU cache size
+ */
+ public static void setVisibilityCacheSize(Configuration conf, int visibilityCacheSize) {
+ conf.setInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE), visibilityCacheSize);
+ }
+
+ /**
+ * Gets the valid visibility count for this job.
+ *
+ * @param conf
+ * the Hadoop configuration object to configure
+ * @return the valid visibility count
+ */
+ public static int getVisibilityCacheSize(Configuration conf) {
+ return conf.getInt(enumToConfKey(GeneralOpts.VISIBILITY_CACHE_SIZE),Constants.DEFAULT_VISIBILITY_CACHE_SIZE);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
index aad544b..3a4d641 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
@@ -28,6 +28,7 @@ import java.io.IOException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.lib.impl.ConfiguratorBase;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -53,6 +54,7 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
public class AccumuloFileOutputFormatTest {
+ private static final int JOB_VISIBILITY_CACHE_SIZE = 3000;
private static final String PREFIX = AccumuloFileOutputFormatTest.class.getSimpleName();
private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
private static final String BAD_TABLE = PREFIX + "_mapred_bad_table";
@@ -145,6 +147,7 @@ public class AccumuloFileOutputFormatTest {
JobConf job = new JobConf(getConf());
job.setJarByClass(this.getClass());
+ ConfiguratorBase.setVisibilityCacheSize(job, JOB_VISIBILITY_CACHE_SIZE);
job.setInputFormat(AccumuloInputFormat.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ae1682d4/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
index d5ebb22..7c1f98b 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/impl/ConfiguratorBaseTest.java
@@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.ClientConfiguration;
import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
@@ -126,4 +127,11 @@ public class ConfiguratorBaseTest {
assertEquals(Level.FATAL, ConfiguratorBase.getLogLevel(this.getClass(), conf));
}
+ @Test
+ public void testSetVisibiltyCacheSize(){
+ Configuration conf = new Configuration();
+ assertEquals(Constants.DEFAULT_VISIBILITY_CACHE_SIZE,ConfiguratorBase.getVisibilityCacheSize(conf));
+ ConfiguratorBase.setVisibilityCacheSize(conf, 2000);
+ assertEquals(2000,ConfiguratorBase.getVisibilityCacheSize(conf));
+ }
}