You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2017/09/12 02:24:47 UTC
[accumulo] 01/03: ACCUMULO-4086 Improve volume chooser fallback
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 513d1f500ca62374225008d1a2b93dd9384b9bf9
Author: Ivan Bella <iv...@bella.name>
AuthorDate: Wed Jul 26 21:28:05 2017 -0400
ACCUMULO-4086 Improve volume chooser fallback
Implement a better fallback mechanism when the user's selected
VolumeChooser implementation cannot be loaded or is not specified.
Handles all such cases, including non-table scopes (logger), and
per-table scopes.
* Modified to fail upon misconfiguration instead of falling back to an
alternate chooser
* Ensure we have an appropriate default
* Changed the property names, and made the properties consistent between
the PerTableVolumeChooser and the PreferredVolumeChooser
* Use random volume chooser by default
* Use random volume chooser on init scope for PerTableVolumeChooser
This squashed commit contains contributions from the following authors:
matthpeterson <mp...@appliedtech-group.com>
Mike Walch <mw...@apache.org>
Christopher Tubbs <ct...@apache.org>
Ivan Bella <iv...@bella.name>
---
.../org/apache/accumulo/core/conf/Property.java | 5 +-
.../accumulo/server/fs/PerTableVolumeChooser.java | 166 +++++++++---
.../accumulo/server/fs/PreferredVolumeChooser.java | 148 ++++++++---
.../accumulo/server/fs/VolumeManagerImpl.java | 30 ++-
.../server/fs/PerTableVolumeChooserTest.java | 293 +++++++++++++++++++++
.../server/fs/PreferredVolumeChooserTest.java | 252 ++++++++++++++++++
.../accumulo/server/fs/VolumeManagerImplTest.java | 33 ++-
.../accumulo/tserver/tablet/RootFilesTest.java | 2 +
.../org/apache/accumulo/test/ShellConfigIT.java | 10 +-
.../accumulo/test/VolumeChooserFailureIT.java | 153 +++++++++++
.../org/apache/accumulo/test/VolumeChooserIT.java | 276 ++++++++-----------
11 files changed, 1106 insertions(+), 262 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 810c3fe..d0dffa4 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -191,7 +191,7 @@ public enum Property {
+ "server-internal scheduled tasks"),
// If you update the default type, be sure to update the default used for initialization failures in VolumeManagerImpl
@Experimental
- GENERAL_VOLUME_CHOOSER("general.volume.chooser", "org.apache.accumulo.server.fs.PerTableVolumeChooser", PropertyType.CLASSNAME,
+ GENERAL_VOLUME_CHOOSER("general.volume.chooser", "org.apache.accumulo.server.fs.RandomVolumeChooser", PropertyType.CLASSNAME,
"The class that will be used to select which volume will be used to create new files."),
GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS("general.security.credential.provider.paths", "", PropertyType.STRING,
"Comma-separated list of paths to CredentialProviders"),
@@ -554,9 +554,6 @@ public enum Property {
TABLE_REPLICATION_TARGET("table.replication.target.", null, PropertyType.PREFIX, "Enumerate a mapping of other systems which this table should "
+ "replicate their data to. The key suffix is the identifying cluster name and the value is an identifier for a location on the target system, "
+ "e.g. the ID of the table on the target to replicate to"),
- @Experimental
- TABLE_VOLUME_CHOOSER("table.volume.chooser", "org.apache.accumulo.server.fs.RandomVolumeChooser", PropertyType.CLASSNAME,
- "The class that will be used to select which volume will be used to create new files for this table."),
TABLE_SAMPLER(
"table.sampler",
"",
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/PerTableVolumeChooser.java b/server/base/src/main/java/org/apache/accumulo/server/fs/PerTableVolumeChooser.java
index 4a57100..4f630b1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/PerTableVolumeChooser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/PerTableVolumeChooser.java
@@ -18,61 +18,157 @@ package org.apache.accumulo.server.fs;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * A {@link VolumeChooser} that delegates to another volume chooser based on the presence of an experimental table property,
- * {@link Property#TABLE_VOLUME_CHOOSER}. If it isn't found, defaults back to {@link RandomVolumeChooser}.
+ * A {@link VolumeChooser} that delegates to another volume chooser based on other properties: table.custom.volume.chooser for tables, and
+ * general.custom.scoped.volume.chooser for scopes. general.custor.{scope}.volume.chooser can override the system wide setting for
+ * general.custom.scoped.volume.chooser. At the this this was written, the only known scope was "logger".
*/
public class PerTableVolumeChooser implements VolumeChooser {
-
- private final VolumeChooser fallbackVolumeChooser = new RandomVolumeChooser();
+ private static final Logger log = LoggerFactory.getLogger(PerTableVolumeChooser.class);
// TODO Add hint of expected size to construction, see ACCUMULO-3410
/* Track VolumeChooser instances so they can keep state. */
- private final ConcurrentHashMap<Table.ID,VolumeChooser> tableSpecificChooser = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String,VolumeChooser> tableSpecificChooser = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String,VolumeChooser> scopeSpecificChooser = new ConcurrentHashMap<>();
+ private final RandomVolumeChooser randomChooser = new RandomVolumeChooser();
+
// TODO has to be lazily initialized currently because of the reliance on HdfsZooInstance. see ACCUMULO-3411
private volatile ServerConfigurationFactory serverConfs;
+ public static final String INIT_SCOPE = "init";
+
+ public static final String TABLE_VOLUME_CHOOSER = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "volume.chooser";
+
+ public static final String SCOPED_VOLUME_CHOOSER(String scope) {
+ return Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey() + scope + ".volume.chooser";
+ }
+
+ public static final String DEFAULT_SCOPED_VOLUME_CHOOSER = SCOPED_VOLUME_CHOOSER("scoped");
+
@Override
public String choose(VolumeChooserEnvironment env, String[] options) {
- VolumeChooser chooser = null;
+ log.trace("PerTableVolumeChooser.choose");
+
+ VolumeChooser chooser;
+ if (!env.hasTableId() && (!env.hasScope() || env.getScope().equals(INIT_SCOPE))) {
+ // Should only get here during Initialize. Configurations are not yet available.
+ return randomChooser.choose(env, options);
+ }
+
+ ServerConfigurationFactory localConf = loadConf();
if (env.hasTableId()) {
- // This local variable is an intentional component of the single-check idiom.
- ServerConfigurationFactory localConf = serverConfs;
- if (localConf == null) {
- // If we're under contention when first getting here we'll throw away some initializations.
- localConf = new ServerConfigurationFactory(HdfsZooInstance.getInstance());
- serverConfs = localConf;
+ // use the table configuration
+ chooser = getVolumeChooserForTable(env, localConf);
+ } else {
+ // use the system configuration
+ chooser = getVolumeChooserForNonTable(env, localConf);
+ }
+
+ return chooser.choose(env, options);
+ }
+
+ private VolumeChooser getVolumeChooserForTable(VolumeChooserEnvironment env, ServerConfigurationFactory localConf) {
+ log.trace("Looking up property {} for Table id: {}", TABLE_VOLUME_CHOOSER, env.getTableId());
+ final TableConfiguration tableConf = localConf.getTableConfiguration(env.getTableId());
+ String clazz = tableConf.get(TABLE_VOLUME_CHOOSER);
+
+ if (null == clazz || clazz.isEmpty()) {
+ String msg = "Property " + TABLE_VOLUME_CHOOSER + " must be set" + (null == clazz ? " " : " properly ") + "to use the " + getClass().getSimpleName();
+ throw new RuntimeException(msg);
+ }
+
+ return createVolumeChooser(clazz, TABLE_VOLUME_CHOOSER, env.getTableId().canonicalID(), tableSpecificChooser);
+ }
+
+ private VolumeChooser getVolumeChooserForNonTable(VolumeChooserEnvironment env, ServerConfigurationFactory localConf) {
+ String scope = env.getScope();
+ String property = SCOPED_VOLUME_CHOOSER(scope);
+
+ log.trace("Looking up property: {}", property);
+
+ AccumuloConfiguration systemConfiguration = localConf.getSystemConfiguration();
+ String clazz = systemConfiguration.get(property);
+ // only if the custom property is not set do we fallback to the default scope volume chooser setting
+ if (null == clazz) {
+ log.debug("Property not found: {} using {}", property, DEFAULT_SCOPED_VOLUME_CHOOSER);
+ clazz = systemConfiguration.get(DEFAULT_SCOPED_VOLUME_CHOOSER);
+
+ if (null == clazz || clazz.isEmpty()) {
+ String msg = "Property " + property + " or " + DEFAULT_SCOPED_VOLUME_CHOOSER + " must be set" + (null == clazz ? " " : " properly ") + "to use the "
+ + getClass().getSimpleName();
+ throw new RuntimeException(msg);
}
- final TableConfiguration tableConf = localConf.getTableConfiguration(env.getTableId());
- chooser = tableSpecificChooser.get(env.getTableId());
- if (chooser == null) {
- VolumeChooser temp = Property.createTableInstanceFromPropertyName(tableConf, Property.TABLE_VOLUME_CHOOSER, VolumeChooser.class, fallbackVolumeChooser);
- chooser = tableSpecificChooser.putIfAbsent(env.getTableId(), temp);
- if (chooser == null) {
- chooser = temp;
- // Otherwise, someone else beat us to initializing; use theirs.
- }
- } else if (!(chooser.getClass().getName().equals(tableConf.get(Property.TABLE_VOLUME_CHOOSER)))) {
- // the configuration for this table's chooser has been updated. In the case of failure to instantiate we'll repeat here next call.
+
+ property = DEFAULT_SCOPED_VOLUME_CHOOSER;
+ }
+
+ return createVolumeChooser(clazz, property, scope, scopeSpecificChooser);
+ }
+
+ /**
+ * Create a volume chooser, using the cached version if any. This will replace the cached version if the class name has changed.
+ *
+ * @param clazz
+ * The volume chooser class name
+ * @param property
+ * The property from which it was obtained
+ * @param key
+ * The key to user in the cache
+ * @param cache
+ * The cache
+ * @return The volume chooser instance
+ */
+ private VolumeChooser createVolumeChooser(String clazz, String property, String key, ConcurrentHashMap<String,VolumeChooser> cache) {
+ VolumeChooser chooser = cache.get(key);
+ // if we do not have a chooser or the class has changed, then create a new one
+ if (chooser == null || !(chooser.getClass().getName().equals(clazz))) {
+ if (chooser != null) {
// TODO stricter definition of when the updated property is used, ref ACCUMULO-3412
- VolumeChooser temp = Property.createTableInstanceFromPropertyName(tableConf, Property.TABLE_VOLUME_CHOOSER, VolumeChooser.class, fallbackVolumeChooser);
- VolumeChooser last = tableSpecificChooser.replace(env.getTableId(), temp);
- if (chooser.equals(last)) {
- chooser = temp;
- } else {
- // Someone else beat us to updating; use theirs.
- chooser = last;
- }
+ log.trace("Change detected for {} for {}", property, key);
+ }
+ // create a new volume chooser instance
+ VolumeChooser temp;
+ try {
+ temp = loadClass(clazz);
+ } catch (Exception e) {
+ String msg = "Failed to create instance for " + key + " configured to use " + clazz + " via " + property;
+ throw new RuntimeException(msg, e);
+ }
+ if (chooser == null) {
+ // if we did not have one previously, then put this one in the cache
+ // but use the one already in the cache if another thread beat us here
+ chooser = cache.computeIfAbsent(key, k -> temp);
+ } else {
+ // otherwise the class has changed, so replace the one in the cache
+ // unless another thread beat us here
+ chooser = cache.computeIfPresent(key, (k, v) -> (v.getClass().getName().equals(clazz) ? v : temp));
}
- } else {
- chooser = fallbackVolumeChooser;
}
+ return chooser;
+ }
- return chooser.choose(env, options);
+ private VolumeChooser loadClass(String className) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
+ // not attempting to load context because this approach to loading the class is for non-tables only
+ return AccumuloVFSClassLoader.loadClass(className, VolumeChooser.class).newInstance();
}
+
+ private ServerConfigurationFactory loadConf() {
+ // This local variable is an intentional component of the single-check idiom.
+ ServerConfigurationFactory localConf = serverConfs;
+ if (localConf == null) {
+ // If we're under contention when first getting here we'll throw away some initializations.
+ localConf = new ServerConfigurationFactory(HdfsZooInstance.getInstance());
+ serverConfs = localConf;
+ }
+ return localConf;
+ }
+
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java b/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
index 0143f24..edd9931 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
@@ -21,12 +21,12 @@ import static org.apache.commons.lang.ArrayUtils.EMPTY_STRING_ARRAY;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.function.Predicate;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.client.HdfsZooInstance;
@@ -39,18 +39,21 @@ import org.slf4j.LoggerFactory;
/**
* A {@link RandomVolumeChooser} that limits its choices from a given set of options to the subset of those options preferred for a particular table. Defaults
- * to selecting from all of the options presented. Can be customized via the table property {@value #PREFERRED_VOLUMES_CUSTOM_KEY}, which should contain a comma
+ * to selecting from all of the options presented. Can be customized via the table property table.custom.preferredVolumes, which should contain a comma
* separated list of {@link Volume} URIs. Note that both the property name and the format of its value are specific to this particular implementation.
*/
public class PreferredVolumeChooser extends RandomVolumeChooser {
private static final Logger log = LoggerFactory.getLogger(PreferredVolumeChooser.class);
- /**
- * This should match {@link Property#TABLE_ARBITRARY_PROP_PREFIX}
- */
- public static final String PREFERRED_VOLUMES_CUSTOM_KEY = "table.custom.preferredVolumes";
- // TODO ACCUMULO-3417 replace this with the ability to retrieve by String key.
- private static final Predicate<String> PREFERRED_VOLUMES_FILTER = key -> PREFERRED_VOLUMES_CUSTOM_KEY.equals(key);
+ public static final String INIT_SCOPE = "init";
+
+ public static final String TABLE_PREFERRED_VOLUMES = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "preferred.volumes";
+
+ public static final String SCOPED_PREFERRED_VOLUMES(String scope) {
+ return Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey() + scope + ".preferred.volumes";
+ }
+
+ public static final String DEFAULT_SCOPED_PREFERRED_VOLUMES = SCOPED_PREFERRED_VOLUMES("scoped");
@SuppressWarnings("unchecked")
private final Map<String,Set<String>> parsedPreferredVolumes = Collections.synchronizedMap(new LRUMap(1000));
@@ -59,53 +62,114 @@ public class PreferredVolumeChooser extends RandomVolumeChooser {
@Override
public String choose(VolumeChooserEnvironment env, String[] options) {
- if (!env.hasTableId())
+ if (!env.hasTableId() && (!env.hasScope() || env.getScope().equals(INIT_SCOPE))) {
+ // this should only happen during initialize
+ log.warn("No table id or scope, so it's not possible to determine preferred volumes. Using all volumes.");
return super.choose(env, options);
+ }
- // Get the current table's properties, and find the preferred volumes property
- // This local variable is an intentional component of the single-check idiom.
- ServerConfigurationFactory localConf = serverConfs;
- if (localConf == null) {
- // If we're under contention when first getting here we'll throw away some initializations.
- localConf = new ServerConfigurationFactory(HdfsZooInstance.getInstance());
- serverConfs = localConf;
+ // get the volumes property
+ ServerConfigurationFactory localConf = loadConf();
+ List<String> volumes;
+ if (env.hasTableId()) {
+ volumes = getPreferredVolumesForTable(env, localConf, options);
+ } else {
+ volumes = getPreferredVolumesForNonTable(env, localConf, options);
}
- TableConfiguration tableConf = localConf.getTableConfiguration(env.getTableId());
- final Map<String,String> props = new HashMap<>();
- tableConf.getProperties(props, PREFERRED_VOLUMES_FILTER);
- if (props.isEmpty()) {
- log.warn("No preferred volumes specified. Defaulting to randomly choosing from instance volumes");
- return super.choose(env, options);
+
+ // Randomly choose the volume from the preferred volumes
+ String choice = super.choose(env, volumes.toArray(EMPTY_STRING_ARRAY));
+ log.trace("Choice = {}", choice);
+
+ return choice;
+ }
+
+ private List<String> getPreferredVolumesForTable(VolumeChooserEnvironment env, ServerConfigurationFactory localConf, String[] options) {
+ log.trace("Looking up property {} + for Table id: {}", TABLE_PREFERRED_VOLUMES, env.getTableId());
+
+ final TableConfiguration tableConf = localConf.getTableConfiguration(env.getTableId());
+ String volumes = tableConf.get(TABLE_PREFERRED_VOLUMES);
+
+ // throw an error if volumes not specified or empty
+ if (null == volumes || volumes.isEmpty()) {
+ String logmsg = "Missing or empty " + TABLE_PREFERRED_VOLUMES + " property";
+ throw new RuntimeException(logmsg);
}
- String volumes = props.get(PREFERRED_VOLUMES_CUSTOM_KEY);
- if (log.isTraceEnabled()) {
- log.trace("In custom chooser");
- log.trace("Volumes: {}", volumes);
- log.trace("TableID: {}", env.getTableId());
+ return getFilteredOptions(TABLE_PREFERRED_VOLUMES, volumes, options);
+ }
+
+ private List<String> getPreferredVolumesForNonTable(VolumeChooserEnvironment env, ServerConfigurationFactory localConf, String[] options) {
+ String scope = env.getScope();
+ String property = SCOPED_PREFERRED_VOLUMES(scope);
+
+ log.trace("Looking up property: {}", property);
+
+ AccumuloConfiguration systemConfiguration = localConf.getSystemConfiguration();
+ String volumes = systemConfiguration.get(property);
+
+ // only if the custom property is not set to we fallback to the default scoped preferred volumes
+ if (null == volumes) {
+ log.debug("Property not found: {} using {}", property, DEFAULT_SCOPED_PREFERRED_VOLUMES);
+ volumes = systemConfiguration.get(DEFAULT_SCOPED_PREFERRED_VOLUMES);
+
+ if (null == volumes || volumes.isEmpty()) {
+ String logmsg = "Missing or empty " + property + " and " + DEFAULT_SCOPED_PREFERRED_VOLUMES + " properties";
+ throw new RuntimeException(logmsg);
+ }
+
+ property = DEFAULT_SCOPED_PREFERRED_VOLUMES;
}
- // If the preferred volumes property was specified, split the returned string by the comma and add use it to filter the given options.
- Set<String> preferred = parsedPreferredVolumes.get(volumes);
- if (preferred == null) {
- preferred = new HashSet<>(Arrays.asList(StringUtils.split(volumes, ',')));
- parsedPreferredVolumes.put(volumes, preferred);
+
+ return getFilteredOptions(property, volumes, options);
+ }
+
+ private List<String> getFilteredOptions(String property, String volumes, String[] options) {
+ log.trace("Found {} = {}", property, volumes);
+
+ ArrayList<String> filteredOptions = getIntersection(options, volumes);
+
+ // throw error if intersecting with preferred volumes resulted in the empty set
+ if (filteredOptions.isEmpty()) {
+ String logMessage = "After filtering preferred volumes, there are no valid instance volumes.";
+ log.error(logMessage);
+ throw new RuntimeException(logMessage);
}
+ return filteredOptions;
+ }
+
+ private ArrayList<String> getIntersection(String[] options, String volumes) {
+ Set<String> preferred = parseVolumes(volumes);
+ return filterWithPreferred(options, preferred);
+ }
+
+ private ArrayList<String> filterWithPreferred(String[] options, Set<String> preferred) {
// Only keep the options that are in the preferred set
final ArrayList<String> filteredOptions = new ArrayList<>(Arrays.asList(options));
filteredOptions.retainAll(preferred);
+ return filteredOptions;
+ }
- // If there are no preferred volumes left, then warn the user and choose randomly from the instance volumes
- if (filteredOptions.isEmpty()) {
- log.warn("Preferred volumes are not instance volumes. Defaulting to randomly choosing from instance volumes");
- return super.choose(env, options);
+ private Set<String> parseVolumes(String volumes) {
+ // If the preferred volumes property was specified, split the returned string by the comma and add use it to filter the given options.
+ Set<String> preferred = parsedPreferredVolumes.get(volumes);
+ if (preferred == null) {
+ preferred = new HashSet<>(Arrays.asList(StringUtils.split(volumes, ',')));
+ parsedPreferredVolumes.put(volumes, preferred);
}
+ return preferred;
+ }
- // Randomly choose the volume from the preferred volumes
- String choice = super.choose(env, filteredOptions.toArray(EMPTY_STRING_ARRAY));
- if (log.isTraceEnabled()) {
- log.trace("Choice = {}", choice);
+ private ServerConfigurationFactory loadConf() {
+ // Get the current table's properties, and find the preferred volumes property
+ // This local variable is an intentional component of the single-check idiom.
+ ServerConfigurationFactory localConf = serverConfs;
+ if (localConf == null) {
+ // If we're under contention when first getting here we'll throw away some initializations.
+ localConf = new ServerConfigurationFactory(HdfsZooInstance.getInstance());
+ serverConfs = localConf;
}
- return choice;
+ return localConf;
}
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index 4495e79..04e86fc 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -79,8 +79,18 @@ public class VolumeManagerImpl implements VolumeManager {
this.volumesByFileSystemUri = HashMultimap.create();
invertVolumesByFileSystem(volumesByName, volumesByFileSystemUri);
ensureSyncIsEnabled();
- // Keep in sync with default type in the property definition.
- chooser = Property.createInstanceFromPropertyName(conf, Property.GENERAL_VOLUME_CHOOSER, VolumeChooser.class, new PerTableVolumeChooser());
+ // if they supplied a property and we cannot load it, then fail hard
+ VolumeChooser chooser1;
+ try {
+ chooser1 = Property.createInstanceFromPropertyName(conf, Property.GENERAL_VOLUME_CHOOSER, VolumeChooser.class, null);
+ } catch (NullPointerException npe) {
+ chooser1 = null;
+ // null chooser handled below
+ }
+ if (chooser1 == null) {
+ throw new RuntimeException("Failed to load volume chooser specified by " + Property.GENERAL_VOLUME_CHOOSER);
+ }
+ chooser = chooser1;
}
private void invertVolumesByFileSystem(Map<String,Volume> forward, Multimap<URI,Volume> inverted) {
@@ -408,7 +418,7 @@ public class VolumeManagerImpl implements VolumeManager {
if (path.startsWith("../"))
path = path.substring(2);
else if (path.startsWith("/"))
- path = "/" + tableId + path;
+ path = "/" + tableId.canonicalID() + path;
else
throw new IllegalArgumentException("Unexpected path prefix " + path);
@@ -464,17 +474,17 @@ public class VolumeManagerImpl implements VolumeManager {
return getVolumeByPath(dir).getFileSystem().getContentSummary(dir);
}
- // Only used as a fall back if the configured chooser misbehaves.
- private final VolumeChooser failsafeChooser = new RandomVolumeChooser();
-
@Override
public String choose(VolumeChooserEnvironment env, String[] options) {
- final String choice = chooser.choose(env, options);
+ final String choice;
+ choice = chooser.choose(env, options);
+
if (!(ArrayUtils.contains(options, choice))) {
- log.error("The configured volume chooser, '{}', or one of its delegates returned a volume not in the set of options provided; "
- + "will continue by relying on a RandomVolumeChooser. You should investigate and correct the named chooser.", chooser.getClass());
- return failsafeChooser.choose(env, options);
+ // we may want to go with random if they chooser was not overridden
+ String msg = "The configured volume chooser, '" + chooser.getClass() + "', or one of its delegates returned a volume not in the set of options provided";
+ throw new RuntimeException(msg);
}
+
return choice;
}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
new file mode 100644
index 0000000..f8946c4
--- /dev/null
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/PerTableVolumeChooserTest.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.fs;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import com.google.common.collect.Sets;
+
+public class PerTableVolumeChooserTest {
+ private static final int REQUIRED_NUMBER_TRIES = 20; // times to call choose for likely exercising of each preferred volume
+ private static final String[] ALL_OPTIONS = new String[] {"1", "2", "3"};
+ public static final String INVALID_CHOOSER_CLASSNAME = "MysteriousVolumeChooser";
+ private ServerConfigurationFactory mockedServerConfigurationFactory;
+ private TableConfiguration mockedTableConfiguration;
+ private PerTableVolumeChooser perTableVolumeChooser;
+ private AccumuloConfiguration mockedAccumuloConfiguration;
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void before() throws Exception {
+ perTableVolumeChooser = new PerTableVolumeChooser();
+
+ mockedServerConfigurationFactory = EasyMock.createMock(ServerConfigurationFactory.class);
+ Field field = perTableVolumeChooser.getClass().getDeclaredField("serverConfs");
+ field.setAccessible(true);
+ field.set(perTableVolumeChooser, mockedServerConfigurationFactory);
+
+ mockedTableConfiguration = EasyMock.createMock(TableConfiguration.class);
+ mockedAccumuloConfiguration = EasyMock.createMock(AccumuloConfiguration.class);
+ }
+
+ private void configureDefaultVolumeChooser(String className) {
+ EasyMock.expect(mockedServerConfigurationFactory.getSystemConfiguration()).andReturn(mockedAccumuloConfiguration).anyTimes();
+ EasyMock.expect(mockedAccumuloConfiguration.get(PerTableVolumeChooser.TABLE_VOLUME_CHOOSER)).andReturn(className).anyTimes();
+ }
+
+ private void configureScopedVolumeChooser(String className, String scope) {
+ EasyMock.expect(mockedServerConfigurationFactory.getSystemConfiguration()).andReturn(mockedAccumuloConfiguration).anyTimes();
+ EasyMock.expect(mockedAccumuloConfiguration.get(PerTableVolumeChooser.SCOPED_VOLUME_CHOOSER(scope))).andReturn(className).anyTimes();
+ }
+
+ private void configureChooserForTable(String className) {
+ EasyMock.expect(mockedServerConfigurationFactory.getTableConfiguration(EasyMock.<Table.ID> anyObject())).andReturn(mockedTableConfiguration).anyTimes();
+ EasyMock.expect(mockedTableConfiguration.get(Property.TABLE_CLASSPATH)).andReturn(null).anyTimes();
+ EasyMock.expect(mockedTableConfiguration.get(PerTableVolumeChooser.TABLE_VOLUME_CHOOSER)).andReturn(className).anyTimes();
+ }
+
+ private void configureDefaultContextVolumeChooser(String className) {
+ EasyMock.expect(mockedAccumuloConfiguration.get(PerTableVolumeChooser.DEFAULT_SCOPED_VOLUME_CHOOSER)).andReturn(className).anyTimes();
+ }
+
+ private void configureContextVolumeChooser(String className) {
+ EasyMock.expect(mockedAccumuloConfiguration.get(PerTableVolumeChooser.SCOPED_VOLUME_CHOOSER("logger"))).andReturn(className).anyTimes();
+ }
+
+ private Set<String> chooseRepeatedlyForTable() {
+ VolumeChooserEnvironment volumeChooserEnvironment = new VolumeChooserEnvironment(Optional.of(Table.ID.of("h")));
+ Set<String> results = new HashSet<>();
+ for (int i = 0; i < REQUIRED_NUMBER_TRIES; i++) {
+ results.add(perTableVolumeChooser.choose(volumeChooserEnvironment, ALL_OPTIONS));
+ }
+ return results;
+ }
+
+ public static class VolumeChooserAlwaysOne extends VolumeChooserForFixedVolume {
+ public VolumeChooserAlwaysOne() {
+ super("1");
+ }
+ }
+
+ public static class VolumeChooserAlwaysTwo extends VolumeChooserForFixedVolume {
+ public VolumeChooserAlwaysTwo() {
+ super("2");
+ }
+ }
+
+ public static class VolumeChooserAlwaysThree extends VolumeChooserForFixedVolume {
+ public VolumeChooserAlwaysThree() {
+ super("3");
+ }
+ }
+
+ public static class VolumeChooserForFixedVolume implements VolumeChooser {
+ private final String onlyValidOption;
+
+ public VolumeChooserForFixedVolume(String fixedVolume) {
+ onlyValidOption = fixedVolume;
+ }
+
+ @Override
+ public String choose(VolumeChooserEnvironment env, String[] options) {
+ for (String option : options) {
+ if (onlyValidOption.equals(option)) {
+ return onlyValidOption;
+ }
+ }
+ return null;
+ }
+ }
+
+ private Set<String> chooseRepeatedlyForContext() {
+ VolumeChooserEnvironment volumeChooserEnvironment = new VolumeChooserEnvironment(Optional.empty());
+ volumeChooserEnvironment.setScope("logger");
+ Set<String> results = new HashSet<>();
+
+ for (int i = 0; i < REQUIRED_NUMBER_TRIES; i++) {
+ results.add(perTableVolumeChooser.choose(volumeChooserEnvironment, ALL_OPTIONS));
+ }
+ return results;
+ }
+
+ @Test
+ public void testEmptyEnvUsesRandomChooser() throws Exception {
+ VolumeChooserEnvironment volumeChooserEnvironment = new VolumeChooserEnvironment(Optional.empty());
+
+ Set<String> results = new HashSet<>();
+ for (int i = 0; i < REQUIRED_NUMBER_TRIES; i++) {
+ results.add(perTableVolumeChooser.choose(volumeChooserEnvironment, ALL_OPTIONS));
+ }
+
+ Assert.assertEquals(Sets.newHashSet(Arrays.asList(ALL_OPTIONS)), results);
+ }
+
+ @Test
+ public void testTableConfig() throws Exception {
+ configureDefaultVolumeChooser(VolumeChooserAlwaysOne.class.getName());
+ configureChooserForTable(VolumeChooserAlwaysTwo.class.getName());
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ Set<String> results = chooseRepeatedlyForTable();
+
+ EasyMock.verify(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+ Assert.assertEquals(Sets.newHashSet(Arrays.asList("2")), results);
+ }
+
+ @Test
+ public void testTableMisconfigured() throws Exception {
+ configureDefaultVolumeChooser(VolumeChooserAlwaysOne.class.getName());
+ configureChooserForTable(INVALID_CHOOSER_CLASSNAME);
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testTableMissing() throws Exception {
+ configureDefaultVolumeChooser(VolumeChooserAlwaysOne.class.getName());
+ configureChooserForTable(null);
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testTableMisconfiguredAndDefaultEmpty() throws Exception {
+ configureDefaultVolumeChooser("");
+ configureChooserForTable(INVALID_CHOOSER_CLASSNAME);
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testTableEmptyConfig() throws Exception {
+ configureDefaultVolumeChooser(VolumeChooserAlwaysThree.class.getName());
+ configureChooserForTable("");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testTableAndDefaultEmpty() throws Exception {
+ configureDefaultVolumeChooser("");
+ configureChooserForTable("");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testContextConfig() throws Exception {
+ configureDefaultVolumeChooser(VolumeChooserAlwaysThree.class.getName());
+ configureContextVolumeChooser(VolumeChooserAlwaysOne.class.getName());
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ Set<String> results = chooseRepeatedlyForContext();
+
+ EasyMock.verify(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+ Assert.assertEquals(Sets.newHashSet(Arrays.asList("1")), results);
+ }
+
+ @Test
+ public void testContextMisconfigured() throws Exception {
+ configureDefaultVolumeChooser(VolumeChooserAlwaysThree.class.getName());
+ configureContextVolumeChooser(INVALID_CHOOSER_CLASSNAME);
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+
+ @Test
+ public void testContextMissing() throws Exception {
+ configureDefaultVolumeChooser(VolumeChooserAlwaysTwo.class.getName());
+ configureContextVolumeChooser(null);
+ configureDefaultContextVolumeChooser(null);
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+
+ @Test
+ public void testContextMisconfiguredAndDefaultEmpty() throws Exception {
+ configureDefaultVolumeChooser("");
+ configureChooserForTable("");
+ configureContextVolumeChooser(INVALID_CHOOSER_CLASSNAME);
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+
+ @Test
+ public void testContextAndDefaultBothEmpty() throws Exception {
+ this.configureDefaultVolumeChooser("");
+ configureContextVolumeChooser("");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+
+ @Test
+ public void testContextEmptyConfig() throws Exception {
+ configureDefaultVolumeChooser(VolumeChooserAlwaysTwo.class.getName());
+ configureContextVolumeChooser("");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
new file mode 100644
index 0000000..fa3f672
--- /dev/null
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/PreferredVolumeChooserTest.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.fs;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.impl.Table;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import com.google.common.collect.Sets;
+
+public class PreferredVolumeChooserTest {
+ private static final int REQUIRED_NUMBER_TRIES = 20; // times to call choose for likely exercising of each preferred volume
+ private static final String[] ALL_OPTIONS = new String[] {"1", "2", "3"};
+ private ServerConfigurationFactory mockedServerConfigurationFactory;
+ private TableConfiguration mockedTableConfiguration;
+ private PreferredVolumeChooser preferredVolumeChooser;
+ private AccumuloConfiguration mockedAccumuloConfiguration;
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void before() throws Exception {
+ preferredVolumeChooser = new PreferredVolumeChooser();
+
+ mockedServerConfigurationFactory = EasyMock.createMock(ServerConfigurationFactory.class);
+ Field field = preferredVolumeChooser.getClass().getDeclaredField("serverConfs");
+ field.setAccessible(true);
+ field.set(preferredVolumeChooser, mockedServerConfigurationFactory);
+
+ mockedTableConfiguration = EasyMock.createMock(TableConfiguration.class);
+ mockedAccumuloConfiguration = EasyMock.createMock(AccumuloConfiguration.class);
+ }
+
+ private void configureDefaultVolumes(String configuredVolumes) {
+ EasyMock.expect(mockedServerConfigurationFactory.getSystemConfiguration()).andReturn(mockedAccumuloConfiguration).anyTimes();
+ EasyMock.expect(mockedAccumuloConfiguration.get(PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES)).andReturn(configuredVolumes).anyTimes();
+ }
+
+ private void configureTableVolumes(String configuredVolumes) {
+ EasyMock.expect(mockedServerConfigurationFactory.getTableConfiguration(EasyMock.<Table.ID> anyObject())).andReturn(mockedTableConfiguration).anyTimes();
+ EasyMock.expect(mockedTableConfiguration.get(PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES)).andReturn(configuredVolumes).anyTimes();
+ }
+
+ private void configureScopedVolumes(String configuredVolumes, String scope) {
+ EasyMock.expect(mockedServerConfigurationFactory.getSystemConfiguration()).andReturn(mockedAccumuloConfiguration).anyTimes();
+ EasyMock.expect(mockedAccumuloConfiguration.get(PreferredVolumeChooser.SCOPED_PREFERRED_VOLUMES(scope))).andReturn(configuredVolumes).anyTimes();
+ }
+
+ private void configureDefaultContextVolumes(String configuredVolumes) {
+ EasyMock.expect(mockedAccumuloConfiguration.get(PreferredVolumeChooser.DEFAULT_SCOPED_PREFERRED_VOLUMES)).andReturn(configuredVolumes).anyTimes();
+ }
+
+ private void configureContextVolumes(String configuredVolumes) {
+ EasyMock.expect(mockedAccumuloConfiguration.get(PreferredVolumeChooser.SCOPED_PREFERRED_VOLUMES("logger"))).andReturn(configuredVolumes).anyTimes();
+ }
+
+ private Set<String> chooseRepeatedlyForTable() {
+ VolumeChooserEnvironment volumeChooserEnvironment = new VolumeChooserEnvironment(Optional.of(Table.ID.of("h")));
+ Set<String> results = new HashSet<>();
+ for (int i = 0; i < REQUIRED_NUMBER_TRIES; i++) {
+ results.add(preferredVolumeChooser.choose(volumeChooserEnvironment, ALL_OPTIONS));
+ }
+ return results;
+ }
+
+ private Set<String> chooseRepeatedlyForContext() {
+ VolumeChooserEnvironment volumeChooserEnvironment = new VolumeChooserEnvironment(Optional.empty());
+ volumeChooserEnvironment.setScope("logger");
+ Set<String> results = new HashSet<>();
+
+ for (int i = 0; i < REQUIRED_NUMBER_TRIES; i++) {
+ results.add(preferredVolumeChooser.choose(volumeChooserEnvironment, ALL_OPTIONS));
+ }
+ return results;
+ }
+
+ @Test
+ public void testEmptyEnvUsesRandomChooser() throws Exception {
+ VolumeChooserEnvironment volumeChooserEnvironment = new VolumeChooserEnvironment(Optional.empty());
+ Set<String> results = new HashSet<>();
+ for (int i = 0; i < REQUIRED_NUMBER_TRIES; i++) {
+ results.add(preferredVolumeChooser.choose(volumeChooserEnvironment, ALL_OPTIONS));
+ }
+
+ Assert.assertEquals(Sets.newHashSet(Arrays.asList(ALL_OPTIONS)), results);
+ }
+
+ @Test
+ public void testTableConfig() throws Exception {
+ configureDefaultVolumes("1,3");
+ configureTableVolumes("1,2");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ Set<String> results = chooseRepeatedlyForTable();
+
+ EasyMock.verify(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+ Assert.assertEquals(Sets.newHashSet(Arrays.asList("1", "2")), results);
+ }
+
+ @Test
+ public void testTableMisconfigured() throws Exception {
+ configureDefaultVolumes("1,3");
+ configureTableVolumes("4");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testTableMissing() throws Exception {
+ configureDefaultVolumes("1,3");
+ configureTableVolumes(null);
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testTableEmptyConfig() throws Exception {
+ configureDefaultVolumes("1,3");
+ configureTableVolumes("");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testTableMisconfiguredAndDefaultEmpty() throws Exception {
+ configureDefaultVolumes("");
+ configureTableVolumes("4");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testTableAndDefaultEmpty() throws Exception {
+ configureDefaultVolumes("");
+ configureTableVolumes("");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedTableConfiguration, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForTable();
+ }
+
+ @Test
+ public void testContextConfig() throws Exception {
+ configureDefaultVolumes("1,3");
+ configureContextVolumes("1,2");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ Set<String> results = chooseRepeatedlyForContext();
+
+ EasyMock.verify(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+ Assert.assertEquals(Sets.newHashSet(Arrays.asList("1", "2")), results);
+ }
+
+ @Test
+ public void testContextMisconfigured() throws Exception {
+ configureDefaultVolumes("1,3");
+ configureContextVolumes("4");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+
+ @Test
+ public void testContextMissing() throws Exception {
+ configureDefaultVolumes("1,3");
+ configureContextVolumes(null);
+ configureDefaultContextVolumes(null);
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+
+ @Test
+ public void testContextMisconfiguredAndDefaultEmpty() throws Exception {
+ configureDefaultVolumes("");
+ configureContextVolumes("4");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+
+ @Test
+ public void testContextAndDefaultBothEmpty() throws Exception {
+ this.configureDefaultVolumes("");
+ configureContextVolumes("");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+
+ @Test
+ public void testContextEmptyConfig() throws Exception {
+ configureDefaultVolumes("1,3");
+ configureContextVolumes("");
+
+ EasyMock.replay(mockedServerConfigurationFactory, mockedAccumuloConfiguration);
+
+ thrown.expect(RuntimeException.class);
+ chooseRepeatedlyForContext();
+ }
+}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
index 13557bf..328f8f2 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeManagerImplTest.java
@@ -28,7 +28,9 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
/**
*
@@ -37,32 +39,50 @@ public class VolumeManagerImplTest {
protected VolumeManager fs;
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
@Before
public void setup() throws Exception {
fs = VolumeManagerImpl.getLocal(System.getProperty("user.dir"));
}
- @Test(expected = IllegalArgumentException.class)
+ @Test
public void defaultTabletDirWithoutTableId() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
fs.getFullPath(FileType.TABLE, "/default_tablet/");
}
- @Test(expected = IllegalArgumentException.class)
+ @Test
public void tabletDirWithoutTableId() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
fs.getFullPath(FileType.TABLE, "/t-0000001/");
}
- @Test(expected = IllegalArgumentException.class)
+ @Test
public void defaultTabletFileWithoutTableId() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
fs.getFullPath(FileType.TABLE, "/default_tablet/C0000001.rf");
}
- @Test(expected = IllegalArgumentException.class)
+ @Test
public void tabletFileWithoutTableId() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
fs.getFullPath(FileType.TABLE, "/t-0000001/C0000001.rf");
}
@Test
+ public void invalidChooserConfigured() throws Exception {
+ List<String> volumes = Arrays.asList("file://one/", "file://two/", "file://three/");
+ ConfigurationCopy conf = new ConfigurationCopy();
+ conf.set(INSTANCE_DFS_URI, volumes.get(0));
+ conf.set(Property.INSTANCE_VOLUMES, StringUtils.join(volumes, ","));
+ conf.set(Property.GENERAL_VOLUME_CHOOSER, "org.apache.accumulo.server.fs.ChooserThatDoesntExist");
+ thrown.expect(RuntimeException.class);
+ VolumeManagerImpl.get(conf);
+ }
+
+ @Test
public void tabletDirWithTableId() throws Exception {
String basePath = fs.getDefaultVolume().getBasePath();
String scheme = fs.getDefaultVolume().getFileSystem().getUri().toURL().getProtocol();
@@ -88,10 +108,11 @@ public class VolumeManagerImplTest {
}
}
- @Test(expected = IllegalArgumentException.class)
+ @Test
public void noViewFS() throws Exception {
ConfigurationCopy conf = new ConfigurationCopy();
conf.set(Property.INSTANCE_VOLUMES, "viewfs://dummy");
+ thrown.expect(IllegalArgumentException.class);
VolumeManagerImpl.get(conf);
}
@@ -105,6 +126,7 @@ public class VolumeManagerImplTest {
@SuppressWarnings("deprecation")
private static final Property INSTANCE_DFS_URI = Property.INSTANCE_DFS_URI;
+ // Expected to throw a runtime exception when the WrongVolumeChooser picks an invalid volume.
@Test
public void chooseFromOptions() throws Exception {
List<String> volumes = Arrays.asList("file://one/", "file://two/", "file://three/");
@@ -112,6 +134,7 @@ public class VolumeManagerImplTest {
conf.set(INSTANCE_DFS_URI, volumes.get(0));
conf.set(Property.INSTANCE_VOLUMES, StringUtils.join(volumes, ","));
conf.set(Property.GENERAL_VOLUME_CHOOSER, WrongVolumeChooser.class.getName());
+ thrown.expect(RuntimeException.class);
VolumeManager vm = VolumeManagerImpl.get(conf);
VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(Optional.of(Table.ID.of("sometable")));
String choice = vm.choose(chooserEnv, volumes.toArray(new String[0]));
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java
index d9c6862..fe87344 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/RootFilesTest.java
@@ -29,6 +29,7 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.RandomVolumeChooser;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.fs.Path;
@@ -123,6 +124,7 @@ public class RootFilesTest {
ConfigurationCopy conf = new ConfigurationCopy();
conf.set(Property.INSTANCE_DFS_URI, "file:///");
conf.set(Property.INSTANCE_DFS_DIR, "/");
+ conf.set(Property.GENERAL_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
VolumeManager vm = VolumeManagerImpl.get(conf);
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellConfigIT.java b/test/src/main/java/org/apache/accumulo/test/ShellConfigIT.java
index ae2e4cc..977cc36 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellConfigIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellConfigIT.java
@@ -30,6 +30,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.harness.conf.StandaloneAccumuloClusterConfiguration;
import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.server.fs.PerTableVolumeChooser;
import org.apache.accumulo.test.ShellServerIT.TestShell;
import org.junit.After;
import org.junit.Assert;
@@ -49,15 +50,15 @@ public class ShellConfigIT extends AccumuloClusterHarness {
Connector conn = getConnector();
// TABLE_VOLUME_CHOOSER is a valid property that can be updated in ZK, whereas the crypto properties are not.
// This lets us run this test more generically rather than forcibly needing to update some property in accumulo-site.xml
- origPropValue = conn.instanceOperations().getSystemConfiguration().get(Property.TABLE_VOLUME_CHOOSER.getKey());
- conn.instanceOperations().setProperty(Property.TABLE_VOLUME_CHOOSER.getKey(), FairVolumeChooser.class.getName());
+ origPropValue = conn.instanceOperations().getSystemConfiguration().get(PerTableVolumeChooser.TABLE_VOLUME_CHOOSER);
+ conn.instanceOperations().setProperty(PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, FairVolumeChooser.class.getName());
}
@After
public void resetProperty() throws Exception {
if (null != origPropValue) {
Connector conn = getConnector();
- conn.instanceOperations().setProperty(Property.TABLE_VOLUME_CHOOSER.getKey(), origPropValue);
+ conn.instanceOperations().setProperty(PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, origPropValue);
}
}
@@ -92,12 +93,11 @@ public class ShellConfigIT extends AccumuloClusterHarness {
Assert.fail("Unknown token type");
}
- assertTrue(Property.TABLE_VOLUME_CHOOSER.isExperimental());
assertTrue(Property.CRYPTO_CIPHER_ALGORITHM_NAME.isExperimental());
String configOutput = ts.exec("config");
- assertTrue(configOutput.contains(Property.TABLE_VOLUME_CHOOSER.getKey()));
+ assertTrue(configOutput.contains(PerTableVolumeChooser.TABLE_VOLUME_CHOOSER));
assertFalse(configOutput.contains(Property.CRYPTO_CIPHER_ALGORITHM_NAME.getKey()));
}
}
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeChooserFailureIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeChooserFailureIT.java
new file mode 100644
index 0000000..dc0cb41
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeChooserFailureIT.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.fs.PerTableVolumeChooser;
+import org.apache.accumulo.server.fs.PreferredVolumeChooser;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class VolumeChooserFailureIT extends ConfigurableMacBase {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ private File volDirBase;
+ private Path v1, v2, v3, v4;
+ private static String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ private String namespace1;
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ // Get 2 tablet servers
+ cfg.setNumTservers(2);
+ namespace1 = "ns_" + getUniqueNames(1)[0];
+
+ // Set the general volume chooser to the PerTableVolumeChooser so that different choosers can be specified
+ Map<String,String> siteConfig = new HashMap<>();
+ siteConfig.put(Property.GENERAL_VOLUME_CHOOSER.getKey(), PerTableVolumeChooser.class.getName());
+ // if a table doesn't have a volume chooser, use the preferred volume chooser
+ siteConfig.put(PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
+
+ // Set up 4 different volume paths
+ File baseDir = cfg.getDir();
+ volDirBase = new File(baseDir, "volumes");
+ File v1f = new File(volDirBase, "v1");
+ File v2f = new File(volDirBase, "v2");
+ File v3f = new File(volDirBase, "v3");
+ File v4f = new File(volDirBase, "v4");
+ v1 = new Path("file://" + v1f.getAbsolutePath());
+ v2 = new Path("file://" + v2f.getAbsolutePath());
+ v3 = new Path("file://" + v3f.getAbsolutePath());
+ v4 = new Path("file://" + v4f.getAbsolutePath());
+
+ cfg.setSiteConfig(siteConfig);
+
+ siteConfig.put(PerTableVolumeChooser.SCOPED_VOLUME_CHOOSER("logger"), PreferredVolumeChooser.class.getName());
+ // do not set preferred volumes
+ cfg.setSiteConfig(siteConfig);
+
+ // Only add volumes 1, 2, and 4 to the list of instance volumes to have one volume that isn't in the options list when they are choosing
+ cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString() + "," + v4.toString());
+ // no not set preferred volumes
+
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+
+ super.configure(cfg, hadoopCoreSite);
+
+ }
+
+ public static void addSplits(Connector connector, String tableName) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ // Add 10 splits to the table
+ SortedSet<Text> partitions = new TreeSet<>();
+ for (String s : rows)
+ partitions.add(new Text(s));
+ connector.tableOperations().addSplits(tableName, partitions);
+ }
+
+ // Test that uses one table with 10 split points each. It uses the PreferredVolumeChooser, but no preferred volume is specified.
+ // This means that the volume chooser will fail and no instance volumes will be assigned.
+ @Test
+ public void missingVolumePreferredVolumeChooser() throws Exception {
+ log.info("Starting missingVolumePreferredVolumeChooser");
+
+ // Create namespace
+ Connector connector = getConnector();
+ connector.namespaceOperations().create(namespace1);
+
+ // Set properties on the namespace
+ connector.namespaceOperations().setProperty(namespace1, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
+ // deliberately do not set preferred volumes
+
+ // Create table1 on namespace1 (will fail)
+ String tableName = namespace1 + ".1";
+ thrown.expect(AccumuloException.class);
+ connector.tableOperations().create(tableName);
+ }
+
+ // Test that uses one table with 10 split points each. It uses the PreferredVolumeChooser, but preferred volume is not an instance volume.
+ // This should fail.
+ @Test
+ public void notInstancePreferredVolumeChooser() throws Exception {
+ log.info("Starting notInstancePreferredVolumeChooser");
+
+ // Create namespace
+ Connector connector = getConnector();
+ connector.namespaceOperations().create(namespace1);
+
+ // Set properties on the namespace
+ String propertyName = PerTableVolumeChooser.TABLE_VOLUME_CHOOSER;
+ String volume = PreferredVolumeChooser.class.getName();
+ connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
+
+ // set to v3 which is not included in the list of instance volumes, so it should go to the
+ // system default preferred volumes
+ propertyName = PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES;
+ volume = v3.toString();
+ connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
+
+ // Create table1 on namespace1 (will fail)
+ String tableName = namespace1 + ".1";
+ thrown.expect(AccumuloException.class);
+ connector.tableOperations().create(tableName);
+ }
+
+}
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
index 4d940a8..b894681 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.util.ArrayList;
@@ -63,14 +64,16 @@ public class VolumeChooserIT extends ConfigurableMacBase {
private static final Text EMPTY = new Text();
private static final Value EMPTY_VALUE = new Value(new byte[] {});
private File volDirBase;
+ @SuppressWarnings("unused")
private Path v1, v2, v3, v4;
- private String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ private static String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
private String namespace1;
private String namespace2;
+ private String systemPreferredVolumes;
@Override
protected int defaultTimeoutSeconds() {
- return 30;
+ return 60;
}
@Override
@@ -83,7 +86,8 @@ public class VolumeChooserIT extends ConfigurableMacBase {
// Set the general volume chooser to the PerTableVolumeChooser so that different choosers can be specified
Map<String,String> siteConfig = new HashMap<>();
siteConfig.put(Property.GENERAL_VOLUME_CHOOSER.getKey(), PerTableVolumeChooser.class.getName());
- cfg.setSiteConfig(siteConfig);
+ // if a table doesn't have a volume chooser, use the preferred volume chooser
+ siteConfig.put(PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
// Set up 4 different volume paths
File baseDir = cfg.getDir();
@@ -97,6 +101,14 @@ public class VolumeChooserIT extends ConfigurableMacBase {
v3 = new Path("file://" + v3f.getAbsolutePath());
v4 = new Path("file://" + v4f.getAbsolutePath());
+ systemPreferredVolumes = v1.toString() + "," + v2.toString();
+ siteConfig.put(PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, systemPreferredVolumes); // exclude v4
+ cfg.setSiteConfig(siteConfig);
+
+ siteConfig.put(PerTableVolumeChooser.SCOPED_VOLUME_CHOOSER("logger"), PreferredVolumeChooser.class.getName());
+ siteConfig.put(PreferredVolumeChooser.SCOPED_PREFERRED_VOLUMES("logger"), v2.toString());
+ cfg.setSiteConfig(siteConfig);
+
// Only add volumes 1, 2, and 4 to the list of instance volumes to have one volume that isn't in the options list when they are choosing
cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString() + "," + v4.toString());
@@ -107,23 +119,16 @@ public class VolumeChooserIT extends ConfigurableMacBase {
}
- public void addSplits(Connector connector, String tableName) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ public static void addSplits(Connector connector, String tableName) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
// Add 10 splits to the table
SortedSet<Text> partitions = new TreeSet<>();
- for (String s : "b,e,g,j,l,o,q,t,v,y".split(","))
+ for (String s : rows)
partitions.add(new Text(s));
connector.tableOperations().addSplits(tableName, partitions);
}
- public void writeAndReadData(Connector connector, String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- // Write some data to the table
- BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
- for (String s : rows) {
- Mutation m = new Mutation(new Text(s));
- m.put(EMPTY, EMPTY, EMPTY_VALUE);
- bw.addMutation(m);
- }
- bw.close();
+ public static void writeAndReadData(Connector connector, String tableName) throws Exception {
+ writeDataToTable(connector, tableName);
// Write the data to disk, read it back
connector.tableOperations().flush(tableName, null, null, true);
@@ -134,12 +139,24 @@ public class VolumeChooserIT extends ConfigurableMacBase {
}
}
- public void verifyVolumes(Connector connector, String tableName, Range tableRange, String vol) throws TableNotFoundException {
+ public static void writeDataToTable(Connector connector, String tableName) throws Exception {
+ // Write some data to the table
+ BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
+ for (String s : rows) {
+ Mutation m = new Mutation(new Text(s));
+ m.put(EMPTY, EMPTY, EMPTY_VALUE);
+ bw.addMutation(m);
+ }
+ bw.close();
+ }
+
+ public static void verifyVolumes(Connector connector, String tableName, Range tableRange, String vol) throws Exception {
// Verify the new files are written to the Volumes specified
ArrayList<String> volumes = new ArrayList<>();
for (String s : vol.split(","))
volumes.add(s);
+ TreeSet<String> volumesSeen = new TreeSet<>();
Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
scanner.setRange(tableRange);
scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
@@ -147,35 +164,37 @@ public class VolumeChooserIT extends ConfigurableMacBase {
for (Entry<Key,Value> entry : scanner) {
boolean inVolume = false;
for (String volume : volumes) {
- if (entry.getKey().getColumnQualifier().toString().contains(volume))
+ if (entry.getKey().getColumnQualifier().toString().contains(volume)) {
+ volumesSeen.add(volume);
inVolume = true;
+ }
}
- assertTrue("Data not written to the correct volumes", inVolume);
+ assertTrue("Data not written to the correct volumes. " + entry.getKey().getColumnQualifier().toString(), inVolume);
fileCount++;
}
- assertEquals("Wrong number of files", 11, fileCount);
+ assertEquals("Did not see all the volumes. volumes: " + volumes.toString() + " volumes seen: " + volumesSeen.toString(), volumes.size(), volumesSeen.size());
+ assertEquals("Wrong number of files", 26, fileCount);
}
- // Test that uses two tables with 10 split points each. They each use the PreferredVolumeChooser to choose volumes.
- @Test
- public void twoTablesPreferredVolumeChooser() throws Exception {
- log.info("Starting twoTablesPreferredVolumeChooser");
-
- // Create namespace
- Connector connector = getConnector();
- connector.namespaceOperations().create(namespace1);
+ public static void verifyNoVolumes(Connector connector, String tableName, Range tableRange) throws Exception {
+ Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(tableRange);
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : scanner) {
+ fail("Data incorrectly written to " + entry.getKey().getColumnQualifier().toString());
+ }
+ }
+ private void configureNamespace(Connector connector, String volumeChooserClassName, String configuredVolumes, String namespace) throws Exception {
+ connector.namespaceOperations().create(namespace);
// Set properties on the namespace
- String propertyName = Property.TABLE_VOLUME_CHOOSER.getKey();
- String volume = PreferredVolumeChooser.class.getName();
- connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
+ connector.namespaceOperations().setProperty(namespace, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, volumeChooserClassName);
+ connector.namespaceOperations().setProperty(namespace, PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, configuredVolumes);
+ }
- propertyName = "table.custom.preferredVolumes";
- volume = v2.toString();
- connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
+ private void verifyVolumesForWritesToNewTable(Connector connector, String myNamespace, String expectedVolumes) throws Exception {
+ String tableName = myNamespace + ".1";
- // Create table1 on namespace1
- String tableName = namespace1 + ".1";
connector.tableOperations().create(tableName);
Table.ID tableID = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
@@ -184,94 +203,67 @@ public class VolumeChooserIT extends ConfigurableMacBase {
// Write some data to the table
writeAndReadData(connector, tableName);
// Verify the new files are written to the Volumes specified
- verifyVolumes(connector, tableName, TabletsSection.getRange(tableID), volume);
-
- connector.namespaceOperations().create(namespace2);
-
- // Set properties on the namespace
- propertyName = Property.TABLE_VOLUME_CHOOSER.getKey();
- volume = PreferredVolumeChooser.class.getName();
- connector.namespaceOperations().setProperty(namespace2, propertyName, volume);
-
- propertyName = "table.custom.preferredVolumes";
- volume = v1.toString();
- connector.namespaceOperations().setProperty(namespace2, propertyName, volume);
-
- // Create table2 on namespace2
- String tableName2 = namespace2 + ".1";
-
- connector.tableOperations().create(tableName2);
- Table.ID tableID2 = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName2));
+ verifyVolumes(connector, tableName, TabletsSection.getRange(tableID), expectedVolumes);
+ }
- // Add 10 splits to the table
- addSplits(connector, tableName2);
- // Write some data to the table
- writeAndReadData(connector, tableName2);
+ public static void verifyWaLogVolumes(Connector connector, Range tableRange, String vol) throws TableNotFoundException {
// Verify the new files are written to the Volumes specified
- verifyVolumes(connector, tableName2, TabletsSection.getRange(tableID2), volume);
+ ArrayList<String> volumes = new ArrayList<>();
+ for (String s : vol.split(","))
+ volumes.add(s);
+
+ TreeSet<String> volumesSeen = new TreeSet<>();
+ Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(tableRange);
+ scanner.fetchColumnFamily(TabletsSection.LogColumnFamily.NAME);
+ for (Entry<Key,Value> entry : scanner) {
+ boolean inVolume = false;
+ for (String volume : volumes) {
+ if (entry.getKey().getColumnQualifier().toString().contains(volume))
+ volumesSeen.add(volume);
+ inVolume = true;
+ }
+ assertTrue("Data not written to the correct volumes. " + entry.getKey().getColumnQualifier().toString(), inVolume);
+ }
}
- // Test that uses two tables with 10 split points each. They each use the RandomVolumeChooser to choose volumes.
+ // Test that uses two tables with 10 split points each. They each use the PreferredVolumeChooser to choose volumes.
@Test
- public void twoTablesRandomVolumeChooser() throws Exception {
- log.info("Starting twoTablesRandomVolumeChooser()");
+ public void twoTablesPreferredVolumeChooser() throws Exception {
+ log.info("Starting twoTablesPreferredVolumeChooser");
// Create namespace
Connector connector = getConnector();
connector.namespaceOperations().create(namespace1);
// Set properties on the namespace
- String propertyName = Property.TABLE_VOLUME_CHOOSER.getKey();
- String volume = RandomVolumeChooser.class.getName();
- connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
+ // namespace 1 -> v2
+ connector.namespaceOperations().setProperty(namespace1, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
+ connector.namespaceOperations().setProperty(namespace1, PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v2.toString());
// Create table1 on namespace1
- String tableName = namespace1 + ".1";
- connector.tableOperations().create(tableName);
- Table.ID tableID = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
-
- // Add 10 splits to the table
- addSplits(connector, tableName);
- // Write some data to the table
- writeAndReadData(connector, tableName);
- // Verify the new files are written to the Volumes specified
-
- verifyVolumes(connector, tableName, TabletsSection.getRange(tableID), v1.toString() + "," + v2.toString() + "," + v4.toString());
+ verifyVolumesForWritesToNewTable(connector, namespace1, v2.toString());
connector.namespaceOperations().create(namespace2);
-
// Set properties on the namespace
- propertyName = Property.TABLE_VOLUME_CHOOSER.getKey();
- volume = RandomVolumeChooser.class.getName();
- connector.namespaceOperations().setProperty(namespace2, propertyName, volume);
+ connector.namespaceOperations().setProperty(namespace2, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
+ connector.namespaceOperations().setProperty(namespace2, PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v1.toString());
// Create table2 on namespace2
- String tableName2 = namespace2 + ".1";
- connector.tableOperations().create(tableName2);
- Table.ID tableID2 = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName2));
-
- // / Add 10 splits to the table
- addSplits(connector, tableName2);
- // Write some data to the table
- writeAndReadData(connector, tableName2);
- // Verify the new files are written to the Volumes specified
- verifyVolumes(connector, tableName2, TabletsSection.getRange(tableID2), v1.toString() + "," + v2.toString() + "," + v4.toString());
+ verifyVolumesForWritesToNewTable(connector, namespace2, v1.toString());
}
- // Test that uses two tables with 10 split points each. The first uses the RandomVolumeChooser and the second uses the
- // StaticVolumeChooser to choose volumes.
+ // Test that uses two tables with 10 split points each. They each use the RandomVolumeChooser to choose volumes.
@Test
- public void twoTablesDiffChoosers() throws Exception {
- log.info("Starting twoTablesDiffChoosers");
+ public void twoTablesRandomVolumeChooser() throws Exception {
+ log.info("Starting twoTablesRandomVolumeChooser()");
// Create namespace
Connector connector = getConnector();
connector.namespaceOperations().create(namespace1);
// Set properties on the namespace
- String propertyName = Property.TABLE_VOLUME_CHOOSER.getKey();
- String volume = RandomVolumeChooser.class.getName();
- connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
+ connector.namespaceOperations().setProperty(namespace1, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
// Create table1 on namespace1
String tableName = namespace1 + ".1";
@@ -289,105 +281,67 @@ public class VolumeChooserIT extends ConfigurableMacBase {
connector.namespaceOperations().create(namespace2);
// Set properties on the namespace
- propertyName = Property.TABLE_VOLUME_CHOOSER.getKey();
- volume = PreferredVolumeChooser.class.getName();
- connector.namespaceOperations().setProperty(namespace2, propertyName, volume);
-
- propertyName = "table.custom.preferredVolumes";
- volume = v1.toString();
- connector.namespaceOperations().setProperty(namespace2, propertyName, volume);
+ connector.namespaceOperations().setProperty(namespace2, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
// Create table2 on namespace2
String tableName2 = namespace2 + ".1";
connector.tableOperations().create(tableName2);
Table.ID tableID2 = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName2));
- // Add 10 splits to the table
+ // / Add 10 splits to the table
addSplits(connector, tableName2);
// Write some data to the table
writeAndReadData(connector, tableName2);
// Verify the new files are written to the Volumes specified
- verifyVolumes(connector, tableName2, TabletsSection.getRange(tableID2), volume);
+ verifyVolumes(connector, tableName2, TabletsSection.getRange(tableID2), v1.toString() + "," + v2.toString() + "," + v4.toString());
}
- // Test that uses one table with 10 split points each. It uses the StaticVolumeChooser, but no preferred volume is specified. This means that the volume
- // is chosen randomly from all instance volumes.
+ // Test that uses two tables with 10 split points each. The first uses the RandomVolumeChooser and the second uses the
+ // StaticVolumeChooser to choose volumes.
@Test
- public void missingVolumePreferredVolumeChooser() throws Exception {
- log.info("Starting missingVolumePreferredVolumeChooser");
+ public void twoTablesDiffChoosers() throws Exception {
+ log.info("Starting twoTablesDiffChoosers");
// Create namespace
Connector connector = getConnector();
connector.namespaceOperations().create(namespace1);
// Set properties on the namespace
- String propertyName = Property.TABLE_VOLUME_CHOOSER.getKey();
- String volume = PreferredVolumeChooser.class.getName();
- connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
+ connector.namespaceOperations().setProperty(namespace1, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, RandomVolumeChooser.class.getName());
// Create table1 on namespace1
- String tableName = namespace1 + ".1";
- connector.tableOperations().create(tableName);
- Table.ID tableID = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
+ verifyVolumesForWritesToNewTable(connector, namespace1, v1.toString() + "," + v2.toString() + "," + v4.toString());
+ connector.namespaceOperations().create(namespace2);
- // Add 10 splits to the table
- addSplits(connector, tableName);
- // Write some data to the table
- writeAndReadData(connector, tableName);
- // Verify the new files are written to the Volumes specified
- verifyVolumes(connector, tableName, TabletsSection.getRange(tableID), v1.toString() + "," + v2.toString() + "," + v4.toString());
+ connector.namespaceOperations().setProperty(namespace2, PerTableVolumeChooser.TABLE_VOLUME_CHOOSER, PreferredVolumeChooser.class.getName());
+ connector.namespaceOperations().setProperty(namespace2, PreferredVolumeChooser.TABLE_PREFERRED_VOLUMES, v1.toString());
+
+ // Create table2 on namespace2
+ verifyVolumesForWritesToNewTable(connector, namespace2, v1.toString());
}
- // Test that uses one table with 10 split points each. It uses the PreferredVolumeChooser, but preferred volume is not an instance volume. This means that the
- // volume is chosen randomly from all instance volumes
@Test
- public void notInstancePreferredVolumeChooser() throws Exception {
- log.info("Starting notInstancePreferredVolumeChooser");
-
- // Create namespace
+ public void includeSpecialVolumeForTable() throws Exception {
+ log.info("Starting includeSpecialVolumeForTable");
Connector connector = getConnector();
- connector.namespaceOperations().create(namespace1);
- // Set properties on the namespace
- String propertyName = Property.TABLE_VOLUME_CHOOSER.getKey();
- String volume = PreferredVolumeChooser.class.getName();
- connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
-
- propertyName = "table.custom.preferredVolumes";
- volume = v3.toString();
- connector.namespaceOperations().setProperty(namespace1, propertyName, volume);
-
- // Create table1 on namespace1
- String tableName = namespace1 + ".1";
- connector.tableOperations().create(tableName);
- Table.ID tableID = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
-
- // Add 10 splits to the table
- addSplits(connector, tableName);
- // Write some data to the table
- writeAndReadData(connector, tableName);
- // Verify the new files are written to the Volumes specified
- verifyVolumes(connector, tableName, TabletsSection.getRange(tableID), v1.toString() + "," + v2.toString() + "," + v4.toString());
+ // the following table will be configured to go to the excluded volume
+ String configuredVolumes = v4.toString();
+ configureNamespace(connector, PreferredVolumeChooser.class.getName(), configuredVolumes, namespace2);
+ verifyVolumesForWritesToNewTable(connector, namespace2, configuredVolumes);
}
- // Test that uses one table with 10 split points each. It does not specify a specific chooser, so the volume is chosen randomly from all instance volumes.
@Test
- public void chooserNotSpecified() throws Exception {
- log.info("Starting chooserNotSpecified");
+ public void waLogsSentToConfiguredVolumes() throws Exception {
+ log.info("Starting waLogsSentToConfiguredVolumes");
- // Create a table
Connector connector = getConnector();
- String tableName = getUniqueNames(2)[0];
+ String tableName = "anotherTable";
connector.tableOperations().create(tableName);
- Table.ID tableID = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
- // Add 10 splits to the table
- addSplits(connector, tableName);
- // Write some data to the table
- writeAndReadData(connector, tableName);
-
- // Verify the new files are written to the Volumes specified
- verifyVolumes(connector, tableName, TabletsSection.getRange(tableID), v1.toString() + "," + v2.toString() + "," + v4.toString());
+ VolumeChooserIT.addSplits(connector, tableName);
+ VolumeChooserIT.writeDataToTable(connector, tableName);
+ // should only go to v2 as per configuration in configure()
+ VolumeChooserIT.verifyWaLogVolumes(connector, new Range(), v2.toString());
}
-
}
--
To stop receiving notification emails like this one, please contact
"commits@accumulo.apache.org" <co...@accumulo.apache.org>.