You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/11/01 20:27:20 UTC
[accumulo] branch master updated: ACCUMULO-4706 Updates to support
Accumulo docker image (#316)
This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new cd5b73b ACCUMULO-4706 Updates to support Accumulo docker image (#316)
cd5b73b is described below
commit cd5b73b6def3be480934e6fc33c1895eed4367b6
Author: Mike Walch <mw...@apache.org>
AuthorDate: Wed Nov 1 16:27:18 2017 -0400
ACCUMULO-4706 Updates to support Accumulo docker image (#316)
* Added -o option to Accumulo services to override configuration
* Added --upload-accumulo-site option to 'accumulo init' command
* Tserver replication service now supports port search
---
.../accumulo/core/conf/CliConfiguration.java | 63 ++++++++++++++++++++++
.../accumulo/core/conf/SiteConfiguration.java | 19 +++++--
.../accumulo/core/conf/CliConfigurationTest.java | 50 +++++++++++++++++
.../accumulo/core/conf/SiteConfigurationTest.java | 11 ++++
.../org/apache/accumulo/server/ServerOpts.java | 53 +++++++++++++++++-
.../apache/accumulo/server/init/Initialize.java | 26 +++++++++
.../org/apache/accumulo/server/ServerOptsTest.java | 11 ++++
.../apache/accumulo/gc/SimpleGarbageCollector.java | 6 +--
.../java/org/apache/accumulo/master/Master.java | 3 +-
.../java/org/apache/accumulo/monitor/Monitor.java | 3 +-
.../org/apache/accumulo/tracer/TraceServer.java | 2 +-
.../org/apache/accumulo/tserver/TabletServer.java | 5 +-
12 files changed, 237 insertions(+), 15 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/CliConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/CliConfiguration.java
new file mode 100644
index 0000000..7be1053
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/conf/CliConfiguration.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.conf;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Predicate;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CliConfiguration {
+
+ private static final Logger log = LoggerFactory.getLogger(CliConfiguration.class);
+ private static volatile Map<String,String> config = new HashMap<>();
+
+ /**
+ * Sets CliConfiguration with map of configuration. Additional calls will overwrite existing properties and values.
+ *
+ * @param conf
+ * Map of configuration
+ */
+ public static void set(Map<String,String> conf) {
+ Objects.requireNonNull(conf);
+ config = conf;
+ }
+
+ public static void print() {
+ log.info("The following configuration was set on the command line:");
+ for (Map.Entry<String,String> entry : config.entrySet()) {
+ String key = entry.getKey();
+ log.info(key + " = " + (Property.isSensitive(key) ? "<hidden>" : entry.getValue()));
+ }
+ }
+
+ public static String get(Property property) {
+ return config.get(property.getKey());
+ }
+
+ public static void getProperties(Map<String,String> props, Predicate<String> filter) {
+ for (Map.Entry<String,String> entry : config.entrySet()) {
+ if (filter.test(entry.getKey())) {
+ props.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
index cd35ad8..4135de0 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
@@ -34,8 +34,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * An {@link AccumuloConfiguration} which loads properties from an XML file, usually accumulo-site.xml. This implementation supports defaulting undefined
- * property values to a parent configuration's definitions.
+ * An {@link AccumuloConfiguration} which first loads any properties set on the command-line (using the -o option) and then from an XML file, usually
+ * accumulo-site.xml. This implementation supports defaulting undefined property values to a parent configuration's definitions.
* <p>
* The system property "accumulo.configuration" can be used to specify the location of the XML configuration file on the classpath or filesystem if the path is
* prefixed with 'file://'. If the system property is not defined, it defaults to "accumulo-site.xml" and will look on classpath for file.
@@ -119,8 +119,11 @@ public class SiteConfiguration extends AccumuloConfiguration {
@Override
public String get(Property property) {
- String key = property.getKey();
+ if (CliConfiguration.get(property) != null) {
+ return CliConfiguration.get(property);
+ }
+ String key = property.getKey();
// If the property is sensitive, see if CredentialProvider was configured.
if (property.isSensitive()) {
Configuration hadoopConf = getHadoopConfiguration();
@@ -145,12 +148,19 @@ public class SiteConfiguration extends AccumuloConfiguration {
log.error("Using default value for {} due to improperly formatted {}: {}", key, property.getType(), value);
value = parent.get(property);
}
+
return value;
}
@Override
public void getProperties(Map<String,String> props, Predicate<String> filter) {
- parent.getProperties(props, filter);
+ getProperties(props, filter, true);
+ }
+
+ public void getProperties(Map<String,String> props, Predicate<String> filter, boolean useDefaults) {
+ if (useDefaults) {
+ parent.getProperties(props, filter);
+ }
for (Entry<String,String> entry : getXmlConfig())
if (filter.test(entry.getKey()))
@@ -176,6 +186,7 @@ public class SiteConfiguration extends AccumuloConfiguration {
log.warn("Failed to extract sensitive properties from Hadoop CredentialProvider, falling back to accumulo-site.xml", e);
}
}
+ CliConfiguration.getProperties(props, filter);
}
protected Configuration getHadoopConfiguration() {
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/CliConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/CliConfigurationTest.java
new file mode 100644
index 0000000..e7f76dc
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/conf/CliConfigurationTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.conf;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+
+public class CliConfigurationTest {
+
+ @Test
+ public void testBasic() {
+ try {
+ CliConfiguration.set(null);
+ Assert.fail();
+ } catch (NullPointerException e) {
+ // expected
+ }
+ CliConfiguration.set(new HashMap<>());
+
+ Map<String,String> expected = new HashMap<>();
+ expected.put(Property.TRACE_USER.getKey(), "test");
+ expected.put(Property.TSERV_CLIENTPORT.getKey(), "123");
+ CliConfiguration.set(expected);
+
+ Assert.assertEquals("test", CliConfiguration.get(Property.TRACE_USER));
+
+ Map<String,String> results = new HashMap<>();
+ CliConfiguration.getProperties(results, p -> p.startsWith("trace"));
+ Assert.assertEquals(ImmutableMap.of(Property.TRACE_USER.getKey(), "test"), results);
+ }
+}
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
index 92a0da4..ee51d28 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
@@ -28,6 +28,8 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import com.google.common.collect.ImmutableMap;
+
public class SiteConfigurationTest {
private static boolean isCredentialProviderAvailable;
@@ -71,4 +73,13 @@ public class SiteConfigurationTest {
Assert.assertEquals(null, props.get("ignored.property"));
Assert.assertEquals(Property.GENERAL_RPC_TIMEOUT.getDefaultValue(), props.get(Property.GENERAL_RPC_TIMEOUT.getKey()));
}
+
+ @Test
+ public void testCliConfig() {
+ SiteConfiguration conf = SiteConfiguration.getInstance();
+ Assert.assertEquals("localhost:2181", conf.get(Property.INSTANCE_ZK_HOST));
+
+ CliConfiguration.set(ImmutableMap.of(Property.INSTANCE_ZK_HOST.getKey(), "myhost:2181"));
+ Assert.assertEquals("myhost:2181", conf.get(Property.INSTANCE_ZK_HOST));
+ }
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerOpts.java b/server/base/src/main/java/org/apache/accumulo/server/ServerOpts.java
index bbe0dd2..b173ae3 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerOpts.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerOpts.java
@@ -16,17 +16,68 @@
*/
package org.apache.accumulo.server;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.conf.CliConfiguration;
import com.beust.jcommander.Parameter;
+import com.beust.jcommander.converters.IParameterSplitter;
public class ServerOpts extends Help {
@Parameter(names = {"-a", "--address"}, description = "address to bind to")
- String address = null;
+ private String address = null;
+
+ public static class NullSplitter implements IParameterSplitter {
+ @Override
+ public List<String> split(String value) {
+ return Collections.singletonList(value);
+ }
+ }
+
+ @Parameter(names = "-o", splitter = NullSplitter.class,
+ description = "Overrides configuration set in accumulo-site.xml (but NOT system-wide config set in Zookeeper). Expected format: -o <key>=<value>")
+ private List<String> properties = new ArrayList<>();
public String getAddress() {
if (address != null)
return address;
return "0.0.0.0";
}
+
+ public List<String> getProperties() {
+ return properties;
+ }
+
+ public Map<String,String> getConfig() {
+ Map<String,String> config = new HashMap<>();
+ for (String prop : getProperties()) {
+ String[] propArgs = prop.split("=", 2);
+ if (propArgs.length == 2) {
+ String key = propArgs[0].trim();
+ String value = propArgs[1].trim();
+ if (key.isEmpty() || value.isEmpty()) {
+ throw new IllegalArgumentException("Invalid command line -o option: " + prop);
+ } else {
+ config.put(key, value);
+ }
+ } else {
+ throw new IllegalArgumentException("Invalid command line -o option: " + prop);
+ }
+ }
+ return config;
+ }
+
+ @Override
+ public void parseArgs(String programName, String[] args, Object... others) {
+ super.parseArgs(programName, args, others);
+ CliConfiguration.set(getConfig());
+ if (getConfig().size() > 0) {
+ CliConfiguration.print();
+ }
+ }
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index be3869b..8198418 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -29,6 +29,7 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
@@ -100,6 +101,7 @@ import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.tables.TableManager;
import org.apache.accumulo.server.tablets.TabletTime;
import org.apache.accumulo.server.util.ReplicationTableUtil;
+import org.apache.accumulo.server.util.SystemPropUtil;
import org.apache.accumulo.server.util.TablePropUtil;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.start.spi.KeywordExecutable;
@@ -393,6 +395,28 @@ public class Initialize implements KeywordExecutable {
log.error("FATAL: Failed to initialize security", e);
return false;
}
+
+ if (opts.uploadAccumuloSite) {
+ try {
+ log.info("Uploading properties in accumulo-site.xml to Zookeeper. Properties that cannot be set in Zookeeper will be skipped:");
+ Map<String,String> entries = new TreeMap<>();
+ SiteConfiguration.getInstance().getProperties(entries, x -> true, false);
+ for (Map.Entry<String,String> entry : entries.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ if (Property.isValidZooPropertyKey(key)) {
+ SystemPropUtil.setSystemProperty(key, value);
+ log.info("Uploaded - {} = {}", key, Property.isSensitive(key) ? "<hidden>" : value);
+ } else {
+ log.info("Skipped - {} = {}", key, Property.isSensitive(key) ? "<hidden>" : value);
+ }
+ }
+ } catch (Exception e) {
+ log.error("FATAL: Failed to upload accumulo-site.xml to Zookeeper", e);
+ return false;
+ }
+ }
+
return true;
}
@@ -760,6 +784,8 @@ public class Initialize implements KeywordExecutable {
boolean forceResetSecurity = false;
@Parameter(names = "--clear-instance-name", description = "delete any existing instance name without prompting")
boolean clearInstanceName = false;
+ @Parameter(names = "--upload-accumulo-site", description = "Uploads properties in accumulo-site.xml to Zookeeper")
+ boolean uploadAccumuloSite = false;
@Parameter(names = "--instance-name", description = "the instance name, if not provided, will prompt")
String cliInstanceName;
@Parameter(names = "--password", description = "set the password on the command line")
diff --git a/server/base/src/test/java/org/apache/accumulo/server/ServerOptsTest.java b/server/base/src/test/java/org/apache/accumulo/server/ServerOptsTest.java
index 2c6889b..fbd8e14 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/ServerOptsTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/ServerOptsTest.java
@@ -18,6 +18,9 @@ package org.apache.accumulo.server;
import static org.junit.Assert.assertEquals;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -40,4 +43,12 @@ public class ServerOptsTest {
opts.parseArgs(ServerOptsTest.class.getName(), new String[] {});
assertEquals("0.0.0.0", opts.getAddress());
}
+
+ @Test
+ public void testOverrideConfig() {
+ SiteConfiguration siteConf = SiteConfiguration.getInstance();
+ Assert.assertEquals("localhost:2181", siteConf.get(Property.INSTANCE_ZK_HOST));
+ opts.parseArgs(ServerOptsTest.class.getName(), new String[] {"-o", "instance.zookeeper.host=test:123"});
+ Assert.assertEquals("test:123", siteConf.get(Property.INSTANCE_ZK_HOST));
+ }
}
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index ba23505..74ef6c9 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -144,8 +144,10 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements Ifa
private GCStatus status = new GCStatus(new GcCycleStats(), new GcCycleStats(), new GcCycleStats(), new GcCycleStats());
- public static void main(String[] args) throws UnknownHostException, IOException {
+ public static void main(String[] args) throws IOException {
final String app = "gc";
+ Opts opts = new Opts();
+ opts.parseArgs(app, args);
SecurityUtil.serverLogin(SiteConfiguration.getInstance());
Instance instance = HdfsZooInstance.getInstance();
ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
@@ -153,8 +155,6 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements Ifa
log.info("Instance " + instance.getInstanceID());
final VolumeManager fs = VolumeManagerImpl.get();
Accumulo.init(fs, instance, conf, app);
- Opts opts = new Opts();
- opts.parseArgs(app, args);
SimpleGarbageCollector gc = new SimpleGarbageCollector(opts, instance, fs, conf);
DistributedTrace.enable(opts.getAddress(), app, conf.getSystemConfiguration());
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 8660751..1e72e4b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -1438,10 +1438,9 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
public static void main(String[] args) throws Exception {
try {
final String app = "master";
- SecurityUtil.serverLogin(SiteConfiguration.getInstance());
-
ServerOpts opts = new ServerOpts();
opts.parseArgs(app, args);
+ SecurityUtil.serverLogin(SiteConfiguration.getInstance());
String hostname = opts.getAddress();
Instance instance = HdfsZooInstance.getInstance();
ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index b1d6db4..68317b3 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -418,11 +418,10 @@ public class Monitor implements HighlyAvailableService {
public static void main(String[] args) throws Exception {
final String app = "monitor";
- SecurityUtil.serverLogin(SiteConfiguration.getInstance());
-
ServerOpts opts = new ServerOpts();
opts.parseArgs(app, args);
String hostname = opts.getAddress();
+ SecurityUtil.serverLogin(SiteConfiguration.getInstance());
VolumeManager fs = VolumeManagerImpl.get();
instance = HdfsZooInstance.getInstance();
diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index c3e6d5f..16d72db 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -379,9 +379,9 @@ public class TraceServer implements Watcher {
public static void main(String[] args) throws Exception {
final String app = "tracer";
- loginTracer(SiteConfiguration.getInstance());
ServerOpts opts = new ServerOpts();
opts.parseArgs(app, args);
+ loginTracer(SiteConfiguration.getInstance());
Instance instance = HdfsZooInstance.getInstance();
ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
VolumeManager fs = VolumeManagerImpl.get();
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 5e24e59..6bdcfd0 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -2441,7 +2441,8 @@ public class TabletServer extends AccumuloServerContext implements Runnable {
AccumuloConfiguration conf = getServerConfigurationFactory().getSystemConfiguration();
Property maxMessageSizeProperty = (conf.get(Property.TSERV_MAX_MESSAGE_SIZE) != null ? Property.TSERV_MAX_MESSAGE_SIZE : Property.GENERAL_MAX_MESSAGE_SIZE);
ServerAddress sp = TServerUtils.startServer(this, clientAddress.getHost(), Property.REPLICATION_RECEIPT_SERVICE_PORT, processor,
- "ReplicationServicerHandler", "Replication Servicer", null, Property.REPLICATION_MIN_THREADS, Property.REPLICATION_THREADCHECK, maxMessageSizeProperty);
+ "ReplicationServicerHandler", "Replication Servicer", Property.TSERV_PORTSEARCH, Property.REPLICATION_MIN_THREADS, Property.REPLICATION_THREADCHECK,
+ maxMessageSizeProperty);
this.replServer = sp.server;
log.info("Started replication service on {}", sp.address);
@@ -3061,9 +3062,9 @@ public class TabletServer extends AccumuloServerContext implements Runnable {
public static void main(String[] args) throws IOException {
try {
final String app = "tserver";
- SecurityUtil.serverLogin(SiteConfiguration.getInstance());
ServerOpts opts = new ServerOpts();
opts.parseArgs(app, args);
+ SecurityUtil.serverLogin(SiteConfiguration.getInstance());
String hostname = opts.getAddress();
Instance instance = HdfsZooInstance.getInstance();
ServerConfigurationFactory conf = new ServerConfigurationFactory(instance);
--
To stop receiving notification emails like this one, please contact
['"commits@accumulo.apache.org" <co...@accumulo.apache.org>'].