You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/07 21:10:28 UTC
[28/76] [abbrv] hadoop git commit: YARN-5461. Initial code ported
from slider-core module. (jianhe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
new file mode 100644
index 0000000..5140059
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ComponentArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * This is a listing of the roles to create
+ */
+ @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT, ARG_ROLE},
+ arity = 2,
+ description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
+ splitter = DontSplitArguments.class)
+ public List<String> componentTuples = new ArrayList<>(0);
+
+
+ /**
+ * Get the role mapping (may be empty, but never null)
+ * @return role mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
+ return convertTupleListToMap("component", componentTuples);
+ }
+
+ public List<String> getComponentTuples() {
+ return componentTuples;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
new file mode 100644
index 0000000..0344305
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+ @Override
+ public List<String> split(String value) {
+ List<String> list = new ArrayList<>(1);
+ list.add(value);
+ return list;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
new file mode 100644
index 0000000..7524053
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import java.io.File;
+
+/**
+ * Launch args for create and start and anything else that can start something
+ */
+public interface LaunchArgsAccessor extends WaitTimeAccessor {
+ String getRmAddress();
+
+ File getOutputFile();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
new file mode 100644
index 0000000..bc7e94c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+import java.io.File;
+
+/**
+ * Any launch-time args
+ */
+public class LaunchArgsDelegate extends WaitArgsDelegate implements
+ LaunchArgsAccessor {
+
+
+ //TODO: do we need this?
+ @Parameter(names = ARG_RESOURCE_MANAGER,
+ description = "Resource manager hostname:port ",
+ required = false)
+ private String rmAddress;
+
+ @Override
+ public String getRmAddress() {
+ return rmAddress;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File outputFile;
+
+ @Override
+ public File getOutputFile() {
+ return outputFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
new file mode 100644
index 0000000..ccb526c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+import org.apache.hadoop.fs.Path;
+
+public class PathArgumentConverter extends BaseConverter<Path> {
+
+ public PathArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public Path convert(String value) {
+ return new Path(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
new file mode 100644
index 0000000..f9516d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.slider.common.params;
+
+/**
+ * Parameters sent by the Client to the AM
+ */
+public class SliderAMArgs extends CommonArgs {
+
+ SliderAMCreateAction createAction = new SliderAMCreateAction();
+
+ public SliderAMArgs(String[] args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+ addActions(createAction);
+ }
+
+ public String getImage() {
+ return createAction.image;
+ }
+
+ /**
+ * This is the URI in the FS to the Slider cluster; the conf file (and any
+ * other cluster-specifics) can be picked up here
+ */
+ public String getSliderClusterURI() {
+ return createAction.sliderClusterURI;
+ }
+
+ /**
+ * Am binding is simple: there is only one action
+ */
+ @Override
+ public void applyAction() {
+ bindCoreAction(createAction);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
new file mode 100644
index 0000000..197c22b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+import java.io.File;
+
+
+@Parameters(commandNames = {SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class SliderAMCreateAction extends AbstractActionArgs implements
+ LaunchArgsAccessor {
+
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+
+ @Parameter(names = ARG_IMAGE, description = "image", required = false)
+ public String image;
+
+ /**
+ * This is the URI in the FS to the Slider cluster; the conf file (and any
+ * other cluster-specifics) can be picked up here
+ */
+ @Parameter(names = ARG_CLUSTER_URI,
+ description = "URI to the Slider cluster", required = true)
+ public String sliderClusterURI;
+
+ @ParametersDelegate
+ LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
new file mode 100644
index 0000000..204ad9a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+/**
+ * Actions.
+ * Only some of these are supported by specific Slider Services; they
+ * are listed here to ensure the names are consistent
+ */
+public interface SliderActions {
+ String ACTION_AM_SUICIDE = "am-suicide";
+ String ACTION_BUILD = "build";
+ String ACTION_CLIENT = "client";
+ String ACTION_CREATE = "create";
+ String ACTION_DIAGNOSTICS = "diagnostics";
+ String ACTION_DEPENDENCY = "dependency";
+ String ACTION_UPDATE = "update";
+ String ACTION_UPGRADE = "upgrade";
+ String ACTION_DESTROY = "destroy";
+ String ACTION_ECHO = "echo";
+ String ACTION_EXISTS = "exists";
+ String ACTION_FLEX = "flex";
+ String ACTION_FREEZE = "stop";
+ String ACTION_HELP = "help";
+ String ACTION_INSTALL_KEYTAB = "install-keytab";
+ String ACTION_INSTALL_PACKAGE = "install-package";
+ String ACTION_KDIAG = "kdiag";
+ String ACTION_KEYTAB = "keytab";
+ String ACTION_KILL_CONTAINER = "kill-container";
+ String ACTION_LIST = "list";
+ String ACTION_LOOKUP = "lookup";
+ String ACTION_NODES = "nodes";
+ String ACTION_PACKAGE = "package";
+ String ACTION_PREFLIGHT = "preflight";
+ String ACTION_RECONFIGURE = "reconfigure";
+ String ACTION_REGISTRY = "registry";
+ String ACTION_RESOLVE = "resolve";
+ String ACTION_RESOURCE = "resource";
+ String ACTION_STATUS = "status";
+ String ACTION_THAW = "start";
+ String ACTION_TOKENS = "tokens";
+
+ String ACTION_VERSION = "version";
+ String DESCRIBE_ACTION_AM_SUICIDE =
+ "Tell the Slider Application Master to simulate a process failure by terminating itself";
+ String DESCRIBE_ACTION_BUILD =
+ "Build a Slider cluster specification, but do not start it";
+ String DESCRIBE_ACTION_CREATE =
+ "Create a live Slider application";
+ String DESCRIBE_ACTION_DEPENDENCY =
+ "Slider AM and agent dependency (libraries) management";
+ String DESCRIBE_ACTION_UPDATE =
+ "Update template for a Slider application";
+ String DESCRIBE_ACTION_UPGRADE =
+ "Rolling upgrade/downgrade the application to a newer/previous version";
+ String DESCRIBE_ACTION_DESTROY =
+ "Destroy a stopped Slider application";
+ String DESCRIBE_ACTION_EXISTS =
+ "Probe for an application running";
+ String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
+ String DESCRIBE_ACTION_FREEZE =
+ "Stop a running application";
+ String DESCRIBE_ACTION_GETCONF =
+ "Get the configuration of an application";
+ String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
+ String DESCRIBE_ACTION_KILL_CONTAINER =
+ "Kill a container in the application";
+ String DESCRIBE_ACTION_HELP = "Print help information";
+ String DESCRIBE_ACTION_LIST =
+ "List running Slider applications";
+ String DESCRIBE_ACTION_LOOKUP =
+ "look up a YARN application";
+ String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application";
+ String DESCRIBE_ACTION_MONITOR =
+ "Monitor a running application";
+ String DESCRIBE_ACTION_REGISTRY =
+ "Query the registry of a YARN application";
+ String DESCRIBE_ACTION_RESOLVE =
+ "Resolve or list records in the YARN registry";
+ String DESCRIBE_ACTION_STATUS =
+ "Get the status of an application";
+ String DESCRIBE_ACTION_THAW =
+ "Start a stopped application";
+ String DESCRIBE_ACTION_VERSION =
+ "Print the Slider version information";
+ String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install application package." +
+ " Deprecated, use '" + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.";
+ String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use the packages";
+ String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
+ String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab." +
+ " Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'.";
+ String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
+ String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
+ String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
+
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
new file mode 100644
index 0000000..b0d1ebf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class URIArgumentConverter extends BaseConverter<URI> {
+
+ public URIArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public URI convert(String value) {
+ try {
+ return new URI(value);
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("Cannot make a URI from " + value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
new file mode 100644
index 0000000..8894309
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class URLArgumentConverter extends BaseConverter<URL> {
+ public URLArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public URL convert(String value) {
+ try {
+ return new URL(value);
+ } catch (MalformedURLException e) {
+ throw new RuntimeException("Cannot make a URL from " + value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
new file mode 100644
index 0000000..1c27c01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+public class WaitArgsDelegate extends AbstractArgsDelegate implements
+ WaitTimeAccessor {
+
+
+ //--wait [timeout]
+ @Parameter(names = {ARG_WAIT},
+ description = "time to wait for an action to complete")
+ public int waittime = 0;
+
+
+ @Override
+ public int getWaittime() {
+ return waittime;
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ this.waittime = waittime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
new file mode 100644
index 0000000..13d4d5a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+public interface WaitTimeAccessor {
+ int getWaittime();
+ void setWaittime(int waittime);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
new file mode 100644
index 0000000..6380d0c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Some general comparators
+ */
+public class Comparators {
+
+ public static class LongComparator implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ long result = o1 - o2;
+ // need to comparisons with a diff greater than integer size
+ if (result < 0 ) {
+ return -1;
+ } else if (result > 0) {
+ return 1;
+ }
+ return 0;
+ }
+ }
+public static class InvertedLongComparator implements Comparator<Long>, Serializable {
+ private static final LongComparator inner = new LongComparator();
+ @Override
+ public int compare(Long o1, Long o2) {
+ return -inner.compare(o1, o2);
+ }
+ }
+
+
+ /**
+ * Little template class to reverse any comparitor
+ * @param <CompareType> the type that is being compared
+ */
+ public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
+ Serializable {
+
+ final Comparator<CompareType> instance;
+
+ public ComparatorReverser(Comparator<CompareType> instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public int compare(CompareType first, CompareType second) {
+ return instance.compare(second, first);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
new file mode 100644
index 0000000..9db241d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -0,0 +1,658 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Methods to aid in config, both in the Configuration class and
+ * with other parts of setting up Slider-initated processes.
+ *
+ * Some of the methods take an argument of a map iterable for their sources; this allows
+ * the same method
+ */
+public class ConfigHelper {
+ private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
+
+ private static AtomicBoolean sliderResourceInjected =
+ new AtomicBoolean(false);
+ private static AtomicBoolean sliderResourceInjectionAttempted =
+ new AtomicBoolean(false);
+
+ /**
+ * Dump the (sorted) configuration
+ * @param conf config
+ * @return the sorted keyset
+ */
+ public static Set<String> dumpConf(Configuration conf) {
+ Set<String> keys = sortedConfigKeys(conf);
+ for (String key : keys) {
+ log.info("{}={}", key, conf.get(key));
+ }
+ return keys;
+ }
+
+ /**
+ * Take a configuration and return a sorted set
+ * @param conf config
+ * @return the sorted keyset
+
+ */
+ public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
+ TreeSet<String> sorted = new TreeSet<String>();
+ for (Map.Entry<String, String> entry : conf) {
+ sorted.add(entry.getKey());
+ }
+ return sorted;
+ }
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Map<String, String> map,
+ String origin) throws BadConfigException {
+ addConfigMap(config, map.entrySet(), origin);
+ }
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Iterable<Map.Entry<String, String>> map,
+ String origin) throws BadConfigException {
+ for (Map.Entry<String, String> mapEntry : map) {
+ String key = mapEntry.getKey();
+ String value = mapEntry.getValue();
+ if (value == null) {
+ throw new BadConfigException("Null value for property " + key);
+ }
+ config.set(key, value, origin);
+ }
+ }
+
+
+ /**
+ * Save a config file in a destination directory on a given filesystem
+ * @param systemConf system conf used for creating filesystems
+ * @param confToSave config to save
+ * @param confdir the directory path where the file is to go
+ * @param filename the filename
+ * @return the destination path where the file was saved
+ * @throws IOException IO problems
+ */
+ public static Path saveConfig(Configuration systemConf,
+ Configuration confToSave,
+ Path confdir,
+ String filename) throws IOException {
+ FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
+ Path destPath = new Path(confdir, filename);
+ saveConfig(fs, destPath, confToSave);
+ return destPath;
+ }
+
+ /**
+ * Save a config
+ * @param fs filesystem
+ * @param destPath dest to save
+ * @param confToSave config to save
+ * @throws IOException IO problems
+ */
+ public static void saveConfig(FileSystem fs,
+ Path destPath,
+ Configuration confToSave) throws
+ IOException {
+ FSDataOutputStream fos = fs.create(destPath);
+ try {
+ confToSave.writeXml(fos);
+ } finally {
+ IOUtils.closeStream(fos);
+ }
+ }
+
+ /**
+ * Convert to an XML string
+ * @param conf configuration
+ * @return conf
+ * @throws IOException
+ */
+ public static String toXml(Configuration conf) throws IOException {
+ StringWriter writer = new StringWriter();
+ conf.writeXml(writer);
+ return writer.toString();
+ }
+
+ /**
+ * This will load and parse a configuration to an XML document
+ * @param fs filesystem
+ * @param path path
+ * @return an XML document
+ * @throws IOException IO failure
+ */
+ public Document parseConfiguration(FileSystem fs,
+ Path path) throws
+ IOException {
+
+
+ byte[] data = loadBytes(fs, path);
+ //this is here to track down a parse issue
+ //related to configurations
+ String s = new String(data, 0, data.length);
+ log.debug("XML resource {} is \"{}\"", path, s);
+/* JDK7
+ try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+ Document document = parseConfigXML(in);
+ return document;
+ } catch (ParserConfigurationException | SAXException e) {
+ throw new IOException(e);
+ }
+*/
+ ByteArrayInputStream in= null;
+ try {
+ in = new ByteArrayInputStream(data);
+ Document document = parseConfigXML(in);
+ return document;
+ } catch (ParserConfigurationException e) {
+ throw new IOException(e);
+ } catch (SAXException e) {
+ throw new IOException(e);
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ }
+
+ public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
+ int len = (int) fs.getLength(path);
+ byte[] data = new byte[len];
+ /* JDK7
+ try(FSDataInputStream in = fs.open(path)) {
+ in.readFully(0, data);
+ }
+*/
+ FSDataInputStream in = null;
+ in = fs.open(path);
+ try {
+ in.readFully(0, data);
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ return data;
+ }
+
+ /**
+ * Load a configuration from ANY FS path. The normal Configuration
+ * loader only works with file:// URIs
+ * @param fs filesystem
+ * @param path path
+ * @return a loaded resource
+ * @throws IOException
+ */
+ public static Configuration loadConfiguration(FileSystem fs,
+ Path path) throws IOException {
+ byte[] data = loadBytes(fs, path);
+
+ ByteArrayInputStream in2;
+
+ in2 = new ByteArrayInputStream(data);
+ Configuration conf1 = new Configuration(false);
+ conf1.addResource(in2);
+ //now clone it while dropping all its sources
+ Configuration conf2 = new Configuration(false);
+ String src = path.toString();
+ for (Map.Entry<String, String> entry : conf1) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ conf2.set(key, value, src);
+ }
+ return conf2;
+ }
+
+
+ /**
+ * Generate a config file in a destination directory on the local filesystem
+ * @param confdir the directory path where the file is to go
+ * @param filename the filename
+ * @return the destination path
+ */
+ public static File saveConfig(Configuration generatingConf,
+ File confdir,
+ String filename) throws IOException {
+
+
+ File destPath = new File(confdir, filename);
+ OutputStream fos = new FileOutputStream(destPath);
+ try {
+ generatingConf.writeXml(fos);
+ } finally {
+ IOUtils.closeStream(fos);
+ }
+ return destPath;
+ }
+
+ /**
+ * Parse an XML Hadoop configuration into an XML document. x-include
+ * is supported, but as the location isn't passed in, relative
+ * URIs are out.
+ * @param in instream
+ * @return a document
+ * @throws ParserConfigurationException parser feature problems
+ * @throws IOException IO problems
+ * @throws SAXException XML is invalid
+ */
+ public static Document parseConfigXML(InputStream in) throws
+ ParserConfigurationException,
+ IOException,
+ SAXException {
+ DocumentBuilderFactory docBuilderFactory
+ = DocumentBuilderFactory.newInstance();
+ //ignore all comments inside the xml file
+ docBuilderFactory.setIgnoringComments(true);
+
+ //allow includes in the xml file
+ docBuilderFactory.setNamespaceAware(true);
+ docBuilderFactory.setXIncludeAware(true);
+ DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
+ return builder.parse(in);
+ }
+
+ /**
+ * Load a Hadoop configuration from a local file.
+ * @param file file to load
+ * @return a configuration which hasn't actually had the load triggered
+ * yet.
+ * @throws FileNotFoundException file is not there
+ * @throws IOException any other IO problem
+ */
+ public static Configuration loadConfFromFile(File file) throws
+ IOException {
+ return loadConfFromFile(file, false);
+ }
+
+ /**
+ *
+ * Load a Hadoop configuration from a local file.
+ * @param file file to load
+ * @param loadDefaults flag to indicate if the defaults should be loaded yet
+ * @return a configuration which hasn't actually had the load triggered
+ * yet.
+ * @throws FileNotFoundException file is not there
+ * @throws IOException any other IO problem
+ */
+ public static Configuration loadConfFromFile(File file,
+ boolean loadDefaults) throws IOException {
+ if (!file.exists()) {
+ throw new FileNotFoundException("File not found :"
+ + file.getAbsoluteFile());
+ }
+ Configuration conf = new Configuration(loadDefaults);
+ try {
+ conf.addResource(file.toURI().toURL());
+ } catch (MalformedURLException e) {
+ // should never happen...
+ throw new IOException(
+ "File " + file.toURI() + " doesn't have a valid URL");
+ }
+ return conf;
+ }
+
+ /**
+ * Add a configuration from a file to an existing configuration
+ * @param conf existing configuration
+ * @param file file to load
+ * @param overwrite flag to indicate new values should overwrite the predecessor
+ * @return the merged configuration
+ * @throws IOException
+ */
+ public static Configuration addConfigurationFile(Configuration conf,
+ File file, boolean overwrite)
+ throws IOException {
+ Configuration c2 = loadConfFromFile(file, false);
+ mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite);
+ return conf;
+ }
+
+ /**
+ * Add the system env variables with the given prefix (by convention, env.)
+ * @param conf existing configuration
+ * @param prefix prefix
+ */
+ public static void addEnvironmentVariables(Configuration conf, String prefix) {
+ Map<String, String> env = System.getenv();
+ for (Map.Entry<String, String> entry : env.entrySet()) {
+ conf.set(prefix + entry.getKey(),entry.getValue(), "env");
+ }
+ }
+
+ /**
+ * looks for the config under $confdir/$templateFilename; if not there
+ * loads it from /conf/templateFile.
+ * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
+ * origin to help debug what's happening
+ * @param systemConf system conf
+ * @param confdir conf dir in FS
+ * @param templateFilename filename in the confdir
+ * @param fallbackResource resource to fall back on
+ * @return loaded conf
+ * @throws IOException IO problems
+ */
+ public static Configuration loadTemplateConfiguration(Configuration systemConf,
+ Path confdir,
+ String templateFilename,
+ String fallbackResource) throws
+ IOException {
+ FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
+
+ Path templatePath = new Path(confdir, templateFilename);
+ return loadTemplateConfiguration(fs, templatePath, fallbackResource);
+ }
+
+ /**
+ * looks for the config under $confdir/$templateFilename; if not there
+ * loads it from /conf/templateFile.
+ * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
+ * origin to help debug what's happening.
+ * @param fs Filesystem
+ * @param templatePath HDFS path for template
+ * @param fallbackResource resource to fall back on, or "" for no fallback
+ * @return loaded conf
+ * @throws IOException IO problems
+ * @throws FileNotFoundException if the path doesn't have a file and there
+ * was no fallback.
+ */
+ public static Configuration loadTemplateConfiguration(FileSystem fs,
+ Path templatePath,
+ String fallbackResource)
+ throws IOException {
+ Configuration conf;
+ String origin;
+ if (fs.exists(templatePath)) {
+ log.debug("Loading template configuration {}", templatePath);
+ conf = loadConfiguration(fs, templatePath);
+ origin = templatePath.toString();
+ } else {
+ if (fallbackResource.isEmpty()) {
+ throw new FileNotFoundException("No config file found at " + templatePath);
+ }
+ log.debug("Template {} not found" +
+ " -reverting to classpath resource {}", templatePath, fallbackResource);
+ conf = new Configuration(false);
+ conf.addResource(fallbackResource);
+ origin = "Resource " + fallbackResource;
+ }
+ //force a get
+ conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN);
+ //now set the origin
+ conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin);
+ return conf;
+ }
+
+
+ /**
+ * For testing: dump a configuration
+ * @param conf configuration
+ * @return listing in key=value style
+ */
+ public static String dumpConfigToString(Configuration conf) {
+ Set<String> sorted = sortedConfigKeys(conf);
+
+ StringBuilder builder = new StringBuilder();
+ for (String key : sorted) {
+
+ builder.append(key)
+ .append("=")
+ .append(conf.get(key))
+ .append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Merge in one configuration above another
+ * @param base base config
+ * @param merge one to merge. This MUST be a non-default-load config to avoid
+ * merge origin confusion
+ * @param origin description of the origin for the put operation
+ * @param overwrite flag to indicate new values should overwrite the predecessor
+ * @return the base with the merged values
+ */
+ public static Configuration mergeConfigurations(Configuration base,
+ Iterable<Map.Entry<String, String>> merge,
+ String origin,
+ boolean overwrite) {
+ for (Map.Entry<String, String> entry : merge) {
+ String key = entry.getKey();
+ if (overwrite || base.get(key) == null) {
+ base.set(key, entry.getValue(), origin);
+ }
+ }
+ return base;
+ }
+
+ /**
+ * Register a resource as a default resource.
+ * Do not attempt to use this unless you understand that the
+ * order in which default resources are loaded affects the outcome,
+ * and that subclasses of Configuration often register new default
+ * resources
+ * @param resource the resource name
+ * @return the URL or null
+ */
+ public static URL registerDefaultResource(String resource) {
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ Configuration.addDefaultResource(resource);
+ }
+ return resURL;
+ }
+
+ /**
+ * Load a configuration from a resource on this classpath.
+ * If the resource is not found, an empty configuration is returned
+ * @param resource the resource name
+ * @return the loaded configuration.
+ */
+ public static Configuration loadFromResource(String resource) {
+ Configuration conf = new Configuration(false);
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ log.debug("loaded resources from {}", resURL);
+ conf.addResource(resource);
+ } else{
+ log.debug("failed to find {} on the classpath", resource);
+ }
+ return conf;
+
+ }
+
+ /**
+ * Get the URL to a resource, null if not on the CP
+ * @param resource resource to look for
+ * @return the URL or null
+ */
+ public static URL getResourceUrl(String resource) {
+ return ConfigHelper.class.getClassLoader()
+ .getResource(resource);
+ }
+
+ /**
+ * Load a resource that must be on the classpath
+ * @param resource the resource name
+ * @return the loaded configuration
+ * @throws FileNotFoundException if the resource is missing
+ */
+ public static Configuration loadMandatoryResource(String resource)
+ throws FileNotFoundException {
+ Configuration conf = new Configuration(false);
+ URL resURL = getResourceUrl(resource);
+ if (resURL != null) {
+ log.debug("loaded resources from {}", resURL);
+ conf.addResource(resource);
+ } else {
+ throw new FileNotFoundException(resource);
+ }
+ return conf;
+ }
+
+ /**
+ * Propagate a property from a source to a dest config, with a best-effort
+ * attempt at propagating the origin.
+ * If the
+ * @param dest destination
+ * @param src source
+ * @param key key to try to copy
+ * @return true if the key was found and propagated
+ */
+ public static boolean propagate(Configuration dest,
+ Configuration src,
+ String key) {
+ String val = src.get(key);
+ if (val != null) {
+ String[] origin = src.getPropertySources(key);
+ if (origin != null && origin.length > 0) {
+ dest.set(key, val, origin[0]);
+ } else {
+ dest.set(key, val);
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+ /**
+ * Take a configuration, return a hash map
+ * @param conf conf
+ * @return hash map
+ */
+ public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
+ Map<String, String> map = new HashMap<String, String>();
+ return SliderUtils.mergeEntries(map, conf);
+ }
+
+ /**
+ * This goes through the keyset of one configuration and retrieves each value
+ * from a value source -a different or the same configuration. This triggers
+ * the property resolution process of the value, resolving any variables against
+ * in-config or inherited configurations
+ * @param keysource source of keys
+ * @param valuesource the source of values
+ * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
+ */
+ public static Configuration resolveConfiguration(
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ Configuration result = new Configuration(false);
+ for (Map.Entry<String, String> entry : keysource) {
+ String key = entry.getKey();
+ String value = valuesource.get(key);
+ Preconditions.checkState(value != null,
+ "no reference for \"%s\" in values", key);
+ result.set(key, value);
+ }
+ return result;
+ }
+
+ /**
+ * Register anything we consider deprecated
+ */
+ public static void registerDeprecatedConfigItems() {
+ Configuration.addDeprecation(
+ SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+ Configuration.addDeprecation(
+ SliderXmlConfKeys.REGISTRY_PATH,
+ RegistryConstants.KEY_REGISTRY_ZK_ROOT);
+
+ }
+
+ /**
+ * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
+ * included
+ * @return a configuration instance
+ */
+ public static Configuration loadSliderConfiguration() {
+ Configuration conf = new Configuration();
+ conf.addResource(SliderKeys.SLIDER_XML);
+ return conf;
+ }
+
+ /**
+ * Inject the {@link SliderKeys#SLIDER_XML} resource
+ * into the configuration resources <i>of all configurations</i>.
+ * <p>
+ * This operation is idempotent.
+ * <p>
+ * If the resource is not on the classpath, downgrades, rather than
+ * fails.
+ * @return true if the resource was found and loaded.
+ */
+ public static synchronized boolean injectSliderXMLResource() {
+ if (sliderResourceInjectionAttempted.getAndSet(true)) {
+ return sliderResourceInjected.get();
+ }
+ URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML);
+ if (resourceUrl != null) {
+ Configuration.addDefaultResource(SliderKeys.SLIDER_XML);
+ sliderResourceInjected.set(true);
+ }
+ return sliderResourceInjected.get();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org