You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/22 19:31:53 UTC
[27/51] [abbrv] hadoop git commit: YARN-5967. Fix slider core module
findbugs warnings. Contributed by Jian He
YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/562c8263
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/562c8263
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/562c8263
Branch: refs/heads/yarn-native-services
Commit: 562c8263e033ffd74d0dc1177992007bec7986da
Parents: 0cb7071
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Dec 19 12:09:53 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 96 +++++
.../dev-support/findbugs-exclude.xml | 20 -
.../hadoop-yarn-slider-core/pom.xml | 5 +
.../apache/hadoop/security/KerberosDiags.java | 5 +-
.../slider/api/proto/RestTypeMarshalling.java | 283 -------------
.../slider/api/types/ComponentInformation.java | 2 +-
.../slider/api/types/RestTypeMarshalling.java | 284 +++++++++++++
.../org/apache/slider/client/SliderClient.java | 56 +--
.../apache/slider/client/TokensOperation.java | 9 +-
.../client/ipc/SliderClusterOperations.java | 2 +-
.../rest/SliderApplicationApiRestClient.java | 14 -
.../org/apache/slider/common/SliderKeys.java | 8 +-
.../common/params/AbstractActionArgs.java | 5 +-
.../AbstractClusterBuildingActionArgs.java | 1 -
.../apache/slider/common/params/CommonArgs.java | 9 +-
.../apache/slider/common/tools/Comparators.java | 17 +-
.../slider/common/tools/ConfigHelper.java | 2 +-
.../slider/common/tools/CoreFileSystem.java | 31 --
.../apache/slider/common/tools/SliderUtils.java | 102 +----
.../slider/core/buildutils/InstanceBuilder.java | 1 -
.../apache/slider/core/conf/AggregateConf.java | 2 +-
.../org/apache/slider/core/conf/ConfTree.java | 15 +-
.../core/persist/AppDefinitionPersister.java | 15 +-
.../core/registry/retrieve/AMWebClient.java | 51 ---
.../core/restclient/HttpOperationResponse.java | 34 --
.../restclient/UrlConnectionOperations.java | 120 ------
.../slider/core/zk/MiniZooKeeperCluster.java | 27 +-
.../apache/slider/core/zk/ZKIntegration.java | 20 +-
.../apache/slider/providers/ProviderUtils.java | 21 +-
.../providers/docker/DockerProviderService.java | 12 +-
.../server/appmaster/management/BoolMetric.java | 15 -
.../management/MetricsAndMonitoring.java | 25 --
.../management/RangeLimitedCounter.java | 85 ----
.../appmaster/management/RecordedEvent.java | 58 ---
.../server/appmaster/rpc/SliderIPCService.java | 2 +-
.../server/appmaster/state/NodeInstance.java | 2 +-
.../state/OutstandingRequestTracker.java | 2 +-
.../server/appmaster/state/RoleStatus.java | 4 -
.../appmaster/web/rest/InsecureAmFilter.java | 1 -
.../web/rest/publisher/PublisherResource.java | 5 +-
.../appmaster/web/view/ContainerStatsBlock.java | 3 +-
.../server/servicemonitor/ProbeStatus.java | 1 +
.../utility/AbstractSliderLaunchedService.java | 2 -
.../services/workflow/LongLivedProcess.java | 5 +-
.../slider/common/tools/TestSliderUtils.java | 2 +-
.../slider/test/MiniZooKeeperCluster.java | 395 -------------------
46 files changed, 501 insertions(+), 1375 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..d253762
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <Match>
+ <Package name="org.apache.slider.api.proto" />
+ </Match>
+ <Match>
+ <class name="~org\.apache\.slider\.*" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.core.zk.BlockingZKWatcher" />
+ <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.state.ProviderAppState" />
+ <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.providers.ProviderUtils" />
+ <Bug pattern="SWL_SLEEP_WITH_LOCK_HELD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.SliderAppMaster" />
+ <Bug pattern="WA_AWAIT_NOT_IN_LOOP" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.core.zk.ZKIntegration" />
+ <Bug pattern="SBSC_USE_STRINGBUFFER_CONCATENATION" />
+ </Match>
+ <Match>
+ <Bug pattern="NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.servicemonitor.ProbeWorker" />
+ <Bug pattern="SF_SWITCH_FALLTHROUGH" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.core.persist.JsonSerDeser" />
+ <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider" />
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.state.OutstandingRequest" />
+ <Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.web.SliderAMWebApp" />
+ <Bug pattern="LG_LOST_LOGGER_DUE_TO_WEAK_REFERENCE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.servicemonitor.Probe" />
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.services.workflow.ForkedProcessService" />
+ <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.state.RoleInstance"/>
+ <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.client.SliderClient" />
+ <Method name="actionRegistryListConfigsYarn" />
+ <Bug pattern="OS_OPEN_STREAM" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.client.SliderClient" />
+ <Method name="actionRegistryListExports" />
+ <Bug pattern="OS_OPEN_STREAM" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.common.tools.SliderUtils" />
+ <Method name="getApplicationResourceInputStream" />
+ <Bug pattern="OS_OPEN_STREAM" />
+ </Match>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
deleted file mode 100644
index b89146a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
-
-</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 7453d12..81e607b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -26,6 +26,11 @@
<packaging>jar</packaging>
<name>Apache Hadoop YARN Slider Core</name>
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.basedir}</yarn.basedir>
+ </properties>
+
<build>
<!-- resources are filtered for dynamic updates. This gets build info in-->
<resources>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
index 8c572b3..905d4b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
@@ -35,6 +35,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.PrintWriter;
import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
@@ -82,7 +83,7 @@ public class KerberosDiags implements Closeable {
private final Configuration conf;
private final List<String> services;
- private final PrintWriter out;
+ private final PrintStream out;
private final File keytab;
private final String principal;
private final long minKeyLength;
@@ -97,7 +98,7 @@ public class KerberosDiags implements Closeable {
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public KerberosDiags(Configuration conf,
- PrintWriter out,
+ PrintStream out,
List<String> services,
File keytab,
String principal,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
deleted file mode 100644
index ec35028..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.proto;
-
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.api.types.NodeEntryInformation;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.persist.AggregateConfSerDeser;
-import org.apache.slider.core.persist.ConfTreeSerDeser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Class to handle marshalling of REST
- * types to/from Protobuf records.
- */
-public class RestTypeMarshalling {
-
- public static Messages.ApplicationLivenessInformationProto
- marshall(ApplicationLivenessInformation info) {
-
- Messages.ApplicationLivenessInformationProto.Builder builder =
- Messages.ApplicationLivenessInformationProto.newBuilder();
- builder.setAllRequestsSatisfied(info.allRequestsSatisfied);
- builder.setRequestsOutstanding(info.requestsOutstanding);
- return builder.build();
- }
-
- public static ApplicationLivenessInformation
- unmarshall(Messages.ApplicationLivenessInformationProto wire) {
- ApplicationLivenessInformation info = new ApplicationLivenessInformation();
- info.allRequestsSatisfied = wire.getAllRequestsSatisfied();
- info.requestsOutstanding = wire.getRequestsOutstanding();
- return info;
- }
-
- public static ComponentInformation
- unmarshall(Messages.ComponentInformationProto wire) {
- ComponentInformation info = new ComponentInformation();
- info.name = wire.getName();
- info.priority = wire.getPriority();
- info.placementPolicy = wire.getPlacementPolicy();
-
- info.actual = wire.getActual();
- info.completed = wire.getCompleted();
- info.desired = wire.getDesired();
- info.failed = wire.getFailed();
- info.releasing = wire.getReleasing();
- info.requested = wire.getRequested();
- info.started = wire.getStarted();
- info.startFailed = wire.getStartFailed();
- info.totalRequested = wire.getTotalRequested();
- info.containers = new ArrayList<>(wire.getContainersList());
- if (wire.hasFailureMessage()) {
- info.failureMessage = wire.getFailureMessage();
- }
- if (wire.hasPendingAntiAffineRequestCount()) {
- info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
- }
- if (wire.hasIsAARequestOutstanding()) {
- info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
- }
- return info;
- }
- public static Messages.ComponentInformationProto marshall(ComponentInformation info) {
-
- Messages.ComponentInformationProto.Builder builder =
- Messages.ComponentInformationProto.newBuilder();
- builder.setName(info.name);
- builder.setPriority(info.priority);
- builder.setPlacementPolicy(info.placementPolicy);
-
- builder.setActual(info.actual);
- builder.setCompleted(info.completed);
- builder.setDesired(info.desired);
- builder.setFailed(info.failed);
- builder.setReleasing(info.releasing);
- builder.setRequested(info.requested);
- builder.setStarted(info.started);
- builder.setStartFailed(info.startFailed);
- builder.setTotalRequested(info.totalRequested);
- builder.setNodeFailed(info.nodeFailed);
- builder.setPreempted(info.preempted);
- builder.setFailedRecently(info.failedRecently);
- if (info.failureMessage != null) {
- builder.setFailureMessage(info.failureMessage);
- }
- if (info.containers != null) {
- builder.addAllContainers(info.containers);
- }
- builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
- builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
- return builder.build();
- }
-
- public static Messages.NodeInformationProto marshall(NodeInformation info) {
-
- Messages.NodeInformationProto.Builder builder =
- Messages.NodeInformationProto.newBuilder();
- builder.setHostname(info.hostname);
- builder.setLastUpdated(info.lastUpdated);
- builder.setState(info.state != null? info.state : "unknown");
- builder.setRackName(info.rackName != null ? info.rackName : "");
- builder.setHealthReport(info.healthReport != null ? info.healthReport : "");
- builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : "");
- builder.setLabels(info.labels != null ? info.labels: "");
-
-
- if (info.entries != null) {
- for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) {
- NodeEntryInformation entry = elt.getValue();
- Messages.NodeEntryInformationProto.Builder node =
- Messages.NodeEntryInformationProto.newBuilder();
- node.setPriority(entry.priority);
- node.setName(elt.getKey());
- node.setFailed(entry.failed);
- node.setFailedRecently(entry.failedRecently);
- node.setLive(entry.live);
- node.setLastUsed(entry.lastUsed);
- node.setPreempted(entry.preempted);
- node.setRequested(entry.requested);
- node.setReleasing(entry.releasing);
- node.setStartFailed(entry.startFailed);
- node.setStarting(entry.starting);
- builder.addEntries(node.build());
- }
- }
- return builder.build();
- }
-
- public static NodeInformation unmarshall(Messages.NodeInformationProto wire) {
- NodeInformation info = new NodeInformation();
- info.healthReport = wire.getHealthReport();
- info.hostname = wire.getHostname();
- info.httpAddress = wire.getHttpAddress();
- info.labels = wire.getLabels();
- info.lastUpdated = wire.getLastUpdated();
- info.rackName = wire.getRackName();
- info.state = wire.getState();
- List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList();
- if (entriesList != null) {
- info.entries = new HashMap<>(entriesList.size());
- for (Messages.NodeEntryInformationProto entry : entriesList) {
- NodeEntryInformation nei = new NodeEntryInformation();
- nei.failed = entry.getFailed();
- nei.failedRecently = entry.getFailedRecently();
- nei.lastUsed = entry.getLastUsed();
- nei.live = entry.getLive();
- nei.preempted = entry.getPreempted();
- nei.priority = entry.getPriority();
- nei.requested = entry.getRequested();
- nei.releasing = entry.getReleasing();
- nei.startFailed = entry.getStartFailed();
- nei.starting = entry.getStarting();
- info.entries.put(entry.getName(), nei);
- }
- }
- return info;
- }
-
- public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) {
- ContainerInformation info = new ContainerInformation();
- info.containerId = wire.getContainerId();
- info.component = wire.getComponent();
- info.appVersion = wire.getAppVersion();
- info.state = wire.getState();
- if (wire.hasReleased()) {
- info.released = wire.getReleased();
- }
- if (wire.hasExitCode()) {
- info.exitCode = wire.getExitCode();
- }
- if (wire.hasDiagnostics()) {
- info.diagnostics = wire.getDiagnostics();
- }
- if (wire.hasHost()) {
- info.host = wire.getHost();
- }
- if (wire.hasHostURL()) {
- info.host = wire.getHostURL();
- }
- info.createTime = wire.getCreateTime();
- info.startTime = wire.getStartTime();
- info.output = wire.getOutputList().toArray(
- new String[wire.getOutputCount()]
- );
- if (wire.hasPlacement()) {
- info.placement = wire.getPlacement();
- }
- return info;
- }
-
- public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) {
- List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size());
- for (Messages.ContainerInformationProto container : wire.getContainersList()) {
- infoList.add(unmarshall(container));
- }
- return infoList;
- }
-
- public static Messages.ContainerInformationProto marshall(ContainerInformation info) {
-
- Messages.ContainerInformationProto.Builder builder =
- Messages.ContainerInformationProto.newBuilder();
- if (info.containerId != null) {
- builder.setContainerId(info.containerId);
- }
- if (info.component != null) {
- builder.setComponent(info.component);
- }
- if (info.appVersion != null) {
- builder.setAppVersion(info.appVersion);
- }
- builder.setCreateTime(info.createTime);
- if (info.diagnostics != null) {
- builder.setDiagnostics(info.diagnostics);
- }
- if (info.host != null) {
- builder.setHost(info.host);
- }
- if (info.hostURL != null) {
- builder.setHostURL(info.hostURL);
- }
- if (info.output != null) {
- builder.addAllOutput(Arrays.asList(info.output));
- }
- if (info.released != null) {
- builder.setReleased(info.released);
- }
- if (info.placement != null) {
- builder.setPlacement(info.placement);
- }
- builder.setStartTime(info.startTime);
- builder.setState(info.state);
- return builder.build();
- }
-
- public static String unmarshall(Messages.WrappedJsonProto wire) {
- return wire.getJson();
- }
-
- public static ConfTree unmarshallToConfTree(Messages.WrappedJsonProto wire) throws
- IOException {
- return new ConfTreeSerDeser().fromJson(wire.getJson());
- }
-
- public static ConfTreeOperations unmarshallToCTO(Messages.WrappedJsonProto wire) throws
- IOException {
- return new ConfTreeOperations(new ConfTreeSerDeser().fromJson(wire.getJson()));
- }
-
- public static AggregateConf unmarshallToAggregateConf(Messages.WrappedJsonProto wire) throws
- IOException {
- return new AggregateConfSerDeser().fromJson(wire.getJson());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
index c46a59f..d2fdd62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -31,7 +31,7 @@ import java.util.Map;
* Serializable version of component data.
* <p>
* This is sent in REST calls as a JSON object \u2014but is also marshalled into
- * a protobuf structure. Look at {@link org.apache.slider.api.proto.RestTypeMarshalling}
+ * a protobuf structure. Look at {@link RestTypeMarshalling}
* for the specifics there.
* <p>
* This means that if any fields are added here. they must be added to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
new file mode 100644
index 0000000..713cffd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeEntryInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.persist.AggregateConfSerDeser;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class to handle marshalling of REST
+ * types to/from Protobuf records.
+ */
+public class RestTypeMarshalling {
+
+ public static Messages.ApplicationLivenessInformationProto
+ marshall(ApplicationLivenessInformation info) {
+
+ Messages.ApplicationLivenessInformationProto.Builder builder =
+ Messages.ApplicationLivenessInformationProto.newBuilder();
+ builder.setAllRequestsSatisfied(info.allRequestsSatisfied);
+ builder.setRequestsOutstanding(info.requestsOutstanding);
+ return builder.build();
+ }
+
+ public static ApplicationLivenessInformation
+ unmarshall(Messages.ApplicationLivenessInformationProto wire) {
+ ApplicationLivenessInformation info = new ApplicationLivenessInformation();
+ info.allRequestsSatisfied = wire.getAllRequestsSatisfied();
+ info.requestsOutstanding = wire.getRequestsOutstanding();
+ return info;
+ }
+
+ public static ComponentInformation
+ unmarshall(Messages.ComponentInformationProto wire) {
+ ComponentInformation info = new ComponentInformation();
+ info.name = wire.getName();
+ info.priority = wire.getPriority();
+ info.placementPolicy = wire.getPlacementPolicy();
+
+ info.actual = wire.getActual();
+ info.completed = wire.getCompleted();
+ info.desired = wire.getDesired();
+ info.failed = wire.getFailed();
+ info.releasing = wire.getReleasing();
+ info.requested = wire.getRequested();
+ info.started = wire.getStarted();
+ info.startFailed = wire.getStartFailed();
+ info.totalRequested = wire.getTotalRequested();
+ info.containers = new ArrayList<>(wire.getContainersList());
+ if (wire.hasFailureMessage()) {
+ info.failureMessage = wire.getFailureMessage();
+ }
+ if (wire.hasPendingAntiAffineRequestCount()) {
+ info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
+ }
+ if (wire.hasIsAARequestOutstanding()) {
+ info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
+ }
+ return info;
+ }
+ public static Messages.ComponentInformationProto marshall(ComponentInformation info) {
+
+ Messages.ComponentInformationProto.Builder builder =
+ Messages.ComponentInformationProto.newBuilder();
+ builder.setName(info.name);
+ builder.setPriority(info.priority);
+ builder.setPlacementPolicy(info.placementPolicy);
+
+ builder.setActual(info.actual);
+ builder.setCompleted(info.completed);
+ builder.setDesired(info.desired);
+ builder.setFailed(info.failed);
+ builder.setReleasing(info.releasing);
+ builder.setRequested(info.requested);
+ builder.setStarted(info.started);
+ builder.setStartFailed(info.startFailed);
+ builder.setTotalRequested(info.totalRequested);
+ builder.setNodeFailed(info.nodeFailed);
+ builder.setPreempted(info.preempted);
+ builder.setFailedRecently(info.failedRecently);
+ if (info.failureMessage != null) {
+ builder.setFailureMessage(info.failureMessage);
+ }
+ if (info.containers != null) {
+ builder.addAllContainers(info.containers);
+ }
+ builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
+ builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
+ return builder.build();
+ }
+
+ public static Messages.NodeInformationProto marshall(NodeInformation info) {
+
+ Messages.NodeInformationProto.Builder builder =
+ Messages.NodeInformationProto.newBuilder();
+ builder.setHostname(info.hostname);
+ builder.setLastUpdated(info.lastUpdated);
+ builder.setState(info.state != null? info.state : "unknown");
+ builder.setRackName(info.rackName != null ? info.rackName : "");
+ builder.setHealthReport(info.healthReport != null ? info.healthReport : "");
+ builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : "");
+ builder.setLabels(info.labels != null ? info.labels: "");
+
+
+ if (info.entries != null) {
+ for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) {
+ NodeEntryInformation entry = elt.getValue();
+ Messages.NodeEntryInformationProto.Builder node =
+ Messages.NodeEntryInformationProto.newBuilder();
+ node.setPriority(entry.priority);
+ node.setName(elt.getKey());
+ node.setFailed(entry.failed);
+ node.setFailedRecently(entry.failedRecently);
+ node.setLive(entry.live);
+ node.setLastUsed(entry.lastUsed);
+ node.setPreempted(entry.preempted);
+ node.setRequested(entry.requested);
+ node.setReleasing(entry.releasing);
+ node.setStartFailed(entry.startFailed);
+ node.setStarting(entry.starting);
+ builder.addEntries(node.build());
+ }
+ }
+ return builder.build();
+ }
+
+ public static NodeInformation unmarshall(Messages.NodeInformationProto wire) {
+ NodeInformation info = new NodeInformation();
+ info.healthReport = wire.getHealthReport();
+ info.hostname = wire.getHostname();
+ info.httpAddress = wire.getHttpAddress();
+ info.labels = wire.getLabels();
+ info.lastUpdated = wire.getLastUpdated();
+ info.rackName = wire.getRackName();
+ info.state = wire.getState();
+ List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList();
+ if (entriesList != null) {
+ info.entries = new HashMap<>(entriesList.size());
+ for (Messages.NodeEntryInformationProto entry : entriesList) {
+ NodeEntryInformation nei = new NodeEntryInformation();
+ nei.failed = entry.getFailed();
+ nei.failedRecently = entry.getFailedRecently();
+ nei.lastUsed = entry.getLastUsed();
+ nei.live = entry.getLive();
+ nei.preempted = entry.getPreempted();
+ nei.priority = entry.getPriority();
+ nei.requested = entry.getRequested();
+ nei.releasing = entry.getReleasing();
+ nei.startFailed = entry.getStartFailed();
+ nei.starting = entry.getStarting();
+ info.entries.put(entry.getName(), nei);
+ }
+ }
+ return info;
+ }
+
+ public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) {
+ ContainerInformation info = new ContainerInformation();
+ info.containerId = wire.getContainerId();
+ info.component = wire.getComponent();
+ info.appVersion = wire.getAppVersion();
+ info.state = wire.getState();
+ if (wire.hasReleased()) {
+ info.released = wire.getReleased();
+ }
+ if (wire.hasExitCode()) {
+ info.exitCode = wire.getExitCode();
+ }
+ if (wire.hasDiagnostics()) {
+ info.diagnostics = wire.getDiagnostics();
+ }
+ if (wire.hasHost()) {
+ info.host = wire.getHost();
+ }
+ if (wire.hasHostURL()) {
+ info.host = wire.getHostURL();
+ }
+ info.createTime = wire.getCreateTime();
+ info.startTime = wire.getStartTime();
+ info.output = wire.getOutputList().toArray(
+ new String[wire.getOutputCount()]
+ );
+ if (wire.hasPlacement()) {
+ info.placement = wire.getPlacement();
+ }
+ return info;
+ }
+
+ public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) {
+ List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size());
+ for (Messages.ContainerInformationProto container : wire.getContainersList()) {
+ infoList.add(unmarshall(container));
+ }
+ return infoList;
+ }
+
+ public static Messages.ContainerInformationProto marshall(ContainerInformation info) {
+
+ Messages.ContainerInformationProto.Builder builder =
+ Messages.ContainerInformationProto.newBuilder();
+ if (info.containerId != null) {
+ builder.setContainerId(info.containerId);
+ }
+ if (info.component != null) {
+ builder.setComponent(info.component);
+ }
+ if (info.appVersion != null) {
+ builder.setAppVersion(info.appVersion);
+ }
+ builder.setCreateTime(info.createTime);
+ if (info.diagnostics != null) {
+ builder.setDiagnostics(info.diagnostics);
+ }
+ if (info.host != null) {
+ builder.setHost(info.host);
+ }
+ if (info.hostURL != null) {
+ builder.setHostURL(info.hostURL);
+ }
+ if (info.output != null) {
+ builder.addAllOutput(Arrays.asList(info.output));
+ }
+ if (info.released != null) {
+ builder.setReleased(info.released);
+ }
+ if (info.placement != null) {
+ builder.setPlacement(info.placement);
+ }
+ builder.setStartTime(info.startTime);
+ builder.setState(info.state);
+ return builder.build();
+ }
+
+ public static String unmarshall(Messages.WrappedJsonProto wire) {
+ return wire.getJson();
+ }
+
+ public static ConfTree unmarshallToConfTree(Messages.WrappedJsonProto wire) throws
+ IOException {
+ return new ConfTreeSerDeser().fromJson(wire.getJson());
+ }
+
+ public static ConfTreeOperations unmarshallToCTO(Messages.WrappedJsonProto wire) throws
+ IOException {
+ return new ConfTreeOperations(new ConfTreeSerDeser().fromJson(wire.getJson()));
+ }
+
+ public static AggregateConf unmarshallToAggregateConf(Messages.WrappedJsonProto wire) throws
+ IOException {
+ return new AggregateConfSerDeser().fromJson(wire.getJson());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 1c126ac..1a959d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -192,10 +192,10 @@ import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.Writer;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -1322,9 +1322,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
private void initializeOutputStream(String outFile)
- throws FileNotFoundException {
+ throws IOException {
if (outFile != null) {
- clientOutputStream = new PrintStream(new FileOutputStream(outFile));
+ clientOutputStream = new PrintStream(outFile, "UTF-8");
} else {
clientOutputStream = System.out;
}
@@ -3299,7 +3299,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
int updateCount = Integer.parseInt(updateCountStr);
// if component was specified before, get the current count
if (component.get(COMPONENT_INSTANCES) != null) {
- currentCount = Integer.valueOf(component.get(COMPONENT_INSTANCES));
+ currentCount = Integer.parseInt(component.get(COMPONENT_INSTANCES));
if (currentCount + updateCount < 0) {
throw new BadCommandArgumentsException("The requested count " +
"of \"%s\" for role %s makes the total number of " +
@@ -3610,16 +3610,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// as this is an API entry point, validate
// the arguments
args.validate();
- RegistryOperations operations = getRegistryOperations();
String path = SliderRegistryUtils.resolvePath(args.path);
ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
try {
if (args.list) {
File destDir = args.destdir;
- if (destDir != null) {
- destDir.mkdirs();
+ if (destDir != null && !destDir.exists() && !destDir.mkdirs()) {
+ throw new IOException("Failed to create directory: " + destDir);
}
+
Map<String, ServiceRecord> recordMap;
Map<String, RegistryPathStatus> znodes;
try {
@@ -3656,9 +3656,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else {
String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
File jsonFile = new File(destDir, filename);
- write(jsonFile,
- serviceRecordMarshal.toBytes(instance),
- true);
+ write(jsonFile, serviceRecordMarshal.toBytes(instance));
}
}
} else {
@@ -3669,7 +3667,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
}
if (outFile != null) {
- write(outFile, serviceRecordMarshal.toBytes(instance), true);
+ write(outFile, serviceRecordMarshal.toBytes(instance));
} else {
println(serviceRecordMarshal.toJson(instance));
}
@@ -4062,11 +4060,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
private int actionKDiag(ActionKDiagArgs args)
throws Exception {
- PrintWriter out = new PrintWriter(System.err);
+ PrintStream out;
boolean closeStream = false;
if (args.out != null) {
- out = new PrintWriter(new FileOutputStream(args.out));
+ out = new PrintStream(args.out, "UTF-8");
closeStream = true;
+ } else {
+ out = System.err;
}
try {
KerberosDiags kdiags = new KerberosDiags(getConfig(),
@@ -4137,7 +4137,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
PrintStream out = null;
try {
if (registryArgs.out != null) {
- out = new PrintStream(new FileOutputStream(registryArgs.out));
+ out = new PrintStream(registryArgs.out, "UTF-8");
} else {
out = System.out;
}
@@ -4145,11 +4145,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (!registryArgs.verbose) {
out.println(configName);
} else {
- PublishedConfiguration published =
- configurations.get(configName);
- out.printf("%s: %s\n",
- configName,
- published.description);
+ PublishedConfiguration published = configurations.get(configName);
+ out.printf("%s: %s%n", configName, published.description);
}
}
} finally {
@@ -4178,7 +4175,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
boolean streaming = false;
try {
if (registryArgs.out != null) {
- out = new PrintStream(new FileOutputStream(registryArgs.out));
+ out = new PrintStream(registryArgs.out, "UTF-8");
streaming = true;
log.debug("Saving output to {}", registryArgs.out);
} else {
@@ -4193,9 +4190,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
out.println(exportName);
} else {
PublishedExports published = exports.get(exportName);
- out.printf("%s: %s\n",
- exportName,
- published.description);
+ out.printf("%s: %s%n", exportName, published.description);
}
}
} finally {
@@ -4401,9 +4396,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* Output to standard out/stderr (implementation specific detail)
* @param src source
*/
- @SuppressWarnings("UseOfSystemOutOrSystemErr")
private static void print(CharSequence src) {
- clientOutputStream.append(src);
+ clientOutputStream.print(src);
}
/**
@@ -4411,8 +4405,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param message message
*/
private static void println(String message) {
- print(message);
- print("\n");
+ clientOutputStream.println(message);
}
/**
* Output to standard out/stderr with a newline after, formatted
@@ -4420,8 +4413,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param args arguments for string formatting
*/
private static void println(String message, Object ... args) {
- print(String.format(message, args));
- print("\n");
+ clientOutputStream.println(String.format(message, args));
}
/**
@@ -4497,12 +4489,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw new UsageException(CommonArgs.usage(serviceArgs, actionName));
}
- private int actionHelp(String errMsg, String actionName)
- throws YarnException, IOException {
- throw new UsageException("%s %s", errMsg, CommonArgs.usage(serviceArgs,
- actionName));
- }
-
/**
* List the nodes in the cluster, possibly filtering by node state or label.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
index 9b9c141..84c65b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
@@ -75,11 +75,10 @@ public class TokensOperation {
}
saveTokens(output, credentials);
String filename = output.getCanonicalPath();
- footnote = String.format("%d tokens saved to %s\n" +
- "To use these in the environment:\n" +
- "export %s=%s",
- credentials.numberOfTokens(),
- filename, UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename);
+ footnote = String.format(
+ "%d tokens saved to %s%n" + "To use these in the environment:%n"
+ + "export %s=%s", credentials.numberOfTokens(), filename,
+ UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename);
} else if (args.source != null) {
File source = args.source;
log.info("Reading credentials from file {}", source);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
index eaf15e6..702233a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
@@ -51,7 +51,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.apache.slider.api.proto.RestTypeMarshalling.*;
+import static org.apache.slider.api.types.RestTypeMarshalling.*;
/**
* Cluster operations at a slightly higher level than the RPC code
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
index 4283ee8..573ef64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
@@ -297,20 +297,6 @@ public class SliderApplicationApiRestClient extends BaseRestClient
.type(MediaType.APPLICATION_JSON_TYPE)
.post(PingInformation.class, f);
}
-
- /**
- * Ping as a POST
- * @param text text to include
- * @return the response
- * @throws IOException on any failure
- */
- public PingInformation pingPut(String text) throws IOException {
- WebResource pingResource = applicationResource(ACTION_PING);
- Form f = new Form();
- return pingResource
- .type(MediaType.TEXT_PLAIN)
- .put(PingInformation.class, text);
- }
@Override
public void stop(String text) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index adf40ce..4bf1b5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -18,6 +18,9 @@
package org.apache.slider.common;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
/**
* Keys and various constants for Slider
@@ -79,8 +82,9 @@ public interface SliderKeys extends SliderXmlConfKeys {
*/
String COMPONENT_TYPE_EXTERNAL_APP = "external_app";
String COMPONENT_SEPARATOR = "-";
- String[] COMPONENT_KEYS_TO_SKIP = {"zookeeper.", "env.MALLOC_ARENA_MAX",
- "site.fs.", "site.dfs."};
+ List<String> COMPONENT_KEYS_TO_SKIP = Collections.unmodifiableList(Arrays
+ .asList("zookeeper.", "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."));
+
/**
* A component type for a client component
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index e3cb288..63ccff8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -142,11 +142,12 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
log.error(message);
int index = 1;
+ StringBuilder buf = new StringBuilder(message);
for (String actionArg : parameters) {
log.error("[{}] \"{}\"", index++, actionArg);
- message += " \"" + actionArg + "\" ";
+ buf.append(" \"" + actionArg + "\" ");
}
- throw new BadCommandArgumentsException(message);
+ throw new BadCommandArgumentsException(buf.toString());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 20a9989..1d28c78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -214,7 +214,6 @@ public abstract class AbstractClusterBuildingActionArgs extends
protected ConfTree buildConfTree(Map<String, String> optionsMap) throws
BadCommandArgumentsException {
ConfTree confTree = new ConfTree();
- ConfTreeOperations ops = new ConfTreeOperations(confTree);
confTree.global.putAll(optionsMap);
return confTree;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 162a87d..9c8e65d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -154,12 +154,12 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
public void parse() throws SliderException {
addActionArguments();
try {
- commander.parse(getArgs());
+ commander.parse(args);
} catch (ParameterException e) {
throw new BadCommandArgumentsException(e, "%s in %s",
e.toString(),
- (getArgs() != null
- ? (SliderUtils.join(getArgs(),
+ (args != null
+ ? (SliderUtils.join(args,
" ", false))
: "[]"));
}
@@ -297,7 +297,4 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
return coreAction.parameters;
}
- public String[] getArgs() {
- return args;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
index 6380d0c..a83901b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -29,25 +29,18 @@ public class Comparators {
public static class LongComparator implements Comparator<Long>, Serializable {
@Override
public int compare(Long o1, Long o2) {
- long result = o1 - o2;
- // need to comparisons with a diff greater than integer size
- if (result < 0 ) {
- return -1;
- } else if (result > 0) {
- return 1;
- }
- return 0;
+ return o1.compareTo(o2);
}
}
-public static class InvertedLongComparator implements Comparator<Long>, Serializable {
- private static final LongComparator inner = new LongComparator();
+
+ public static class InvertedLongComparator
+ implements Comparator<Long>, Serializable {
@Override
public int compare(Long o1, Long o2) {
- return -inner.compare(o1, o2);
+ return o2.compareTo(o1);
}
}
-
/**
* Little template class to reverse any comparitor
* @param <CompareType> the type that is being compared
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index 9db241d..02eba49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -194,7 +194,7 @@ public class ConfigHelper {
byte[] data = loadBytes(fs, path);
//this is here to track down a parse issue
//related to configurations
- String s = new String(data, 0, data.length);
+ String s = new String(data, 0, data.length, "UTF-8");
log.debug("XML resource {} is \"{}\"", path, s);
/* JDK7
try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index aa5edf1..c3d6d98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -464,37 +464,6 @@ public class CoreFileSystem {
}
/**
- * Verify that a file exists in the zip file given by path
- * @param path path to zip file
- * @param file file expected to be in zip
- * @throws FileNotFoundException file not found or is not a zip file
- * @throws IOException trouble with FS
- */
- public void verifyFileExistsInZip(Path path, String file) throws IOException {
- fileSystem.copyToLocalFile(path, new Path("/tmp"));
- File dst = new File((new Path("/tmp", path.getName())).toString());
- Enumeration<? extends ZipEntry> entries;
- ZipFile zipFile = new ZipFile(dst);
- boolean found = false;
-
- try {
- entries = zipFile.entries();
- while (entries.hasMoreElements()) {
- ZipEntry entry = entries.nextElement();
- String nm = entry.getName();
- if (nm.endsWith(file)) {
- found = true;
- break;
- }
- }
- } finally {
- zipFile.close();
- }
- dst.delete();
- if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path);
- log.info("Verification of " + path + " passed");
- }
- /**
* Create the application-instance specific temporary directory
* in the DFS
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 4457957..a107dfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.GlobFilter;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -310,10 +309,6 @@ public final class SliderUtils {
}
String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
Enumeration<URL> urlEnumeration = loader.getResources(class_file);
- if (urlEnumeration == null) {
- throw new IOException("Unable to find resources for class " + my_class);
- }
-
for (; urlEnumeration.hasMoreElements(); ) {
URL url = urlEnumeration.nextElement();
if ("jar".equals(url.getProtocol())) {
@@ -756,10 +751,10 @@ public final class SliderUtils {
public static String containersToString(
List<ContainerInformation> containers, String version,
Set<String> components) {
- String containerf = " %-28s %30s %45s %s\n";
+ String containerf = " %-28s %30s %45s %s%n";
StringBuilder builder = new StringBuilder(512);
- builder.append("Containers:\n");
- builder.append(String.format(" %-28s %30s %45s %s\n", "Component Name",
+ builder.append("Containers:%n");
+ builder.append(String.format(" %-28s %30s %45s %s%n", "Component Name",
"App Version", "Container Id", "Container Info/Logs"));
for (ContainerInformation container : containers) {
if (filter(container.appVersion, version)
@@ -969,7 +964,7 @@ public final class SliderUtils {
*/
public static Map<String, String> mergeMapsIgnoreDuplicateKeysAndPrefixes(
Map<String, String> first, Map<String, String> second,
- String... prefixes) {
+ List<String> prefixes) {
Preconditions.checkArgument(first != null, "Null 'first' value");
Preconditions.checkArgument(second != null, "Null 'second' value");
Preconditions.checkArgument(prefixes != null, "Null 'prefixes' value");
@@ -2119,15 +2114,16 @@ public final class SliderUtils {
is = new ByteArrayInputStream(content);
} else {
log.debug("Size unknown. Reading {}", zipEntry.getName());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- while (true) {
- int byteRead = zis.read();
- if (byteRead == -1) {
- break;
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ while (true) {
+ int byteRead = zis.read();
+ if (byteRead == -1) {
+ break;
+ }
+ baos.write(byteRead);
}
- baos.write(byteRead);
+ is = new ByteArrayInputStream(baos.toByteArray());
}
- is = new ByteArrayInputStream(baos.toByteArray());
}
done = true;
}
@@ -2205,90 +2201,26 @@ public final class SliderUtils {
}
/**
- * Look for the windows executable and check it has the right headers.
- * <code>File.canRead()</code> doesn't work on windows, so the reading
- * is mandatory.
- *
- * @param program program name for errors
- * @param exe executable
- * @throws IOException IOE
- */
- public static void verifyWindowsExe(String program, File exe)
- throws IOException {
- verifyIsFile(program, exe);
-
- verifyFileSize(program, exe, 0x100);
-
- // now read two bytes and verify the header.
- try(FileReader reader = new FileReader(exe)) {
- int[] header = new int[2];
- header[0] = reader.read();
- header[1] = reader.read();
- if ((header[0] != 'M' || header[1] != 'Z')) {
- throw new FileNotFoundException(program
- + " at " + exe
- + " is not a windows executable file");
- }
- }
- }
-
- /**
- * Verify that a Unix exe works
- * @param program program name for errors
- * @param exe executable
- * @throws IOException IOE
-
- */
- public static void verifyUnixExe(String program, File exe)
- throws IOException {
- verifyIsFile(program, exe);
-
- // read flag
- if (!exe.canRead()) {
- throw new IOException("Cannot read " + program + " at " + exe);
- }
- // exe flag
- if (!exe.canExecute()) {
- throw new IOException("Cannot execute " + program + " at " + exe);
- }
- }
-
- /**
- * Validate an executable
- * @param program program name for errors
- * @param exe program to look at
- * @throws IOException
- */
- public static void validateExe(String program, File exe) throws IOException {
- if (!Shell.WINDOWS) {
- verifyWindowsExe(program, exe);
- } else {
- verifyUnixExe(program, exe);
- }
- }
-
- /**
* Write bytes to a file
* @param outfile output file
* @param data data to write
- * @param createParent flag to indicate that the parent dir should
- * be created
* @throws IOException on any IO problem
*/
- public static void write(File outfile, byte[] data, boolean createParent)
+ public static void write(File outfile, byte[] data)
throws IOException {
File parentDir = outfile.getCanonicalFile().getParentFile();
if (parentDir == null) {
throw new IOException(outfile.getPath() + " has no parent dir");
}
- if (createParent) {
- parentDir.mkdirs();
+ if (!parentDir.exists()) {
+ if(!parentDir.mkdirs()) {
+ throw new IOException("Failed to create parent directory " + parentDir);
+ }
}
SliderUtils.verifyIsDir(parentDir, log);
try(FileOutputStream out = new FileOutputStream(outfile)) {
out.write(data);
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
index f0686af..f14a07a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
@@ -142,7 +142,6 @@ public class InstanceBuilder {
md.put(StatusKeys.INFO_CREATE_TIME_HUMAN, SliderUtils.toGMTString(time));
md.put(StatusKeys.INFO_CREATE_TIME_MILLIS, Long.toString(time));
- MapOperations globalOptions = internalOps.getGlobalOptions();
BuildHelper.addBuildMetadata(md, "create");
SliderUtils.setInfoTime(md,
StatusKeys.INFO_CREATE_TIME_HUMAN,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
index 18c3156..a272420 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
@@ -163,7 +163,7 @@ public final class AggregateConf {
public String getPassphrase() {
if (passphrase == null) {
passphrase = RandomStringUtils.randomAlphanumeric(
- Integer.valueOf(SliderKeys.PASS_LEN));
+ Integer.parseInt(SliderKeys.PASS_LEN));
}
return passphrase;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
index be7c56f..74b6abb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
@@ -79,17 +79,6 @@ public final class ConfTree {
public Map<String, Map<String, String>> components =
new HashMap<>(INITAL_MAP_CAPACITY);
-
- /**
- * Shallow clone
- * @return a shallow clone
- * @throws CloneNotSupportedException
- */
- @Override
- public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-
@Override
public String toString() {
try {
@@ -105,9 +94,7 @@ public final class ConfTree {
* @return a JSON string description
* @throws IOException Problems mapping/writing the object
*/
- public String toJson() throws IOException,
- JsonGenerationException,
- JsonMappingException {
+ public String toJson() throws IOException {
return ConfTreeSerDeser.toString(this);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
index 7fb3158..9eb7d5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
@@ -147,7 +147,9 @@ public class AppDefinitionPersister {
File tempDir = Files.createTempDir();
File pkgSrcDir = new File(tempDir, "default");
- pkgSrcDir.mkdirs();
+ if (!pkgSrcDir.exists() && !pkgSrcDir.mkdirs()) {
+ throw new IOException("Failed to create directory: " + pkgSrcDir);
+ }
File destMetaInfo = new File(pkgSrcDir, "metainfo.json");
if (isFileUsed) {
if (buildInfo.appMetaInfo.getName().endsWith(".xml")) {
@@ -194,12 +196,13 @@ public class AppDefinitionPersister {
List<String> addons = new ArrayList<String>();
Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap();
- for (String key : addonMap.keySet()) {
- File defPath = new File(addonMap.get(key));
- if (SliderUtils.isUnset(addonMap.get(key))) {
+ for (Map.Entry<String, String > entry : addonMap.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ if (SliderUtils.isUnset(value)) {
throw new BadConfigException("Invalid path for addon package " + key);
}
-
+ File defPath = new File(value);
if (!defPath.exists()) {
throw new BadConfigException("addon folder or package path is not valid.");
}
@@ -234,7 +237,7 @@ public class AppDefinitionPersister {
}
// Helper class to hold details for the app and addon packages
- public class AppDefinition {
+ static class AppDefinition {
// The target folder where the package will be stored
public Path targetFolderInFs;
// The on disk location of the app def package or folder
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
index 40fa217..e204178 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
@@ -62,61 +62,10 @@ public class AMWebClient {
restClient = new BaseRestClient(binding.createJerseyClient());
}
-
-
- private static URLConnectionClientHandler getUrlConnectionClientHandler() {
- return new URLConnectionClientHandler(new HttpURLConnectionFactory() {
- @Override
- public HttpURLConnection getHttpURLConnection(URL url)
- throws IOException {
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
- if (connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) {
- // is a redirect - are we changing schemes?
- String redirectLocation = connection.getHeaderField(HttpHeaders.LOCATION);
- String originalScheme = url.getProtocol();
- String redirectScheme = URI.create(redirectLocation).getScheme();
- if (!originalScheme.equals(redirectScheme)) {
- // need to fake it out by doing redirect ourselves
- log.info("Protocol change during redirect. Redirecting {} to URL {}",
- url, redirectLocation);
- URL redirectURL = new URL(redirectLocation);
- connection = (HttpURLConnection) redirectURL.openConnection();
- }
- }
- if (connection instanceof HttpsURLConnection) {
- log.debug("Attempting to configure HTTPS connection using client "
- + "configuration");
- final SSLFactory factory;
- final SSLSocketFactory sf;
- final HostnameVerifier hv;
-
- try {
- HttpsURLConnection c = (HttpsURLConnection) connection;
- factory = new SSLFactory(SSLFactory.Mode.CLIENT, new Configuration());
- factory.init();
- sf = factory.createSSLSocketFactory();
- hv = factory.getHostnameVerifier();
- c.setSSLSocketFactory(sf);
- c.setHostnameVerifier(hv);
- } catch (Exception e) {
- log.info("Unable to configure HTTPS connection from "
- + "configuration. Using JDK properties.");
- }
-
- }
- return connection;
- }
- });
- }
-
public WebResource resource(String url) {
return restClient.resource(url);
}
- public BaseRestClient getRestClient() {
- return restClient;
- }
-
/**
* Execute the operation. Failures are raised as IOException subclasses
* @param method method to execute
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
deleted file mode 100644
index 0266223..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.restclient;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * A response for use as a return value from operations
- */
-public class HttpOperationResponse {
-
- public int responseCode;
- public long lastModified;
- public String contentType;
- public byte[] data;
- public Map<String, List<String>> headers;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index 20ef198..46f0d02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -87,124 +87,4 @@ public class UrlConnectionOperations extends Configured {
Preconditions.checkArgument(url.getPort() != 0, "no port");
return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
}
-
- public HttpOperationResponse execGet(URL url) throws
- IOException,
- AuthenticationException {
- return execHttpOperation(HttpVerb.GET, url, null, "");
- }
-
- public HttpOperationResponse execHttpOperation(HttpVerb verb,
- URL url,
- byte[] payload,
- String contentType)
- throws IOException, AuthenticationException {
- HttpURLConnection conn = null;
- HttpOperationResponse outcome = new HttpOperationResponse();
- int resultCode;
- byte[] body = null;
- log.debug("{} {} spnego={}", verb, url, useSpnego);
-
- boolean doOutput = verb.hasUploadBody();
- if (doOutput) {
- Preconditions.checkArgument(payload !=null,
- "Null payload on a verb which expects one");
- }
- try {
- conn = openConnection(url);
- conn.setRequestMethod(verb.getVerb());
- conn.setDoOutput(doOutput);
- if (doOutput) {
- conn.setRequestProperty("Content-Type", contentType);
- }
-
- // now do the connection
- conn.connect();
-
- if (doOutput) {
- OutputStream output = conn.getOutputStream();
- IOUtils.write(payload, output);
- output.close();
- }
-
- resultCode = conn.getResponseCode();
- outcome.lastModified = conn.getLastModified();
- outcome.contentType = conn.getContentType();
- outcome.headers = conn.getHeaderFields();
- InputStream stream = conn.getErrorStream();
- if (stream == null) {
- stream = conn.getInputStream();
- }
- if (stream != null) {
- // read into a buffer.
- body = IOUtils.toByteArray(stream);
- } else {
- // no body:
- log.debug("No body in response");
-
- }
- } catch (SSLException e) {
- throw e;
- } catch (IOException e) {
- throw NetUtils.wrapException(url.toString(),
- url.getPort(), "localhost", 0, e);
-
- } catch (AuthenticationException e) {
- throw new AuthenticationException("From " + url + ": " + e, e);
-
- } finally {
- if (conn != null) {
- conn.disconnect();
- }
- }
- uprateFaults(HttpVerb.GET, url.toString(), resultCode, "", body);
- outcome.responseCode = resultCode;
- outcome.data = body;
- return outcome;
- }
-
- /**
- * Uprate error codes 400 and up into faults;
- * 404 is converted to a {@link NotFoundException},
- * 401 to {@link ForbiddenException}
- *
- * @param verb HTTP Verb used
- * @param url URL as string
- * @param resultCode response from the request
- * @param bodyAsString
- *@param body optional body of the request @throws IOException if the result was considered a failure
- */
- public static void uprateFaults(HttpVerb verb, String url,
- int resultCode, String bodyAsString, byte[] body)
- throws IOException {
-
- if (resultCode < 400) {
- //success
- return;
- }
- String msg = verb.toString() +" "+ url;
- if (resultCode == 404) {
- throw new NotFoundException(msg);
- }
- if (resultCode == 401) {
- throw new ForbiddenException(msg);
- }
- // all other error codes
-
- // get a string respnse
- if (bodyAsString == null) {
- if (body != null && body.length > 0) {
- bodyAsString = new String(body);
- } else {
- bodyAsString = "";
- }
- }
- String message = msg +
- " failed with exit code " + resultCode
- + ", body length " + bodyAsString.length()
- + ":\n" + bodyAsString;
- log.error(message);
- throw new IOException(message);
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org