You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2017/08/14 21:13:50 UTC
[12/15] hadoop git commit: YARN-6903. Yarn-native-service framework
core rewrite. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
new file mode 100644
index 0000000..a4a0a15
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -0,0 +1,487 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.ContainerFailureTracker;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.servicemonitor.MonitorUtils;
+import org.apache.slider.server.servicemonitor.Probe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STARTED;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STOP;
+import static org.apache.hadoop.yarn.service.component.ComponentState.*;
+import static org.apache.slider.api.ResourceKeys.CONTAINER_FAILURE_THRESHOLD;
+
+public class Component implements EventHandler<ComponentEvent> {
+ private static final Logger LOG = LoggerFactory.getLogger(Component.class);
+
+ private org.apache.slider.api.resource.Component componentSpec;
+ private long allocateId;
+ private Priority priority;
+ private ServiceMetrics componentMetrics;
+ private ServiceScheduler scheduler;
+ private ServiceContext context;
+ private AMRMClientAsync<ContainerRequest> amrmClient;
+ private AtomicLong instanceIdCounter = new AtomicLong();
+ private Map<ComponentInstanceId, ComponentInstance> compInstances =
+ new ConcurrentHashMap<>();
+ // component instances to be assigned with a container
+ private List<ComponentInstance> pendingInstances = new LinkedList<>();
+ private ContainerFailureTracker failureTracker;
+ private Probe probe;
+ private final ReentrantReadWriteLock.ReadLock readLock;
+ private final ReentrantReadWriteLock.WriteLock writeLock;
+ public int maxContainerFailurePerComp;
+ // The number of containers failed since last reset. This excludes preempted,
+ // disk_failed containers etc. This will be reset to 0 periodically.
+ public volatile int currentContainerFailure;
+
+ private StateMachine<ComponentState, ComponentEventType, ComponentEvent>
+ stateMachine;
+ private AsyncDispatcher compInstanceDispatcher;
+ private static final StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>
+ stateMachineFactory =
+ new StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>(
+ INIT)
+ // INIT will only got to FLEXING
+ .addTransition(INIT, EnumSet.of(STABLE, FLEXING),
+ FLEX, new FlexComponentTransition())
+
+ // container allocated by RM
+ .addTransition(FLEXING, FLEXING, CONTAINER_ALLOCATED,
+ new ContainerAllocatedTransition())
+ // container launched on NM
+ .addTransition(FLEXING, EnumSet.of(STABLE, FLEXING),
+ CONTAINER_STARTED, new ContainerStartedTransition())
+ // container failed while flexing
+ .addTransition(FLEXING, FLEXING, CONTAINER_COMPLETED,
+ new ContainerCompletedTransition())
+ // Flex while previous flex is still in progress
+ .addTransition(FLEXING, EnumSet.of(FLEXING), FLEX,
+ new FlexComponentTransition())
+
+ // container failed while stable
+ .addTransition(STABLE, FLEXING, CONTAINER_COMPLETED,
+ new ContainerCompletedTransition())
+ // Ignore surplus container
+ .addTransition(STABLE, STABLE, CONTAINER_ALLOCATED,
+ new ContainerAllocatedTransition())
+ // Flex by user
+ // For flex up, go to FLEXING state
+ // For flex down, go to STABLE state
+ .addTransition(STABLE, EnumSet.of(STABLE, FLEXING),
+ FLEX, new FlexComponentTransition())
+ .installTopology();
+
+ public Component(org.apache.slider.api.resource.Component component,
+ long allocateId, ServiceContext context) {
+ this.allocateId = allocateId;
+ this.priority = Priority.newInstance((int) allocateId);
+ this.componentSpec = component;
+ componentMetrics = ServiceMetrics.register(component.getName(),
+ "Metrics for component " + component.getName());
+ componentMetrics
+ .tag("type", "Metrics type [component or service]", "component");
+ this.scheduler = context.scheduler;
+ this.context = context;
+ amrmClient = scheduler.getAmRMClient();
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ this.readLock = lock.readLock();
+ this.writeLock = lock.writeLock();
+ this.stateMachine = stateMachineFactory.make(this);
+ compInstanceDispatcher = scheduler.getCompInstanceDispatcher();
+ failureTracker =
+ new ContainerFailureTracker(context, this);
+ probe = MonitorUtils.getProbe(componentSpec.getReadinessCheck());
+ maxContainerFailurePerComp = componentSpec.getConfiguration()
+ .getPropertyInt(CONTAINER_FAILURE_THRESHOLD, 10);
+ createNumCompInstances(component.getNumberOfContainers());
+ }
+
+ private void createNumCompInstances(long count) {
+ for (int i = 0; i < count; i++) {
+ createOneCompInstance();
+ }
+ }
+
+ private void createOneCompInstance() {
+ ComponentInstanceId id =
+ new ComponentInstanceId(instanceIdCounter.getAndIncrement(),
+ componentSpec.getName());
+ ComponentInstance instance = new ComponentInstance(this, id);
+ compInstances.put(id, instance);
+ pendingInstances.add(instance);
+ }
+
+ private static class FlexComponentTransition implements
+ MultipleArcTransition<Component, ComponentEvent, ComponentState> {
+ // For flex up, go to FLEXING state
+ // For flex down, go to STABLE state
+ @Override
+ public ComponentState transition(Component component,
+ ComponentEvent event) {
+ component.setDesiredContainers((int)event.getDesired());
+ if (!component.areDependenciesReady()) {
+ LOG.info("[FLEX COMPONENT {}]: Flex deferred because dependencies not"
+ + " satisfied.", component.getName());
+ return component.getState();
+ }
+ if (component.getState() == INIT) {
+ // This happens on init
+ LOG.info("[INIT COMPONENT " + component.getName() + "]: " + event
+ .getDesired() + " instances.");
+ component.requestContainers(event.getDesired());
+ return FLEXING;
+ }
+ long before = component.getComponentSpec().getNumberOfContainers();
+ long delta = event.getDesired() - before;
+ component.getComponentSpec().setNumberOfContainers(event.getDesired());
+ if (delta > 0) {
+ // Scale up
+ LOG.info("[FLEX UP COMPONENT " + component.getName() + "]: scaling up from "
+ + before + " to " + event.getDesired());
+ component.requestContainers(delta);
+ component.createNumCompInstances(delta);
+ return FLEXING;
+ } else if (delta < 0){
+ delta = 0 - delta;
+ // scale down
+ LOG.info("[FLEX DOWN COMPONENT " + component.getName()
+ + "]: scaling down from " + before + " to " + event.getDesired());
+ List<ComponentInstance> list =
+ new ArrayList<>(component.compInstances.values());
+
+ // sort in Most recent -> oldest order, destroy most recent ones.
+ Collections.sort(list, Collections.reverseOrder());
+ for (int i = 0; i < delta; i++) {
+ ComponentInstance instance = list.get(i);
+ // remove the instance
+ component.compInstances.remove(instance.getCompInstanceId());
+ component.pendingInstances.remove(instance);
+ component.componentMetrics.containersFailed.incr();
+ component.componentMetrics.containersRunning.decr();
+ // decrement id counter
+ component.instanceIdCounter.decrementAndGet();
+ instance.destroy();
+ }
+ return STABLE;
+ } else {
+ LOG.info("[FLEX COMPONENT " + component.getName() + "]: already has " +
+ event.getDesired() + " instances, ignoring");
+ return STABLE;
+ }
+ }
+ }
+
+ private static class ContainerAllocatedTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ component.assignContainerToCompInstance(event.getContainer());
+ }
+ }
+
+ private static class ContainerStartedTransition implements
+ MultipleArcTransition<Component,ComponentEvent,ComponentState> {
+
+ @Override public ComponentState transition(Component component,
+ ComponentEvent event) {
+ component.compInstanceDispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(event.getInstance().getContainerId(),
+ STARTED));
+ component.incRunningContainers();
+ return checkIfStable(component);
+ }
+ }
+
+ private static ComponentState checkIfStable(Component component) {
+ // if desired == running
+ if (component.componentMetrics.containersRunning.value() == component
+ .getComponentSpec().getNumberOfContainers()) {
+ return STABLE;
+ } else {
+ return FLEXING;
+ }
+ }
+
+ private static class ContainerCompletedTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ component.updateMetrics(event.getStatus());
+
+ // add back to pending list
+ component.pendingInstances.add(event.getInstance());
+ LOG.info(
+ "[COMPONENT {}]: {} completed, num pending comp instances increased to {}.",
+ component.getName(), event.getStatus().getContainerId(),
+ component.pendingInstances.size());
+ component.compInstanceDispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(event.getStatus().getContainerId(),
+ STOP).setStatus(event.getStatus()));
+ }
+ }
+
+ public ServiceMetrics getCompMetrics () {
+ return componentMetrics;
+ }
+
+ private void assignContainerToCompInstance(Container container) {
+ if (pendingInstances.size() == 0) {
+ LOG.info(
+ "[COMPONENT {}]: No pending component instance left, release surplus container {}",
+ getName(), container.getId());
+ scheduler.getAmRMClient().releaseAssignedContainer(container.getId());
+ componentMetrics.surplusContainers.incr();
+ scheduler.getServiceMetrics().surplusContainers.incr();
+ return;
+ }
+ ComponentInstance instance = pendingInstances.remove(0);
+ LOG.info(
+ "[COMPONENT {}]: {} allocated, num pending component instances reduced to {}",
+ getName(), container.getId(), pendingInstances.size());
+ instance.setContainer(container);
+ scheduler.addLiveCompInstance(container.getId(), instance);
+ LOG.info(
+ "[COMPONENT {}]: Assigned {} to component instance {} and launch on host {} ",
+ getName(), container.getId(), instance.getCompInstanceName(),
+ container.getNodeId());
+ scheduler.getContainerLaunchService()
+ .launchCompInstance(scheduler.getApp(), instance, container);
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ public void requestContainers(long count) {
+ Resource resource = Resource
+ .newInstance(componentSpec.getResource().getMemoryMB(),
+ componentSpec.getResource().getCpus());
+
+ for (int i = 0; i < count; i++) {
+ //TODO Once YARN-5468 is done, use that for anti-affinity
+ ContainerRequest request =
+ ContainerRequest.newBuilder().capability(resource).priority(priority)
+ .allocationRequestId(allocateId).relaxLocality(true).build();
+ amrmClient.addContainerRequest(request);
+ }
+ }
+
+ private void setDesiredContainers(int n) {
+ int delta = n - scheduler.getServiceMetrics().containersDesired.value();
+ if (delta > 0) {
+ scheduler.getServiceMetrics().containersDesired.incr(delta);
+ } else {
+ scheduler.getServiceMetrics().containersDesired.decr(delta);
+ }
+ componentMetrics.containersDesired.set(n);
+ }
+
+
+
+ private void updateMetrics(ContainerStatus status) {
+ switch (status.getExitStatus()) {
+ case SUCCESS:
+ componentMetrics.containersSucceeded.incr();
+ scheduler.getServiceMetrics().containersSucceeded.incr();
+ return;
+ case PREEMPTED:
+ componentMetrics.containersPreempted.incr();
+ scheduler.getServiceMetrics().containersPreempted.incr();
+ break;
+ case DISKS_FAILED:
+ componentMetrics.containersDiskFailure.incr();
+ scheduler.getServiceMetrics().containersDiskFailure.incr();
+ break;
+ default:
+ break;
+ }
+
+ // containersFailed include preempted, disks_failed etc.
+ componentMetrics.containersFailed.incr();
+ scheduler.getServiceMetrics().containersFailed.incr();
+
+ // dec running container
+ decRunningContainers();
+
+ if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
+ String host = scheduler.getLiveInstances().get(status.getContainerId())
+ .getNodeId().getHost();
+ failureTracker.incNodeFailure(host);
+ currentContainerFailure++ ;
+ }
+ }
+
+ public boolean areDependenciesReady() {
+ List<String> dependencies = componentSpec.getDependencies();
+ if (SliderUtils.isEmpty(dependencies)) {
+ return true;
+ }
+ for (String dependency : dependencies) {
+ Component dependentComponent =
+ scheduler.getAllComponents().get(dependency);
+ if (dependentComponent == null) {
+ LOG.error("Couldn't find dependency {} for {} (should never happen)",
+ dependency, getName());
+ continue;
+ }
+ if (dependentComponent.getNumReadyInstances() < dependentComponent
+ .getNumDesiredInstances()) {
+ LOG.info("[COMPONENT {}]: Dependency {} not satisfied, only {} of {}"
+ + " instances are ready.", getName(), dependency,
+ dependentComponent.getNumReadyInstances(),
+ dependentComponent.getNumDesiredInstances());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void incRunningContainers() {
+ componentMetrics.containersRunning.incr();
+ scheduler.getServiceMetrics().containersRunning.incr();
+ }
+
+ public void incContainersReady() {
+ componentMetrics.containersReady.incr();
+ }
+
+ public void decContainersReady() {
+ componentMetrics.containersReady.decr();
+ }
+
+ private void decRunningContainers() {
+ componentMetrics.containersRunning.decr();
+ scheduler.getServiceMetrics().containersRunning.decr();
+ }
+
+ public int getNumReadyInstances() {
+ return componentMetrics.containersReady.value();
+ }
+
+ public int getNumRunningInstances() {
+ return componentMetrics.containersRunning.value();
+ }
+
+ public int getNumDesiredInstances() {
+ return componentMetrics.containersDesired.value();
+ }
+
+ public Map<ComponentInstanceId, ComponentInstance> getAllComponentInstances() {
+ return compInstances;
+ }
+
+ public org.apache.slider.api.resource.Component getComponentSpec() {
+ return this.componentSpec;
+ }
+
+ public void resetCompFailureCount() {
+ LOG.info("[COMPONENT {}]: Reset container failure count from {} to 0.",
+ getName(), currentContainerFailure);
+ currentContainerFailure = 0;
+ failureTracker.resetContainerFailures();
+ }
+
+ public Probe getProbe() {
+ return probe;
+ }
+
+ public Priority getPriority() {
+ return priority;
+ }
+
+ public String getName () {
+ return componentSpec.getName();
+ }
+
+ public ComponentState getState() {
+ this.readLock.lock();
+
+ try {
+ return this.stateMachine.getCurrentState();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+ public ServiceScheduler getScheduler() {
+ return scheduler;
+ }
+
+ @Override
+ public void handle(ComponentEvent event) {
+ try {
+ writeLock.lock();
+ ComponentState oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitionException e) {
+ LOG.error("Invalid event " + event.getType() +
+ " at " + oldState + " for component " + componentSpec.getName(), e);
+ }
+ if (oldState != getState()) {
+ LOG.info("[COMPONENT {}] Transitioned from {} to {} on {} event.",
+ componentSpec.getName(), oldState, getState(), event.getType());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private static class BaseTransition implements
+ SingleArcTransition<Component, ComponentEvent> {
+
+ @Override public void transition(Component component,
+ ComponentEvent event) {
+ }
+ }
+
+ public ServiceContext getContext() {
+ return context;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
new file mode 100644
index 0000000..ed892dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+
+public class ComponentEvent extends AbstractEvent<ComponentEventType> {
+ private long desired;
+ private final String name;
+ private final ComponentEventType type;
+ private Container container;
+ private ComponentInstance instance;
+ private ContainerStatus status;
+
+ public ComponentEvent(String name, ComponentEventType type) {
+ super(type);
+ this.name = name;
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public ComponentEventType getType() {
+ return type;
+ }
+
+ public long getDesired() {
+ return desired;
+ }
+
+ public ComponentEvent setDesired(long desired) {
+ this.desired = desired;
+ return this;
+ }
+
+ public Container getContainer() {
+ return container;
+ }
+
+ public ComponentEvent setContainer(Container container) {
+ this.container = container;
+ return this;
+ }
+
+ public ComponentInstance getInstance() {
+ return instance;
+ }
+
+ public ComponentEvent setInstance(ComponentInstance instance) {
+ this.instance = instance;
+ return this;
+ }
+
+ public ContainerStatus getStatus() {
+ return status;
+ }
+
+ public ComponentEvent setStatus(ContainerStatus status) {
+ this.status = status;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
new file mode 100644
index 0000000..6729699
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+public enum ComponentEventType {
+ FLEX,
+ CONTAINER_ALLOCATED,
+ CONTAINER_STARTED,
+ CONTAINER_COMPLETED
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
new file mode 100644
index 0000000..a5f9ff4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+public enum ComponentState {
+ INIT,
+ FLEXING,
+ STABLE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
new file mode 100644
index 0000000..d63c1a4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.slider.core.main.LauncherExitCodes;
+
+public interface SliderExitCodes extends LauncherExitCodes {
+
+ /**
+ * starting point for exit codes; not an exception itself
+ */
+ int _EXIT_CODE_BASE = 64;
+
+ /**
+ * service entered the failed state: {@value}
+ */
+ int EXIT_YARN_SERVICE_FAILED = 65;
+
+ /**
+ * service was killed: {@value}
+ */
+ int EXIT_YARN_SERVICE_KILLED = 66;
+
+ /**
+ * timeout on monitoring client: {@value}
+ */
+ int EXIT_TIMED_OUT = 67;
+
+ /**
+ * service finished with an error: {@value}
+ */
+ int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
+
+ /**
+ * the application instance is unknown: {@value}
+ */
+ int EXIT_UNKNOWN_INSTANCE = 69;
+
+ /**
+ * the application instance is in the wrong state for that operation: {@value}
+ */
+ int EXIT_BAD_STATE = 70;
+
+ /**
+ * A spawned master process failed
+ */
+ int EXIT_PROCESS_FAILED = 71;
+
+ /**
+ * The instance failed -too many containers were
+ * failing or some other threshold was reached
+ */
+ int EXIT_DEPLOYMENT_FAILED = 72;
+
+ /**
+ * The application is live -and the requested operation
+ * does not work if the cluster is running
+ */
+ int EXIT_APPLICATION_IN_USE = 73;
+
+ /**
+ * There already is an application instance of that name
+ * when an attempt is made to create a new instance
+ */
+ int EXIT_INSTANCE_EXISTS = 75;
+
+ /**
+ * Exit code when the configurations in valid/incomplete: {@value}
+ */
+ int EXIT_BAD_CONFIGURATION = 77;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
new file mode 100644
index 0000000..e1687d2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+/**
+ * Keys and various constants for Slider
+ */
+public interface SliderKeys extends SliderXmlConfKeys {
+
+ /**
+ * This is the name of the slider appmaster in configurations :{@value}
+ */
+ String COMPONENT_AM = "slider-appmaster";
+
+ /**
+ * Slider role is "special":{@value}
+ */
+ int ROLE_AM_PRIORITY_INDEX = 0;
+
+
+ /**
+ * The path under which cluster and temp data are stored
+ * {@value}
+ */
+ String SLIDER_BASE_DIRECTORY = ".slider";
+
+ /**
+ * The paths under which Slider AM dependency libraries are stored
+ */
+ String SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK = "slider_dep";
+ String SLIDER_DEPENDENCY_HDP_PARENT_DIR = "/hdp";
+ String SLIDER_DEPENDENCY_DIR = "/apps/%s/slider";
+ String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider-dep";
+ String SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
+ String SLIDER_DEPENDENCY_DIR_PERMISSIONS = "755";
+
+ /**
+ *
+ */
+ String HDP_VERSION_PROP_NAME = "HDP_VERSION";
+
+ /**
+ * name of the relative path to expaned an image into: {@value}.
+ * The title of this path is to help people understand it when
+ * they see it in their error messages
+ */
+ String LOCAL_TARBALL_INSTALL_SUBDIR = "expandedarchive";
+
+
+ /**
+ * Application type for YARN {@value}
+ */
+ String APP_TYPE = "org-apache-slider";
+
+ /**
+ * A component type for an external app that has been predefined using the
+ * slider build command
+ */
+ String COMPONENT_SEPARATOR = "-";
+
+ /**
+ * A component type for a client component
+ */
+ String COMPONENT_TYPE_CLIENT = "client";
+
+ /**
+ * Key for application version.
+ */
+ String APP_VERSION_UNKNOWN = "awaiting heartbeat...";
+
+ /**
+ * Keys for application container specific properties, like release timeout
+ */
+ String APP_CONTAINER_RELEASE_TIMEOUT = "site.global.app_container.release_timeout_secs";
+
+ /**
+ * Subdirectories of HDFS cluster dir.
+ */
+ String DATA_DIR_NAME = "data";
+ String HISTORY_DIR_NAME = "history";
+ String HISTORY_FILENAME_SUFFIX = "json";
+ String HISTORY_FILENAME_PREFIX = "rolehistory-";
+ String KEYTAB_DIR = "keytabs";
+ String RESOURCE_DIR = "resources";
+
+ /**
+ * Filename pattern is required to save in strict temporal order.
+ * Important: older files must sort less-than newer files when using
+ * case-sensitive name sort.
+ */
+ String HISTORY_FILENAME_CREATION_PATTERN = HISTORY_FILENAME_PREFIX +"%016x."+
+ HISTORY_FILENAME_SUFFIX;
+ /**
+ * The posix regexp used to locate this
+ */
+ String HISTORY_FILENAME_MATCH_PATTERN = HISTORY_FILENAME_PREFIX +"[0-9a-f]+\\."+
+ HISTORY_FILENAME_SUFFIX;
+ /**
+ * The posix regexp used to locate this
+ */
+ String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
+ HISTORY_FILENAME_SUFFIX;
+
+ String CLUSTER_DIRECTORY = "cluster";
+
+ /**
+ * JVM property to define the slider lib directory;
+ * this is set by the slider script: {@value}
+ */
+ String PROPERTY_LIB_DIR = "slider.libdir";
+
+ /**
+ * name of generated dir for this conf: {@value}
+ */
+ String SUBMITTED_CONF_DIR = "conf";
+
+ /**
+ * Slider AM log4j file name : {@value}
+ */
+ String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties";
+
+ /**
+ * Log4j sysprop to name the resource :{@value}
+ */
+ String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
+
+ /**
+ * sysprop for Slider AM log4j directory :{@value}
+ */
+ String SYSPROP_LOG_DIR = "LOG_DIR";
+
+ /**
+ * name of the Slider client resource
+ * loaded when the service is loaded.
+ */
+ String SLIDER_CLIENT_XML = "slider-client.xml";
+
+ /**
+ * The name of the resource to put on the classpath
+ */
+ String SLIDER_SERVER_XML = "slider-server.xml";
+
+ String TMP_DIR_PREFIX = "tmp";
+
+ /**
+ * Store the default app definition, e.g. metainfo file or content of a folder
+ */
+ String APP_DEF_DIR = "appdef";
+ /**
+ * Store additional app defs - co-processors
+ */
+ String ADDONS_DIR = "addons";
+
+ String SLIDER_JAR = "slider-core.jar";
+
+ String STDOUT_AM = "slider-out.txt";
+ String STDERR_AM = "slider-err.txt";
+
+ String HADOOP_USER_NAME = "HADOOP_USER_NAME";
+
+ /**
+ * Name of the AM filter to use: {@value}
+ */
+ String AM_FILTER_NAME =
+ "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
+
+ String YARN_CONTAINER_PATH = "/node/container/";
+
+ String APP_CONF_DIR = "conf";
+
+ String APP_LIB_DIR = "lib";
+
+ String OUT_FILE = "stdout.txt";
+ String ERR_FILE = "stderr.txt";
+
+ String QUICK_LINKS = "quicklinks";
+
+ String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
new file mode 100644
index 0000000..523e08d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+/**
+ * These are the keys that can be added to <code>conf/slider-client.xml</code>.
+ */
+public interface SliderXmlConfKeys {
+ String PREFIX_PROVIDER = "slider.provider";
+ /**
+ * pattern to identify a provider
+ * {@value}
+ */
+ String KEY_PROVIDER = PREFIX_PROVIDER + ".%s";
+
+ /**
+ * conf option set to point to where the config came from
+ * {@value}
+ */
+ String KEY_TEMPLATE_ORIGIN = "slider.template.origin";
+
+ /**
+ * Original name for the default FS. This is still
+ * expected by applications deployed
+ */
+ String FS_DEFAULT_NAME_CLASSIC = "fs.default.name";
+
+ /**
+ * Slider principal
+ */
+ String KEY_KERBEROS_PRINCIPAL = "slider.kerberos.principal";
+
+ /**
+ * Name of the property for ACLs for Slider AM.
+ * {@value}
+ */
+ String KEY_PROTOCOL_ACL = "slider.security.protocol.acl";
+
+ /**
+ * Limit on restarts for the AM
+ * {@value}
+ */
+ String KEY_AM_RESTART_LIMIT = "slider.yarn.restart.limit";
+
+ /**
+ * queue name, by default let YARN pick the queue
+ */
+ String KEY_YARN_QUEUE = "slider.yarn.queue";
+ String DEFAULT_YARN_QUEUE = null;
+
+ /**
+ * default priority
+ */
+ String KEY_YARN_QUEUE_PRIORITY = "slider.yarn.queue.priority";
+ int DEFAULT_YARN_QUEUE_PRIORITY = 1;
+
+
+ String KEY_AM_RESOURCE_MEM = "slider.am.resource.memory";
+ long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
+
+ /**
+ * The slider base path: {@value}
+ * Defaults to HomeDir/.slider
+ */
+ String KEY_SLIDER_BASE_PATH = "slider.base.path";
+
+
+ /**
+ * Option for the permissions for the cluster directory itself: {@value}
+ */
+ String CLUSTER_DIRECTORY_PERMISSIONS =
+ "slider.cluster.directory.permissions";
+
+ /**
+ * Default value for the permissions :{@value}
+ */
+ String DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS = "750";
+
+ /**
+ *
+ * Option for the permissions for the data directory itself: {@value}
+ */
+ String DATA_DIRECTORY_PERMISSIONS = "slider.data.directory.permissions";
+
+ /**
+ * Default value for the data directory permissions: {@value}
+ */
+ String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750";
+
+ String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH =
+ "ipc.client.fallback-to-simple-auth-allowed";
+ String HADOOP_HTTP_FILTER_INITIALIZERS =
+ "hadoop.http.filter.initializers";
+ String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
+ String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ /** Declare that a keytab must be provided */
+ String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
+ String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
+ String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+ String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
+ String KEY_SECURITY_ENABLED = "site.global.security_enabled";
+
+ /**
+ * Set to disable server-side checks for python, openssl &c.
+ * This should only be set for testing
+ */
+ String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED =
+ "slider.am.dependency.checks.disabled";
+
+ /**
+ * The path to the python executable utilized to launch the agent.
+ */
+ String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path";
+
+ /**
+ * Flag to enable the insecure AM filter: {@value}
+ */
+ String X_DEV_INSECURE_WS = "slider.feature.ws.insecure";
+
+ /**
+ * Flag to indicate the insecure AM filter is enabled by default: {@value}.
+ */
+ boolean X_DEV_INSECURE_DEFAULT = false;
+
+
+ /**
+ * Flag to indicate the insecure AM filter is required for
+ * complex REST Verbs: {@value}.
+ * When Slider switches to being Hadoop 2.7+ only, this flag
+ * can be set to false
+ */
+ boolean X_DEV_INSECURE_REQUIRED = true;
+
+ /**
+ *
+ */
+ String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED =
+ "slider.ipc.client.retry.enabled";
+ boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
+ String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
+ "slider.ipc.client.retry.policy.spec";
+ String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
+ "10000,6,60000,10"; //t1,n1,t2,n2,...
+
+ String KEY_AM_LAUNCH_ENV = "slider.am.launch.env";
+
+ /**
+ * From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY}
+ */
+ String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal";
+
+ String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal";
+
+ //Delegation token related keys
+ String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY
+ = "dfs.namenode.delegation.key.update-interval";
+ long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 *
+ 1000; // 1 day
+ String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY
+ = "dfs.namenode.delegation.token.renew-interval";
+ long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 *
+ 1000; // 1 day
+ String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY
+ = "dfs.namenode.delegation.token.max-lifetime";
+ long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 *
+ 1000; // 7 days
+ String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY
+ = "dfs.namenode.delegation.token.always-use"; // for tests
+ boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
+ String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
+ String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
+
+ String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConfKeys.java
new file mode 100644
index 0000000..4fda686
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConfKeys.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+public interface YarnServiceConfKeys {
+
+ // Retry settings for the ServiceClient to talk to Service AppMaster
+ String CLIENT_AM_RETRY_MAX_WAIT_MS = "yarn.service.client-am.retry.max-wait-ms";
+ String CLIENT_AM_RETRY_MAX_INTERVAL_MS = "yarn.service.client-am.retry-interval-ms";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
new file mode 100644
index 0000000..33e33a6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.client;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.service.impl.pb.service.ClientAMProtocolPB;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+
+public class ClientAMProtocolPBClientImpl
+ implements ClientAMProtocol, Closeable {
+
+ private ClientAMProtocolPB proxy;
+
+ public ClientAMProtocolPBClientImpl(long clientVersion,
+ InetSocketAddress addr, Configuration conf) throws IOException {
+ RPC.setProtocolEngine(conf, ClientAMProtocolPB.class,
+ ProtobufRpcEngine.class);
+ proxy = RPC.getProxy(ClientAMProtocolPB.class, clientVersion, addr, conf);
+
+ }
+
+ @Override public FlexComponentsResponseProto flexComponents(
+ FlexComponentsRequestProto request) throws IOException, YarnException {
+ try {
+ return proxy.flexComponents(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public GetStatusResponseProto getStatus(GetStatusRequestProto request)
+ throws IOException, YarnException {
+ try {
+ return proxy.getStatus(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException {
+ try {
+ return proxy.stop(null, requestProto);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override public void close() {
+ if (this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
new file mode 100644
index 0000000..6a9cd37
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.service;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol;
+
+@ProtocolInfo(
+ protocolName = "org.apache.hadoop.yarn.service.ClientAMProtocol",
+ protocolVersion = 1)
+public interface ClientAMProtocolPB extends
+ ClientAMProtocol.ClientAMProtocolService.BlockingInterface {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
new file mode 100644
index 0000000..7100781
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.service;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+
+import java.io.IOException;
+
+public class ClientAMProtocolPBServiceImpl implements ClientAMProtocolPB {
+
+ private ClientAMProtocol real;
+
+ public ClientAMProtocolPBServiceImpl(ClientAMProtocol impl) {
+ this.real = impl;
+ }
+
+ @Override
+ public FlexComponentsResponseProto flexComponents(RpcController controller,
+ FlexComponentsRequestProto request) throws ServiceException {
+ try {
+ return real.flexComponents(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override public GetStatusResponseProto getStatus(RpcController controller,
+ GetStatusRequestProto request) throws ServiceException {
+ try {
+ return real.getStatus(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto stop(
+ RpcController controller,
+ org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto request)
+ throws ServiceException {
+ try {
+ return real.stop(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
new file mode 100644
index 0000000..bfe3bc9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.metrics;
+
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+@Metrics(context = "yarn-native-service")
+public class ServiceMetrics implements MetricsSource {
+
+ @Metric("containers requested")
+ public MutableGaugeInt containersRequested;
+
+ @Metric("anti-affinity containers pending")
+ public MutableGaugeInt pendingAAContainers;
+
+ @Metric("containers running")
+ public MutableGaugeInt containersRunning;
+
+ @Metric("containers ready")
+ public MutableGaugeInt containersReady;
+
+ @Metric("containers desired")
+ public MutableGaugeInt containersDesired;
+
+ @Metric("containers succeeded")
+ public MutableGaugeInt containersSucceeded;
+
+ @Metric("containers failed")
+ public MutableGaugeInt containersFailed;
+
+ @Metric("containers preempted")
+ public MutableGaugeInt containersPreempted;
+
+ @Metric("containers exceeded limits")
+ public MutableGaugeInt containersLimitsExceeded;
+
+ @Metric("containers surplus")
+ public MutableGaugeInt surplusContainers;
+
+ @Metric("containers failed due to disk failure")
+ public MutableGaugeInt containersDiskFailure;
+
+ protected final MetricsRegistry registry;
+
+ public ServiceMetrics(MetricsInfo metricsInfo) {
+ registry = new MetricsRegistry(metricsInfo);
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ registry.snapshot(collector.addRecord(registry.info()), all);
+ }
+
+ public static ServiceMetrics register(String name, String description) {
+ ServiceMetrics metrics = new ServiceMetrics(info(name, description));
+ DefaultMetricsSystem.instance().register(name, description, metrics);
+ return metrics;
+ }
+
+ public void tag(String name, String description, String value) {
+ registry.tag(name, description, value);
+ }
+
+ @Override public String toString() {
+ return "ServiceMetrics{"
+ + "containersRequested=" + containersRequested.value()
+ + ", pendingAAContainers=" + pendingAAContainers.value()
+ + ", containersRunning=" + containersRunning.value()
+ + ", containersDesired=" + containersDesired.value()
+ + ", containersSucceeded=" + containersSucceeded.value()
+ + ", containersFailed=" + containersFailed.value()
+ + ", containersPreempted=" + containersPreempted.value()
+ + ", surplusContainers=" + surplusContainers.value() + '}';
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
new file mode 100644
index 0000000..6c91a13
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public abstract class AbstractClientProvider {
+
+ public AbstractClientProvider() {
+ }
+
+ /**
+ * Generates a fixed format of application tags given one or more of
+ * application name, version and description. This allows subsequent query for
+ * an application with a name only, version only or description only or any
+ * combination of those as filters.
+ *
+ * @param appName name of the application
+ * @param appVersion version of the application
+ * @param appDescription brief description of the application
+ * @return
+ */
+ public static final Set<String> createApplicationTags(String appName,
+ String appVersion, String appDescription) {
+ Set<String> tags = new HashSet<>();
+ tags.add(SliderUtils.createNameTag(appName));
+ if (appVersion != null) {
+ tags.add(SliderUtils.createVersionTag(appVersion));
+ }
+ if (appDescription != null) {
+ tags.add(SliderUtils.createDescriptionTag(appDescription));
+ }
+ return tags;
+ }
+
+ /**
+ * Validate the artifact.
+ * @param artifact
+ */
+ public abstract void validateArtifact(Artifact artifact, FileSystem
+ fileSystem) throws IOException;
+
+ protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException;
+
+ /**
+ * Validate the config files.
+ * @param configFiles config file list
+ * @param fs file system
+ */
+ public void validateConfigFiles(List<ConfigFile> configFiles,
+ FileSystem fs) throws IOException {
+ Set<String> destFileSet = new HashSet<>();
+
+ for (ConfigFile file : configFiles) {
+ if (file.getType() == null) {
+ throw new IllegalArgumentException("File type is empty");
+ }
+
+ if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
+ .isEmpty(file.getSrcFile())) {
+ throw new IllegalArgumentException(
+ "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+
+ }
+ if (!StringUtils.isEmpty(file.getSrcFile())) {
+ Path p = new Path(file.getSrcFile());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException(
+ "Src_file does not exist for config file: " + file
+ .getSrcFile());
+ }
+ }
+
+ if (StringUtils.isEmpty(file.getDestFile())) {
+ throw new IllegalArgumentException("Dest_file is empty.");
+ }
+
+ if (destFileSet.contains(file.getDestFile())) {
+ throw new IllegalArgumentException(
+ "Duplicated ConfigFile exists: " + file.getDestFile());
+ }
+ destFileSet.add(file.getDestFile());
+
+ java.nio.file.Path destPath = Paths.get(file.getDestFile());
+ if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
+ throw new IllegalArgumentException("Non-absolute dest_file has more " +
+ "than one path element");
+ }
+
+ // provider-specific validation
+ validateConfigFile(file, fs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
new file mode 100644
index 0000000..472ee21
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+public abstract class AbstractProviderService implements ProviderService,
+ SliderKeys {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AbstractProviderService.class);
+
+ public abstract void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Application application)
+ throws IOException;
+
+ public void buildContainerLaunchContext(AbstractLauncher launcher,
+ Application application, ComponentInstance instance,
+ SliderFileSystem fileSystem) throws IOException, SliderException {
+ Component component = instance.getComponent().getComponentSpec();;
+ processArtifact(launcher, instance, fileSystem, application);
+
+ ServiceContext context =
+ instance.getComponent().getScheduler().getContext();
+ // Generate tokens (key-value pair) for config substitution.
+ // Get pre-defined tokens
+ Map<String, String> globalTokens =
+ instance.getComponent().getScheduler().globalTokens;
+ Map<String, String> tokensForSubstitution = ProviderUtils
+ .initCompTokensForSubstitute(instance);
+ tokensForSubstitution.putAll(globalTokens);
+ // Set the environment variables in launcher
+ launcher.putEnv(SliderUtils
+ .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
+ launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
+ launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ if (System.getenv(HADOOP_USER_NAME) != null) {
+ launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
+ }
+ launcher.setEnv("LANG", "en_US.UTF-8");
+ launcher.setEnv("LC_ALL", "en_US.UTF-8");
+ launcher.setEnv("LANGUAGE", "en_US.UTF-8");
+
+ for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
+ tokensForSubstitution.put($(entry.getKey()), entry.getValue());
+ }
+ //TODO add component host tokens?
+// ProviderUtils.addComponentHostTokens(tokensForSubstitution, amState);
+
+ // create config file on hdfs and add local resource
+ ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
+ component, tokensForSubstitution, instance, context);
+
+ // substitute launch command
+ String launchCommand = ProviderUtils
+ .substituteStrWithTokens(component.getLaunchCommand(),
+ tokensForSubstitution);
+ CommandLineBuilder operation = new CommandLineBuilder();
+ operation.add(launchCommand);
+ operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+ launcher.addCommand(operation.build());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
new file mode 100644
index 0000000..b53652a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for factories.
+ */
+public abstract class ProviderFactory {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ProviderFactory.class);
+
+ protected ProviderFactory() {}
+
+ public abstract AbstractClientProvider createClientProvider();
+
+ public abstract ProviderService createServerProvider();
+
+ public static synchronized ProviderService getProviderService(Artifact
+ artifact) {
+ return createSliderProviderFactory(artifact).createServerProvider();
+ }
+
+ public static synchronized AbstractClientProvider getClientProvider(Artifact
+ artifact) {
+ return createSliderProviderFactory(artifact).createClientProvider();
+ }
+
+ /**
+ * Create a provider for a specific application
+ * @param artifact artifact
+ * @return provider factory
+ */
+ public static synchronized ProviderFactory createSliderProviderFactory(
+ Artifact artifact) {
+ if (artifact == null || artifact.getType() == null) {
+ LOG.debug("Loading service provider type default");
+ return DefaultProviderFactory.getInstance();
+ }
+ LOG.debug("Loading service provider type {}", artifact.getType());
+ switch (artifact.getType()) {
+ // TODO add handling for custom types?
+ // TODO handle application
+ case DOCKER:
+ return DockerProviderFactory.getInstance();
+ case TARBALL:
+ return TarballProviderFactory.getInstance();
+ default:
+ throw new IllegalArgumentException(String.format("Resolution error, " +
+ "%s should not be passed to createSliderProviderFactory",
+ artifact.getType()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
new file mode 100644
index 0000000..a28c3b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+
+import java.io.IOException;
+
+public interface ProviderService {
+
+ /**
+ * Set up the entire container launch context
+ */
+ void buildContainerLaunchContext(AbstractLauncher containerLauncher,
+ Application application, ComponentInstance instance,
+ SliderFileSystem sliderFileSystem) throws IOException, SliderException;
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org