You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2014/10/23 02:03:46 UTC
[15/51] [abbrv] [partial] Initial merge of Wake,
Tang and REEF into one repository and project
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendClientControl.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendClientControl.java b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendClientControl.java
new file mode 100644
index 0000000..ee4860c
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendClientControl.java
@@ -0,0 +1,89 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.suspend;
+
+import org.apache.reef.client.RunningJob;
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.wake.EStage;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.impl.ThreadPoolStage;
+import org.apache.reef.wake.remote.impl.ObjectSerializableCodec;
+import org.apache.reef.wake.remote.impl.TransportEvent;
+import org.apache.reef.wake.remote.transport.Transport;
+import org.apache.reef.wake.remote.transport.netty.NettyMessagingTransport;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * (Wake) listener to get suspend/resume commands from Control process.
+ */
+public class SuspendClientControl implements AutoCloseable {
+
+ private static final Logger LOG = Logger.getLogger(Control.class.getName());
+ private static final ObjectSerializableCodec<byte[]> CODEC = new ObjectSerializableCodec<>();
+ private final transient Transport transport;
+ private transient RunningJob runningJob;
+
+ @Inject
+ public SuspendClientControl(
+ final @Parameter(SuspendClientControl.Port.class) int port) throws IOException {
+
+ LOG.log(Level.INFO, "Listen to control port {0}", port);
+
+ final EStage<TransportEvent> stage = new ThreadPoolStage<>(
+ "suspend-control-server", new ControlMessageHandler(), 1, new EventHandler<Throwable>() {
+ @Override
+ public void onNext(final Throwable throwable) {
+ throw new RuntimeException(throwable);
+ }
+ });
+
+ this.transport = new NettyMessagingTransport("localhost", port, stage, stage, 1, 10000);
+ }
+
+ public synchronized void setRunningJob(final RunningJob job) {
+ this.runningJob = job;
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.transport.close();
+ }
+
+ @NamedParameter(doc = "Port for suspend/resume control commands",
+ short_name = "port", default_value = "7008")
+ public static final class Port implements Name<Integer> {
+ }
+
+ /**
+ * Forward remote message to the job driver.
+ */
+ private class ControlMessageHandler implements EventHandler<TransportEvent> {
+ @Override
+ public synchronized void onNext(final TransportEvent msg) {
+ LOG.log(Level.INFO, "Control message: {0} destination: {1}",
+ new Object[]{CODEC.decode(msg.getData()), runningJob});
+ if (runningJob != null) {
+ runningJob.send(msg.getData());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendDriver.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendDriver.java b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendDriver.java
new file mode 100644
index 0000000..519e5dc
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendDriver.java
@@ -0,0 +1,337 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.suspend;
+
+import org.apache.reef.driver.client.JobMessageObserver;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.context.ContextConfiguration;
+import org.apache.reef.driver.evaluator.AllocatedEvaluator;
+import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
+import org.apache.reef.driver.evaluator.EvaluatorRequest;
+import org.apache.reef.driver.evaluator.EvaluatorRequestor;
+import org.apache.reef.driver.task.*;
+import org.apache.reef.io.checkpoint.fs.FSCheckPointServiceConfiguration;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.JavaConfigurationBuilder;
+import org.apache.reef.tang.Tang;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.tang.annotations.Unit;
+import org.apache.reef.tang.exceptions.BindException;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.remote.impl.ObjectSerializableCodec;
+import org.apache.reef.wake.time.event.StartTime;
+import org.apache.reef.wake.time.event.StopTime;
+
+import javax.inject.Inject;
+import javax.xml.bind.DatatypeConverter;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Suspend/resume example job driver. Execute a simple task in all evaluators,
+ * and sendEvaluatorControlMessage suspend/resume events properly.
+ */
+@Unit
+public class SuspendDriver {
+
+ /**
+ * Standard Java logger.
+ */
+ private static final Logger LOG = Logger.getLogger(SuspendDriver.class.getName());
+
+ /**
+ * Number of evaluators to request
+ */
+ private static final int NUM_EVALUATORS = 2;
+
+ /**
+ * String codec is used to encode the results driver sends to the client.
+ */
+ private static final ObjectSerializableCodec<String> CODEC_STR = new ObjectSerializableCodec<>();
+
+ /**
+ * Integer codec is used to decode the results driver gets from the tasks.
+ */
+ private static final ObjectSerializableCodec<Integer> CODEC_INT = new ObjectSerializableCodec<>();
+
+ /**
+ * Job observer on the client.
+ * We use it to send results from the driver back to the client.
+ */
+ private final JobMessageObserver jobMessageObserver;
+
+ /**
+ * Job driver uses EvaluatorRequestor to request Evaluators that will run the Tasks.
+ */
+ private final EvaluatorRequestor evaluatorRequestor;
+
+ /**
+ * TANG Configuration of the Task.
+ */
+ private final Configuration contextConfig;
+
+ /**
+ * Map from task ID (a string) to the TaskRuntime instance (that can be suspended).
+ */
+ private final Map<String, RunningTask> runningTasks =
+ Collections.synchronizedMap(new HashMap<String, RunningTask>());
+
+ /**
+ * Map from task ID (a string) to the SuspendedTask instance (that can be resumed).
+ */
+ private final Map<String, SuspendedTask> suspendedTasks = new HashMap<>();
+
+ /**
+ * Job driver constructor.
+ * All parameters are injected from TANG automatically.
+ *
+ * @param evaluatorRequestor is used to request Evaluators.
+ * @param numCycles number of cycles to run in the task.
+ * @param delay delay in seconds between cycles in the task.
+ */
+ @Inject
+ SuspendDriver(
+ final JobMessageObserver jobMessageObserver,
+ final EvaluatorRequestor evaluatorRequestor,
+ final @Parameter(Launch.Local.class) boolean isLocal,
+ final @Parameter(Launch.NumCycles.class) int numCycles,
+ final @Parameter(Launch.Delay.class) int delay) {
+
+ this.jobMessageObserver = jobMessageObserver;
+ this.evaluatorRequestor = evaluatorRequestor;
+
+ try {
+
+ final Configuration checkpointServiceConfig = FSCheckPointServiceConfiguration.CONF
+ .set(FSCheckPointServiceConfiguration.IS_LOCAL, Boolean.toString(isLocal))
+ .set(FSCheckPointServiceConfiguration.PATH, "/tmp")
+ .set(FSCheckPointServiceConfiguration.PREFIX, "reef-checkpoint-")
+ .set(FSCheckPointServiceConfiguration.REPLICATION_FACTOR, "3")
+ .build();
+
+ final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder()
+ .bindNamedParameter(Launch.NumCycles.class, Integer.toString(numCycles))
+ .bindNamedParameter(Launch.Delay.class, Integer.toString(delay));
+
+ cb.addConfiguration(checkpointServiceConfig);
+ this.contextConfig = cb.build();
+
+ } catch (final BindException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ * Receive notification that the Task is ready to run.
+ */
+ final class RunningTaskHandler implements EventHandler<RunningTask> {
+ @Override
+ public final void onNext(final RunningTask task) {
+ LOG.log(Level.INFO, "Running task: {0}", task.getId());
+ runningTasks.put(task.getId(), task);
+ jobMessageObserver.sendMessageToClient(CODEC_STR.encode("start task: " + task.getId()));
+ }
+ }
+
+ /**
+ * Receive notification that the Task has completed successfully.
+ */
+ final class CompletedTaskHandler implements EventHandler<CompletedTask> {
+ @Override
+ public final void onNext(final CompletedTask task) {
+
+ final EvaluatorDescriptor e = task.getActiveContext().getEvaluatorDescriptor();
+ final String msg = "Task completed " + task.getId() + " on node " + e;
+ LOG.info(msg);
+
+ jobMessageObserver.sendMessageToClient(CODEC_STR.encode(msg));
+ runningTasks.remove(task.getId());
+ task.getActiveContext().close();
+
+ final boolean noTasks;
+
+ synchronized (suspendedTasks) {
+ LOG.log(Level.INFO, "Tasks running: {0} suspended: {1}", new Object[]{
+ runningTasks.size(), suspendedTasks.size()});
+ noTasks = runningTasks.isEmpty() && suspendedTasks.isEmpty();
+ }
+
+ if (noTasks) {
+ LOG.info("All tasks completed; shutting down.");
+ }
+ }
+ }
+
+ /**
+ * Receive notification that the Task has been suspended.
+ */
+ final class SuspendedTaskHandler implements EventHandler<SuspendedTask> {
+ @Override
+ public final void onNext(final SuspendedTask task) {
+
+ final String msg = "Task suspended: " + task.getId();
+ LOG.info(msg);
+
+ synchronized (suspendedTasks) {
+ suspendedTasks.put(task.getId(), task);
+ runningTasks.remove(task.getId());
+ }
+
+ jobMessageObserver.sendMessageToClient(CODEC_STR.encode(msg));
+ }
+ }
+
+ /**
+ * Receive message from the Task.
+ */
+ final class TaskMessageHandler implements EventHandler<TaskMessage> {
+ @Override
+ public void onNext(final TaskMessage message) {
+ final int result = CODEC_INT.decode(message.get());
+ final String msg = "Task message " + message.getId() + ": " + result;
+ LOG.info(msg);
+ jobMessageObserver.sendMessageToClient(CODEC_STR.encode(msg));
+ }
+ }
+
+ /**
+ * Receive notification that an Evaluator had been allocated,
+ * and submitTask a new Task in that Evaluator.
+ */
+ final class AllocatedEvaluatorHandler implements EventHandler<AllocatedEvaluator> {
+ @Override
+ public void onNext(final AllocatedEvaluator eval) {
+ try {
+
+ LOG.log(Level.INFO, "Allocated Evaluator: {0}", eval.getId());
+
+ final Configuration thisContextConfiguration = ContextConfiguration.CONF.set(
+ ContextConfiguration.IDENTIFIER, eval.getId() + "_context").build();
+
+ eval.submitContext(Tang.Factory.getTang()
+ .newConfigurationBuilder(thisContextConfiguration, contextConfig).build());
+
+ } catch (final BindException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ /**
+ * Receive notification that a new Context is available.
+ * Submit a new Task to that Context.
+ */
+ final class ActiveContextHandler implements EventHandler<ActiveContext> {
+ @Override
+ public synchronized void onNext(final ActiveContext context) {
+ LOG.log(Level.INFO, "Active Context: {0}", context.getId());
+ try {
+ context.submitTask(TaskConfiguration.CONF
+ .set(TaskConfiguration.IDENTIFIER, context.getId() + "_task")
+ .set(TaskConfiguration.TASK, SuspendTestTask.class)
+ .set(TaskConfiguration.ON_SUSPEND, SuspendTestTask.SuspendHandler.class)
+ .build());
+ } catch (final BindException ex) {
+ LOG.log(Level.SEVERE, "Bad Task configuration for context: " + context.getId(), ex);
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ /**
+ * Handle notifications from the client.
+ */
+ final class ClientMessageHandler implements EventHandler<byte[]> {
+ @Override
+ public void onNext(final byte[] message) {
+
+ final String commandStr = CODEC_STR.decode(message);
+ LOG.log(Level.INFO, "Client message: {0}", commandStr);
+
+ final String[] split = commandStr.split("\\s+", 2);
+ if (split.length != 2) {
+ throw new IllegalArgumentException("Bad command: " + commandStr);
+ } else {
+
+ final String command = split[0].toLowerCase().intern();
+ final String taskId = split[1];
+
+ switch (command) {
+
+ case "suspend": {
+ final RunningTask task = runningTasks.get(taskId);
+ if (task != null) {
+ task.suspend();
+ } else {
+ throw new IllegalArgumentException("Suspend: Task not found: " + taskId);
+ }
+ break;
+ }
+
+ case "resume": {
+ final SuspendedTask suspendedTask;
+ synchronized (suspendedTasks) {
+ suspendedTask = suspendedTasks.remove(taskId);
+ }
+ if (suspendedTask != null) {
+ try {
+ suspendedTask.getActiveContext().submitTask(TaskConfiguration.CONF
+ .set(TaskConfiguration.IDENTIFIER, taskId)
+ .set(TaskConfiguration.MEMENTO,
+ DatatypeConverter.printBase64Binary(suspendedTask.get()))
+ .build());
+ } catch (final BindException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ throw new IllegalArgumentException("Resume: Task not found: " + taskId);
+ }
+ break;
+ }
+
+ default:
+ throw new IllegalArgumentException("Bad command: " + command);
+ }
+ }
+ }
+ }
+
+ /**
+ * Job Driver is ready and the clock is set up: request the evaluators.
+ */
+ final class StartHandler implements EventHandler<StartTime> {
+ @Override
+ public void onNext(final StartTime time) {
+ LOG.log(Level.INFO, "StartTime: {0}", time);
+ evaluatorRequestor.submit(EvaluatorRequest.newBuilder()
+ .setMemory(128).setNumberOfCores(1).setNumber(NUM_EVALUATORS).build());
+ }
+ }
+
+ /**
+ * Shutting down the job driver: close the evaluators.
+ */
+ final class StopHandler implements EventHandler<StopTime> {
+ @Override
+ public void onNext(final StopTime time) {
+ LOG.log(Level.INFO, "StopTime: {0}", time);
+ jobMessageObserver.sendMessageToClient(CODEC_STR.encode("got StopTime"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendTestTask.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendTestTask.java b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendTestTask.java
new file mode 100644
index 0000000..0acbac1
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendTestTask.java
@@ -0,0 +1,176 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.suspend;
+
+import org.apache.reef.io.checkpoint.CheckpointID;
+import org.apache.reef.io.checkpoint.CheckpointService;
+import org.apache.reef.io.checkpoint.CheckpointService.CheckpointReadChannel;
+import org.apache.reef.io.checkpoint.CheckpointService.CheckpointWriteChannel;
+import org.apache.reef.io.checkpoint.fs.FSCheckpointID;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.tang.annotations.Unit;
+import org.apache.reef.task.Task;
+import org.apache.reef.task.TaskMessage;
+import org.apache.reef.task.TaskMessageSource;
+import org.apache.reef.task.events.SuspendEvent;
+import org.apache.reef.util.Optional;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.remote.impl.ObjectSerializableCodec;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Simple do-nothing task that can send messages to the Driver and can be suspended/resumed.
+ */
+@Unit
+public class SuspendTestTask implements Task, TaskMessageSource {
+
+ /**
+ * Standard java logger.
+ */
+ private static final Logger LOG = Logger.getLogger(SuspendTestTask.class.getName());
+ private final CheckpointService checkpointService;
+ /**
+ * number of cycles to run in the task.
+ */
+ private final int numCycles;
+ /**
+ * delay in milliseconds between cycles in the task.
+ */
+ private final int delay;
+ /**
+ * Codec to serialize/deserialize counter values for the updates.
+ */
+ private final ObjectSerializableCodec<Integer> codecInt = new ObjectSerializableCodec<>();
+ /**
+ * Codec to serialize/deserialize checkpoint IDs for suspend/resume.
+ */
+ private final ObjectWritableCodec<CheckpointID> codecCheckpoint =
+ new ObjectWritableCodec<CheckpointID>(FSCheckpointID.class);
+ /**
+ * Current value of the counter.
+ */
+ private int counter = 0;
+ /**
+ * True if the suspend message has been received, false otherwise.
+ */
+ private boolean suspended = false;
+
+ /**
+ * Task constructor: invoked by TANG.
+ *
+ * @param numCycles number of cycles to run in the task.
+ * @param delay delay in seconds between cycles in the task.
+ */
+ @Inject
+ public SuspendTestTask(
+ final CheckpointService checkpointService,
+ @Parameter(Launch.NumCycles.class) final int numCycles,
+ @Parameter(Launch.Delay.class) final int delay) {
+ this.checkpointService = checkpointService;
+ this.numCycles = numCycles;
+ this.delay = delay * 1000;
+ }
+
+ /**
+ * Main method of the task: run cycle from 0 to numCycles,
+ * and sleep for delay seconds on each cycle.
+ *
+ * @param memento serialized version of the counter.
+ * Empty array for initial run, but can contain value for resumed job.
+ * @return serialized version of the counter.
+ */
+ @Override
+ public synchronized byte[] call(final byte[] memento) throws IOException, InterruptedException {
+
+ LOG.log(Level.INFO, "Start: {0} counter: {1}/{2}",
+ new Object[]{this, this.counter, this.numCycles});
+
+ if (memento != null && memento.length > 0) {
+ this.restore(memento);
+ }
+
+ this.suspended = false;
+ for (; this.counter < this.numCycles && !this.suspended; ++this.counter) {
+ try {
+ LOG.log(Level.INFO, "Run: {0} counter: {1}/{2} sleep: {3}",
+ new Object[]{this, this.counter, this.numCycles, this.delay});
+ this.wait(this.delay);
+ } catch (final InterruptedException ex) {
+ LOG.log(Level.INFO, "{0} interrupted. counter: {1}: {2}",
+ new Object[]{this, this.counter, ex});
+ }
+ }
+
+ return this.suspended ? this.save() : this.codecInt.encode(this.counter);
+ }
+
+ /**
+ * Update driver on current state of the task.
+ *
+ * @return serialized version of the counter.
+ */
+ @Override
+ public synchronized Optional<TaskMessage> getMessage() {
+ LOG.log(Level.INFO, "Message from Task {0} to the Driver: counter: {1}",
+ new Object[]{this, this.counter});
+ return Optional.of(TaskMessage.from(SuspendTestTask.class.getName(), this.codecInt.encode(this.counter)));
+ }
+
+ /**
+ * Save current state of the task in the checkpoint.
+ *
+ * @return checkpoint ID (serialized)
+ */
+ private synchronized byte[] save() throws IOException, InterruptedException {
+ try (final CheckpointWriteChannel channel = this.checkpointService.create()) {
+ channel.write(ByteBuffer.wrap(this.codecInt.encode(this.counter)));
+ return this.codecCheckpoint.encode(this.checkpointService.commit(channel));
+ }
+ }
+
+ /**
+ * Restore the task state from the given checkpoint.
+ *
+ * @param memento serialized checkpoint ID
+ */
+ private synchronized void restore(final byte[] memento) throws IOException, InterruptedException {
+ final CheckpointID checkpointId = this.codecCheckpoint.decode(memento);
+ try (final CheckpointReadChannel channel = this.checkpointService.open(checkpointId)) {
+ final ByteBuffer buffer = ByteBuffer.wrap(this.codecInt.encode(this.counter));
+ channel.read(buffer);
+ this.counter = this.codecInt.decode(buffer.array());
+ }
+ this.checkpointService.delete(checkpointId);
+ }
+
+ public class SuspendHandler implements EventHandler<SuspendEvent> {
+
+ @Override
+ public void onNext(SuspendEvent suspendEvent) {
+ final byte[] message = suspendEvent.get().get();
+ LOG.log(Level.INFO, "Suspend: {0} with: {1} bytes; counter: {2}",
+ new Object[]{this, message.length, SuspendTestTask.this.counter});
+ SuspendTestTask.this.suspended = true;
+ SuspendTestTask.this.notify();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/suspend/package-info.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/suspend/package-info.java b/reef-examples/src/main/java/org/apache/reef/examples/suspend/package-info.java
new file mode 100644
index 0000000..a38fee9
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/suspend/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * suspend/resume demo.
+ */
+package org.apache.reef.examples.suspend;
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/BlockingEventHandler.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/BlockingEventHandler.java b/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/BlockingEventHandler.java
new file mode 100644
index 0000000..ec6526b
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/BlockingEventHandler.java
@@ -0,0 +1,60 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.utils.wake;
+
+import org.apache.reef.wake.EventHandler;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An EventHandler that blocks until a set number of Events has been received.
+ * Once they have been received, the downstream event handler is called with an
+ * Iterable of the events spooled.
+ *
+ * @param <T>
+ */
+public final class BlockingEventHandler<T> implements EventHandler<T> {
+
+ private final int expectedSize;
+ private final EventHandler<Iterable<T>> destination;
+ private List<T> events = new ArrayList<>();
+
+ public BlockingEventHandler(final int expectedSize, final EventHandler<Iterable<T>> destination) {
+ this.expectedSize = expectedSize;
+ this.destination = destination;
+ }
+
+ @Override
+ public final void onNext(final T event) {
+ if (this.isComplete()) {
+ throw new IllegalStateException("Received more Events than expected");
+ }
+ this.events.add(event);
+ if (this.isComplete()) {
+ this.destination.onNext(events);
+ this.reset();
+ }
+ }
+
+ private boolean isComplete() {
+ return this.events.size() >= expectedSize;
+ }
+
+ private void reset() {
+ this.events = new ArrayList<>();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/LoggingEventHandler.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/LoggingEventHandler.java b/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/LoggingEventHandler.java
new file mode 100644
index 0000000..ce87c6e
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/LoggingEventHandler.java
@@ -0,0 +1,55 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.utils.wake;
+
+import org.apache.reef.wake.EventHandler;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * An EventHandler that logs its events before handing it to a downstream
+ * EventHandler.
+ *
+ * @param <T>
+ */
+public class LoggingEventHandler<T> implements EventHandler<T> {
+
+ private final EventHandler<T> downstreamEventHandler;
+ private final String prefix;
+ private final String suffix;
+
+ /**
+ * @param prefix to be logged before the event
+ * @param downstreamEventHandler the event handler to hand the event to
+ * @param suffix to be logged after the event
+ */
+ public LoggingEventHandler(final String prefix, EventHandler<T> downstreamEventHandler, final String suffix) {
+ this.downstreamEventHandler = downstreamEventHandler;
+ this.prefix = prefix;
+ this.suffix = suffix;
+ }
+
+ public LoggingEventHandler(final EventHandler<T> downstreamEventHandler) {
+ this("", downstreamEventHandler, "");
+ }
+
+ @Override
+ public void onNext(final T value) {
+ Logger.getLogger(LoggingEventHandler.class.getName()).log(Level.INFO, prefix + value.toString() + suffix);
+ this.downstreamEventHandler.onNext(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/test/java/com/microsoft/reef/examples/hello/HelloHttpTest.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/test/java/com/microsoft/reef/examples/hello/HelloHttpTest.java b/reef-examples/src/test/java/com/microsoft/reef/examples/hello/HelloHttpTest.java
deleted file mode 100644
index f7cc788..0000000
--- a/reef-examples/src/test/java/com/microsoft/reef/examples/hello/HelloHttpTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.microsoft.reef.examples.hello;
-
-import com.microsoft.reef.client.LauncherStatus;
-import com.microsoft.reef.examples.hellohttp.HelloREEFHttp;
-import com.microsoft.reef.runtime.local.client.LocalRuntimeConfiguration;
-import com.microsoft.tang.Configuration;
-import com.microsoft.tang.exceptions.BindException;
-import com.microsoft.tang.exceptions.InjectionException;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class HelloHttpTest {
- @Test
- public void testHttpServer() throws BindException, InjectionException {
-
- final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
- .set(LocalRuntimeConfiguration.NUMBER_OF_THREADS, 2)
- .build();
-
- final LauncherStatus status = HelloREEFHttp.runHelloReef(runtimeConfiguration, 10 * 1000);
- Assert.assertEquals(LauncherStatus.FORCE_CLOSED, status); // must be force closed by timeout
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/test/java/com/microsoft/reef/examples/suspend/ObjectWritableCodecTest.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/test/java/com/microsoft/reef/examples/suspend/ObjectWritableCodecTest.java b/reef-examples/src/test/java/com/microsoft/reef/examples/suspend/ObjectWritableCodecTest.java
deleted file mode 100644
index 98739cb..0000000
--- a/reef-examples/src/test/java/com/microsoft/reef/examples/suspend/ObjectWritableCodecTest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.examples.suspend;
-
-import com.microsoft.reef.io.checkpoint.CheckpointID;
-import com.microsoft.reef.io.checkpoint.fs.FSCheckpointID;
-import org.apache.hadoop.fs.Path;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class ObjectWritableCodecTest {
-
- private static ObjectWritableCodec<CheckpointID> codec;
-
- /**
- * Test class setup - create the codec.
- */
- @BeforeClass
- public static void setUpClass() {
- codec = new ObjectWritableCodec<CheckpointID>(FSCheckpointID.class);
- }
-
- /**
- * After the encode/decode cycle result equals to the original object.
- */
- @Test
- public void testFSCheckpointIdCodec() {
- final CheckpointID checkpoint1 = new FSCheckpointID(new Path("path"));
- final byte[] serialized = codec.encode(checkpoint1);
- final CheckpointID checkpoint2 = codec.decode(serialized);
- Assert.assertEquals(checkpoint1, checkpoint2);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/test/java/org/apache/reef/examples/hello/HelloHttpTest.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/test/java/org/apache/reef/examples/hello/HelloHttpTest.java b/reef-examples/src/test/java/org/apache/reef/examples/hello/HelloHttpTest.java
new file mode 100644
index 0000000..eb874c7
--- /dev/null
+++ b/reef-examples/src/test/java/org/apache/reef/examples/hello/HelloHttpTest.java
@@ -0,0 +1,39 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.reef.examples.hello;
+
+import org.apache.reef.client.LauncherStatus;
+import org.apache.reef.examples.hellohttp.HelloREEFHttp;
+import org.apache.reef.runtime.local.client.LocalRuntimeConfiguration;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.exceptions.BindException;
+import org.apache.reef.tang.exceptions.InjectionException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HelloHttpTest {
+ @Test
+ public void testHttpServer() throws BindException, InjectionException {
+
+ final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
+ .set(LocalRuntimeConfiguration.NUMBER_OF_THREADS, 2)
+ .build();
+
+ final LauncherStatus status = HelloREEFHttp.runHelloReef(runtimeConfiguration, 10 * 1000);
+ Assert.assertEquals(LauncherStatus.FORCE_CLOSED, status); // must be force closed by timeout
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/test/java/org/apache/reef/examples/suspend/ObjectWritableCodecTest.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/test/java/org/apache/reef/examples/suspend/ObjectWritableCodecTest.java b/reef-examples/src/test/java/org/apache/reef/examples/suspend/ObjectWritableCodecTest.java
new file mode 100644
index 0000000..6e98be9
--- /dev/null
+++ b/reef-examples/src/test/java/org/apache/reef/examples/suspend/ObjectWritableCodecTest.java
@@ -0,0 +1,47 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.suspend;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.reef.io.checkpoint.CheckpointID;
+import org.apache.reef.io.checkpoint.fs.FSCheckpointID;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ObjectWritableCodecTest {
+
+ private static ObjectWritableCodec<CheckpointID> codec;
+
+ /**
+ * Test class setup - create the codec.
+ */
+ @BeforeClass
+ public static void setUpClass() {
+ codec = new ObjectWritableCodec<CheckpointID>(FSCheckpointID.class);
+ }
+
+ /**
+ * After the encode/decode cycle result equals to the original object.
+ */
+ @Test
+ public void testFSCheckpointIdCodec() {
+ final CheckpointID checkpoint1 = new FSCheckpointID(new Path("path"));
+ final byte[] serialized = codec.encode(checkpoint1);
+ final CheckpointID checkpoint2 = codec.decode(serialized);
+ Assert.assertEquals(checkpoint1, checkpoint2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/pom.xml
----------------------------------------------------------------------
diff --git a/reef-io/pom.xml b/reef-io/pom.xml
index 0554149..1b83e11 100644
--- a/reef-io/pom.xml
+++ b/reef-io/pom.xml
@@ -1,8 +1,9 @@
<?xml version="1.0"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.microsoft.reef</groupId>
+ <groupId>org.apache.reef</groupId>
<artifactId>reef-project</artifactId>
<version>0.10-SNAPSHOT</version>
</parent>
@@ -48,6 +49,7 @@
<configuration>
<sources>
<source>target/generated-sources/proto</source>
+ <source>target/generated-sources/avro</source>
</sources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/avro/nameservice.avsc
----------------------------------------------------------------------
diff --git a/reef-io/src/main/avro/nameservice.avsc b/reef-io/src/main/avro/nameservice.avsc
index b4f7a26..2b5bb24 100644
--- a/reef-io/src/main/avro/nameservice.avsc
+++ b/reef-io/src/main/avro/nameservice.avsc
@@ -15,7 +15,7 @@
*/
[
{
- "namespace":"com.microsoft.io.network.naming.avro",
+ "namespace":"org.apache.reef.io.network.naming.avro",
"type":"record",
"name":"AvroNamingLookupRequest",
"fields":[
@@ -23,7 +23,7 @@
]
},
{
- "namespace":"com.microsoft.io.network.naming.avro",
+ "namespace":"org.apache.reef.io.network.naming.avro",
"type":"record",
"name":"AvroNamingAssignment",
"fields":[
@@ -33,7 +33,7 @@
]
},
{
- "namespace":"com.microsoft.io.network.naming.avro",
+ "namespace":"org.apache.reef.io.network.naming.avro",
"type":"record",
"name":"AvroNamingLookupResponse",
"fields":[
@@ -41,7 +41,7 @@
]
},
{
- "namespace":"com.microsoft.io.network.naming.avro",
+ "namespace":"org.apache.reef.io.network.naming.avro",
"type":"record",
"name":"AvroNamingRegisterRequest",
"fields":[
@@ -51,7 +51,7 @@
]
},
{
- "namespace":"com.microsoft.io.network.naming.avro",
+ "namespace":"org.apache.reef.io.network.naming.avro",
"type":"record",
"name":"AvroNamingUnRegisterRequest",
"fields":[
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoader.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoader.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoader.java
deleted file mode 100644
index 5537e05..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoader.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.microsoft.reef.annotations.audience.DriverSide;
-import com.microsoft.reef.driver.context.ContextConfiguration;
-import com.microsoft.reef.driver.evaluator.AllocatedEvaluator;
-import com.microsoft.reef.driver.evaluator.EvaluatorRequest;
-import com.microsoft.reef.driver.evaluator.EvaluatorRequestor;
-import com.microsoft.reef.driver.evaluator.FailedEvaluator;
-import com.microsoft.reef.io.data.loading.impl.EvaluatorRequestSerializer;
-import com.microsoft.reef.io.network.util.Pair;
-import com.microsoft.tang.Configuration;
-import com.microsoft.tang.annotations.Parameter;
-import com.microsoft.tang.annotations.Unit;
-import com.microsoft.tang.exceptions.BindException;
-import com.microsoft.wake.EventHandler;
-import com.microsoft.wake.impl.SingleThreadStage;
-import com.microsoft.wake.time.Clock;
-import com.microsoft.wake.time.event.Alarm;
-import com.microsoft.wake.time.event.StartTime;
-
-/**
- * The driver component for the DataLoadingService
- * Also acts as the central point for resource requests
- * All the allocated evaluators pass through this and
- * the ones that need data loading have a context stacked
- * that enables a task to get access to Data via the
- * {@link DataSet}.
- *
- * TODO: Add timeouts
- */
-@DriverSide
-@Unit
-public class DataLoader {
-
- private static final Logger LOG = Logger.getLogger(DataLoader.class.getName());
-
- private final ConcurrentMap<String, Pair<Configuration, Configuration>> submittedDataEvalConfigs = new ConcurrentHashMap<>();
- private final ConcurrentMap<String, Configuration> submittedComputeEvalConfigs = new ConcurrentHashMap<>();
- private final BlockingQueue<Configuration> failedComputeEvalConfigs = new LinkedBlockingQueue<>();
- private final BlockingQueue<Pair<Configuration, Configuration>> failedDataEvalConfigs = new LinkedBlockingQueue<>();
-
- private final AtomicInteger numComputeRequestsToSubmit = new AtomicInteger(0);
-
- private final DataLoadingService dataLoadingService;
- private final int dataEvalMemoryMB;
- private final int dataEvalCore;
- private final EvaluatorRequest computeRequest;
- private final SingleThreadStage<EvaluatorRequest> resourceRequestStage;
- private final ResourceRequestHandler resourceRequestHandler;
- private final int computeEvalMemoryMB;
- private final int computeEvalCore;
- private final EvaluatorRequestor requestor;
-
- @Inject
- public DataLoader(
- final Clock clock,
- final EvaluatorRequestor requestor,
- final DataLoadingService dataLoadingService,
- final @Parameter(DataLoadingRequestBuilder.DataLoadingEvaluatorMemoryMB.class) int dataEvalMemoryMB,
- final @Parameter(DataLoadingRequestBuilder.DataLoadingEvaluatorNumberOfCores.class) int dataEvalCore,
- final @Parameter(DataLoadingRequestBuilder.DataLoadingComputeRequest.class) String serializedComputeRequest) {
-
- // FIXME: Issue #855: We need this alarm to look busy for REEF.
- clock.scheduleAlarm(30000, new EventHandler<Alarm>() {
- @Override
- public void onNext(final Alarm time) {
- LOG.log(Level.FINE, "Received Alarm: {0}", time);
- }
- });
-
- this.requestor = requestor;
- this.dataLoadingService = dataLoadingService;
- this.dataEvalMemoryMB = dataEvalMemoryMB;
- this.dataEvalCore = dataEvalCore;
- this.resourceRequestHandler = new ResourceRequestHandler(requestor);
- this.resourceRequestStage = new SingleThreadStage<>(this.resourceRequestHandler, 2);
-
- if (serializedComputeRequest.equals("NULL")) {
- this.computeRequest = null;
- this.computeEvalMemoryMB = -1;
- computeEvalCore = 1;
- } else {
- this.computeRequest = EvaluatorRequestSerializer.deserialize(serializedComputeRequest);
- this.computeEvalMemoryMB = this.computeRequest.getMegaBytes();
- this.computeEvalCore = this.computeRequest.getNumberOfCores();
- this.numComputeRequestsToSubmit.set(this.computeRequest.getNumber());
-
- this.resourceRequestStage.onNext(this.computeRequest);
- }
-
- this.resourceRequestStage.onNext(getDataLoadingRequest());
- }
-
- private EvaluatorRequest getDataLoadingRequest() {
- return EvaluatorRequest.newBuilder()
- .setNumber(this.dataLoadingService.getNumberOfPartitions())
- .setMemory(this.dataEvalMemoryMB)
- .setNumberOfCores(this.dataEvalCore)
- .build();
- }
-
- public class StartHandler implements EventHandler<StartTime> {
- @Override
- public void onNext(final StartTime startTime) {
- LOG.log(Level.INFO, "StartTime: {0}", startTime);
- resourceRequestHandler.releaseResourceRequestGate();
- }
- }
-
- public class EvaluatorAllocatedHandler implements EventHandler<AllocatedEvaluator> {
-
- @Override
- public void onNext(final AllocatedEvaluator allocatedEvaluator) {
-
- final String evalId = allocatedEvaluator.getId();
- LOG.log(Level.FINEST, "Allocated evaluator: {0}", evalId);
-
- if (!failedComputeEvalConfigs.isEmpty()) {
- LOG.log(Level.FINE, "Failed Compute requests need to be satisfied for {0}", evalId);
- final Configuration conf = failedComputeEvalConfigs.poll();
- if (conf != null) {
- LOG.log(Level.FINE, "Satisfying failed configuration for {0}", evalId);
- allocatedEvaluator.submitContext(conf);
- submittedComputeEvalConfigs.put(evalId, conf);
- return;
- }
- }
-
- if (!failedDataEvalConfigs.isEmpty()) {
- LOG.log(Level.FINE, "Failed Data requests need to be satisfied for {0}", evalId);
- final Pair<Configuration, Configuration> confPair = failedDataEvalConfigs.poll();
- if (confPair != null) {
- LOG.log(Level.FINE, "Satisfying failed configuration for {0}", evalId);
- allocatedEvaluator.submitContextAndService(confPair.first, confPair.second);
- submittedDataEvalConfigs.put(evalId, confPair);
- return;
- }
- }
-
- final int evaluatorsForComputeRequest = numComputeRequestsToSubmit.decrementAndGet();
- LOG.log(Level.FINE, "Evaluators for compute request: {0}", evaluatorsForComputeRequest);
-
- if (evaluatorsForComputeRequest >= 0) {
- try {
- final Configuration idConfiguration = ContextConfiguration.CONF
- .set(ContextConfiguration.IDENTIFIER,
- dataLoadingService.getComputeContextIdPrefix() + evaluatorsForComputeRequest)
- .build();
- LOG.log(Level.FINE, "Submitting Compute Context to {0}", evalId);
- allocatedEvaluator.submitContext(idConfiguration);
- submittedComputeEvalConfigs.put(allocatedEvaluator.getId(), idConfiguration);
- if (evaluatorsForComputeRequest == 0) {
- LOG.log(Level.FINE, "All Compute requests satisfied. Releasing gate");
- resourceRequestHandler.releaseResourceRequestGate();
- }
- } catch (final BindException e) {
- throw new RuntimeException("Unable to bind context id for Compute request", e);
- }
-
- } else {
-
- final Pair<Configuration, Configuration> confPair = new Pair<>(
- dataLoadingService.getContextConfiguration(allocatedEvaluator),
- dataLoadingService.getServiceConfiguration(allocatedEvaluator));
-
- LOG.log(Level.FINE, "Submitting data loading context to {0}", evalId);
- allocatedEvaluator.submitContextAndService(confPair.first, confPair.second);
- submittedDataEvalConfigs.put(allocatedEvaluator.getId(), confPair);
- }
- }
- }
-
- public class EvaluatorFailedHandler implements EventHandler<FailedEvaluator> {
- @Override
- public void onNext(final FailedEvaluator failedEvaluator) {
-
- final String evalId = failedEvaluator.getId();
-
- final Configuration computeConfig = submittedComputeEvalConfigs.remove(evalId);
- if (computeConfig != null) {
-
- LOG.log(Level.INFO, "Received failed compute evaluator: {0}", evalId);
- failedComputeEvalConfigs.add(computeConfig);
-
- requestor.submit(EvaluatorRequest.newBuilder()
- .setMemory(computeEvalMemoryMB).setNumber(1).setNumberOfCores(computeEvalCore).build());
-
- } else {
-
- final Pair<Configuration, Configuration> confPair = submittedDataEvalConfigs.remove(evalId);
- if (confPair != null) {
-
- LOG.log(Level.INFO, "Received failed data evaluator: {0}", evalId);
- failedDataEvalConfigs.add(confPair);
-
- requestor.submit(EvaluatorRequest.newBuilder()
- .setMemory(dataEvalMemoryMB).setNumber(1).setNumberOfCores(dataEvalCore).build());
-
- } else {
-
- LOG.log(Level.SEVERE, "Received unknown failed evaluator " + evalId,
- failedEvaluator.getEvaluatorException());
-
- throw new RuntimeException("Received failed evaluator that I did not submit: " + evalId);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingDriverConfiguration.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingDriverConfiguration.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingDriverConfiguration.java
deleted file mode 100644
index fbacfaa..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingDriverConfiguration.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import com.microsoft.reef.client.DriverConfiguration;
-import com.microsoft.reef.driver.parameters.EvaluatorAllocatedHandlers;
-import com.microsoft.tang.formats.ConfigurationModule;
-import com.microsoft.tang.formats.ConfigurationModuleBuilder;
-import com.microsoft.wake.time.Clock;
-
-public final class DataLoadingDriverConfiguration extends ConfigurationModuleBuilder {
- public static final ConfigurationModule CONF = new DataLoadingDriverConfiguration()
- .merge(DriverConfiguration.CONF)
- .bindSetEntry(Clock.StartHandler.class, DataLoader.StartHandler.class)
- .bindSetEntry(EvaluatorAllocatedHandlers.class, DataLoader.EvaluatorAllocatedHandler.class)
- .build();
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingRequestBuilder.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingRequestBuilder.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingRequestBuilder.java
deleted file mode 100644
index bb380e4..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingRequestBuilder.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.microsoft.reef.client.DriverConfiguration;
-import com.microsoft.reef.driver.evaluator.EvaluatorRequest;
-import com.microsoft.reef.io.data.loading.impl.EvaluatorRequestSerializer;
-import com.microsoft.reef.io.data.loading.impl.InputFormatExternalConstructor;
-import com.microsoft.reef.io.data.loading.impl.InputFormatLoadingService;
-import com.microsoft.reef.io.data.loading.impl.JobConfExternalConstructor;
-import com.microsoft.tang.Configuration;
-import com.microsoft.tang.JavaConfigurationBuilder;
-import com.microsoft.tang.Tang;
-import com.microsoft.tang.annotations.Name;
-import com.microsoft.tang.annotations.NamedParameter;
-import com.microsoft.tang.exceptions.BindException;
-import com.microsoft.tang.formats.ConfigurationModule;
-
-/**
- * Builder to create a request to the DataLoadingService.
- */
-public final class DataLoadingRequestBuilder
- implements com.microsoft.reef.util.Builder<Configuration> {
-
- @NamedParameter(short_name = "num_splits", default_value = "0")
- public static final class NumberOfDesiredSplits implements Name<Integer> {
- }
-
- @NamedParameter(short_name = "dataLoadingEvaluatorMemoryMB", default_value = "4096")
- public static final class DataLoadingEvaluatorMemoryMB implements Name<Integer> {
- }
-
- @NamedParameter(short_name = "dataLoadingEvaluatorCore", default_value = "1")
- public static final class DataLoadingEvaluatorNumberOfCores implements Name<Integer> {
- }
-
- @NamedParameter(default_value = "NULL")
- public static final class DataLoadingComputeRequest implements Name<String> {
- }
-
- @NamedParameter(default_value = "false")
- public static final class LoadDataIntoMemory implements Name<Boolean> {
- }
-
- private int memoryMB = -1;
- private int numberOfCores = -1;
- private int numberOfDesiredSplits = -1;
- private EvaluatorRequest computeRequest = null;
- private boolean inMemory = false;
- private boolean renewFailedEvaluators = true;
- private ConfigurationModule driverConfigurationModule = null;
- private String inputFormatClass;
- private String inputPath;
-
- public DataLoadingRequestBuilder setNumberOfDesiredSplits(final int numberOfDesiredSplits) {
- this.numberOfDesiredSplits = numberOfDesiredSplits;
- return this;
- }
-
- /**
- * Set the memory to be used for Evaluator allocated.
- *
- * @param memoryMB the amount of memory in MB
- * @return this
- */
- public DataLoadingRequestBuilder setMemoryMB(final int memoryMB) {
- this.memoryMB = memoryMB;
- return this;
- }
-
- /**
- * Set the core number to be used for Evaluator allocated.
- *
- * @param numberOfCores the number of cores
- * @return this
- */
- public DataLoadingRequestBuilder setNumberOfCores(final int numberOfCores) {
- this.numberOfCores = numberOfCores;
- return this;
- }
-
- public DataLoadingRequestBuilder setComputeRequest(final EvaluatorRequest computeRequest) {
- this.computeRequest = computeRequest;
- return this;
- }
-
- public DataLoadingRequestBuilder loadIntoMemory(final boolean inMemory) {
- this.inMemory = inMemory;
- return this;
- }
-
- public DataLoadingRequestBuilder renewFailedEvaluators(final boolean renewFailedEvaluators) {
- this.renewFailedEvaluators = renewFailedEvaluators;
- return this;
- }
-
- public DataLoadingRequestBuilder setDriverConfigurationModule(
- final ConfigurationModule driverConfigurationModule) {
- this.driverConfigurationModule = driverConfigurationModule;
- return this;
- }
-
- public DataLoadingRequestBuilder setInputFormatClass(
- final Class<? extends InputFormat> inputFormatClass) {
- this.inputFormatClass = inputFormatClass.getName();
- return this;
- }
-
- public DataLoadingRequestBuilder setInputPath(final String inputPath) {
- this.inputPath = inputPath;
- return this;
- }
-
- @Override
- public Configuration build() throws BindException {
- if (this.driverConfigurationModule == null) {
- throw new BindException("Driver Configuration Module is a required parameter.");
- }
-
- if(this.inputPath == null) {
- throw new BindException("InputPath is a required parameter.");
- }
-
- if(this.inputFormatClass == null) {
- this.inputFormatClass = TextInputFormat.class.getName();
- }
-
- final Configuration driverConfiguration;
- if (renewFailedEvaluators) {
- driverConfiguration = this.driverConfigurationModule
- .set(DriverConfiguration.ON_DRIVER_STARTED, DataLoader.StartHandler.class)
- .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, DataLoader.EvaluatorAllocatedHandler.class)
- .set(DriverConfiguration.ON_EVALUATOR_FAILED, DataLoader.EvaluatorFailedHandler.class)
- .build();
- } else {
- driverConfiguration = this.driverConfigurationModule
- .set(DriverConfiguration.ON_DRIVER_STARTED, DataLoader.StartHandler.class)
- .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, DataLoader.EvaluatorAllocatedHandler.class)
- .build();
- }
-
- final JavaConfigurationBuilder jcb =
- Tang.Factory.getTang().newConfigurationBuilder(driverConfiguration);
-
- if (this.numberOfDesiredSplits > 0) {
- jcb.bindNamedParameter(NumberOfDesiredSplits.class, "" + this.numberOfDesiredSplits);
- }
-
- if (this.memoryMB > 0) {
- jcb.bindNamedParameter(DataLoadingEvaluatorMemoryMB.class, "" + this.memoryMB);
- }
-
- if (this.numberOfCores > 0) {
- jcb.bindNamedParameter(DataLoadingEvaluatorNumberOfCores.class, "" + this.numberOfCores);
- }
-
- if (this.computeRequest != null) {
- jcb.bindNamedParameter(DataLoadingComputeRequest.class,
- EvaluatorRequestSerializer.serialize(this.computeRequest));
- }
-
- return jcb
- .bindNamedParameter(LoadDataIntoMemory.class, Boolean.toString(this.inMemory))
- .bindConstructor(InputFormat.class, InputFormatExternalConstructor.class)
- .bindConstructor(JobConf.class, JobConfExternalConstructor.class)
- .bindNamedParameter(JobConfExternalConstructor.InputFormatClass.class, inputFormatClass)
- .bindNamedParameter(JobConfExternalConstructor.InputPath.class, inputPath)
- .bindImplementation(DataLoadingService.class, InputFormatLoadingService.class)
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingService.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingService.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingService.java
deleted file mode 100644
index f49c7e0..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingService.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import com.microsoft.reef.annotations.audience.DriverSide;
-import com.microsoft.reef.driver.context.ActiveContext;
-import com.microsoft.reef.driver.evaluator.AllocatedEvaluator;
-import com.microsoft.tang.Configuration;
-
-/**
- * All data loading services should implement this interface.
- */
-@DriverSide
-public interface DataLoadingService {
-
- /**
- * Access to the number of partitions suggested by this DataSource.
- *
- * @return the number of partitions suggested by this DataSource.
- */
- int getNumberOfPartitions();
-
- /**
- * @return the context configuration for the given Evaluator.
- */
- Configuration getContextConfiguration(final AllocatedEvaluator allocatedEvaluator);
-
- /**
- * @return the service configuration for the given Evaluator.
- */
- Configuration getServiceConfiguration(final AllocatedEvaluator allocatedEvaluator);
-
- /**
- * @return Return the prefix to be used to enumerate
- * context ids for compute requests fired other than
- * the data load contexts.
- */
- String getComputeContextIdPrefix();
-
- /**
- * Distinguishes data loaded contexts from compute contexts.
- *
- * @return true if this context has been loaded with data.
- */
- boolean isDataLoadedContext(ActiveContext context);
-
- /**
- * @return true if this is a computation context,
- * false otherwise. (e.g. this is a data loading context).
- */
- boolean isComputeContext(ActiveContext context);
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataSet.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataSet.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataSet.java
deleted file mode 100644
index 2ac1c44..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataSet.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import com.microsoft.reef.annotations.audience.TaskSide;
-import com.microsoft.reef.io.network.util.Pair;
-
-/**
- * A view of the data set to be loaded
- * at an evaluator as an iterable of
- * key value pairs.
- *
- * Implementations need not materialize
- * and clients should not assume that the
- * data is materialized. Any such thing
- * is left as a post-processing step.
- *
- * Client also can't assume that the iterator
- * returned here can be restarted
- *
- * @param <K>
- * @param <V>
- */
-@TaskSide
-public interface DataSet<K,V> extends Iterable<Pair<K,V>>{
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/ResourceRequestHandler.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/ResourceRequestHandler.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/ResourceRequestHandler.java
deleted file mode 100644
index e5390ca..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/ResourceRequestHandler.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import com.microsoft.reef.driver.evaluator.EvaluatorRequest;
-import com.microsoft.reef.driver.evaluator.EvaluatorRequestor;
-import com.microsoft.wake.EventHandler;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class ResourceRequestHandler implements EventHandler<EvaluatorRequest> {
-
- private static final Logger LOG = Logger.getLogger(ResourceRequestHandler.class.getName());
-
- private final EvaluatorRequestor requestor;
-
- private CountDownLatch resourceRequestGate = new CountDownLatch(1);
-
- public ResourceRequestHandler(final EvaluatorRequestor requestor) {
- this.requestor = requestor;
- }
-
- public void releaseResourceRequestGate() {
- LOG.log(Level.FINE, "Releasing Gate");
- this.resourceRequestGate.countDown();
- }
-
- @Override
- public void onNext(final EvaluatorRequest request) {
- try {
-
- LOG.log(Level.FINE,
- "Processing a request with count: {0} - Waiting for gate to be released",
- request.getNumber());
-
- this.resourceRequestGate.await();
-
- LOG.log(Level.FINE, "Gate released. Submitting request: {0}", request);
- this.resourceRequestGate = new CountDownLatch(1);
- this.requestor.submit(request);
-
- } catch (final InterruptedException ex) {
- LOG.log(Level.FINEST, "Interrupted", ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorRequestSerializer.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorRequestSerializer.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorRequestSerializer.java
deleted file mode 100644
index faf1bc7..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorRequestSerializer.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.commons.codec.binary.Base64;
-
-import com.microsoft.reef.driver.evaluator.EvaluatorRequest;
-
-/**
- * Serialize and deserialize EvaluatorRequest objects
- * Currently only supports number & memory
- * Does not take care of Resource Descriptor
- */
-public class EvaluatorRequestSerializer {
- public static String serialize(EvaluatorRequest request){
- try(ByteArrayOutputStream baos = new ByteArrayOutputStream()){
- try(DataOutputStream daos = new DataOutputStream(baos)){
-
- daos.writeInt(request.getNumber());
- daos.writeInt(request.getMegaBytes());
- daos.writeInt(request.getNumberOfCores());
-
- } catch (IOException e) {
- throw e;
- }
-
- return Base64.encodeBase64String(baos.toByteArray());
- } catch (IOException e1) {
- throw new RuntimeException("Unable to serialize compute request", e1);
- }
- }
-
- public static EvaluatorRequest deserialize(String serializedRequest){
- try(ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(serializedRequest))){
- try(DataInputStream dais = new DataInputStream(bais)){
- return EvaluatorRequest.newBuilder()
- .setNumber(dais.readInt())
- .setMemory(dais.readInt())
- .setNumberOfCores(dais.readInt())
- .build();
- }
- } catch (IOException e) {
- throw new RuntimeException("Unable to de-serialize compute request", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorToPartitionMapper.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorToPartitionMapper.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorToPartitionMapper.java
deleted file mode 100644
index 88f3343..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorToPartitionMapper.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-
-import com.microsoft.reef.annotations.audience.DriverSide;
-import com.microsoft.reef.io.data.loading.api.DataLoadingService;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Class that tracks the mapping between
- * evaluators & the data partition assigned
- * to those evaluators. Its part of the
- * implementation of a {@link DataLoadingService}
- * that uses the Hadoop {@link InputFormat} to
- * partition the data and request resources
- * accordingly
- *
- * This is an online version which satisfies
- * requests in a greedy way.
- * @param <V>
- */
-@DriverSide
-public class EvaluatorToPartitionMapper<V extends InputSplit> {
- private static final Logger LOG = Logger
- .getLogger(EvaluatorToPartitionMapper.class.getName());
-
- private final ConcurrentMap<String, BlockingQueue<NumberedSplit<V>>> locationToSplits = new ConcurrentHashMap<>();
- private final ConcurrentMap<String, NumberedSplit<V>> evaluatorToSplits = new ConcurrentHashMap<>();
- private final BlockingQueue<NumberedSplit<V>> unallocatedSplits = new LinkedBlockingQueue<>();
-
- /**
- * Initializes the locations of splits mapping
- * @param splits
- */
- public EvaluatorToPartitionMapper(V[] splits) {
- try {
- for (int splitNum = 0; splitNum < splits.length; splitNum++) {
- LOG.log(Level.FINE,"Processing split: " + splitNum);
- final V split = splits[splitNum];
- final String[] locations = split.getLocations();
- final NumberedSplit<V> numberedSplit = new NumberedSplit<V>(split, splitNum);
- unallocatedSplits.add(numberedSplit);
- for (final String location : locations) {
- BlockingQueue<NumberedSplit<V>> newSplitQue = new LinkedBlockingQueue<NumberedSplit<V>>();
- final BlockingQueue<NumberedSplit<V>> splitQue = locationToSplits.putIfAbsent(location,
- newSplitQue);
- if (splitQue != null) {
- newSplitQue = splitQue;
- }
- newSplitQue.add(numberedSplit);
- }
- }
- for (Map.Entry<String, BlockingQueue<NumberedSplit<V>>> locSplit : locationToSplits.entrySet()) {
- LOG.log(Level.FINE,locSplit.getKey() + ": " + locSplit.getValue().toString());
- }
- } catch (IOException e) {
- throw new RuntimeException(
- "Unable to get InputSplits using the specified InputFormat", e);
- }
- }
-
- /**
- * Get an input split to be assigned to this
- * evaluator
- *
- * Allocates one if its not already allocated
- * @param evaluatorId
- * @return
- */
- public NumberedSplit<V> getInputSplit(final String hostName, final String evaluatorId) {
- synchronized (evaluatorToSplits) {
- if (evaluatorToSplits.containsKey(evaluatorId)) {
- LOG.log(Level.FINE,"Found an already allocated partition");
- LOG.log(Level.FINE,evaluatorToSplits.toString());
- return evaluatorToSplits.get(evaluatorId);
- }
- }
- LOG.log(Level.FINE,"allocated partition not found");
- if (locationToSplits.containsKey(hostName)) {
- LOG.log(Level.FINE,"Found partitions possibly hosted for " + evaluatorId + " at " + hostName);
- final NumberedSplit<V> split = allocateSplit(evaluatorId, locationToSplits.get(hostName));
- LOG.log(Level.FINE,evaluatorToSplits.toString());
- if (split != null) {
- return split;
- }
- }
- //pick random
- LOG.log(
- Level.FINE,
- hostName
- + " does not host any partitions or someone else took partitions hosted here. Picking a random one");
- final NumberedSplit<V> split = allocateSplit(evaluatorId, unallocatedSplits);
- LOG.log(Level.FINE,evaluatorToSplits.toString());
- if (split != null) {
- return split;
- }
- throw new RuntimeException("Unable to find an input partition to evaluator " + evaluatorId);
- }
-
- private NumberedSplit<V> allocateSplit(final String evaluatorId,
- final BlockingQueue<NumberedSplit<V>> value) {
- if (value == null) {
- LOG.log(Level.FINE,"Queue of splits can't be empty. Returning null");
- return null;
- }
- while(true){
- final NumberedSplit<V> split = value.poll();
- if(split==null)
- return null;
- if (value == unallocatedSplits || unallocatedSplits.remove(split)) {
- LOG.log(Level.FINE,"Found split-" + split.getIndex() + " in the queue");
- final NumberedSplit<V> old = evaluatorToSplits.putIfAbsent(evaluatorId, split);
- if (old != null) {
- final String msg = "Trying to assign different partitions to the same evaluator " +
- "is not supported";
- LOG.severe(msg);
- throw new RuntimeException(msg);
- } else {
- LOG.log(Level.FINE,"Returning " + split.getIndex());
- return split;
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InMemoryInputFormatDataSet.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InMemoryInputFormatDataSet.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InMemoryInputFormatDataSet.java
deleted file mode 100644
index 480c71c..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InMemoryInputFormatDataSet.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.inject.Inject;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.microsoft.reef.io.data.loading.api.DataSet;
-import com.microsoft.reef.io.network.util.Pair;
-
-public class InMemoryInputFormatDataSet<K extends WritableComparable<K>,V extends Writable>
- implements DataSet<K,V> {
-
- private final InputFormatDataSet<K, V> inputFormatDataSet;
- private List<Pair<K, V>> recordsList = null;
-
- @Inject
- public InMemoryInputFormatDataSet(InputFormatDataSet<K,V> inputFormatDataSet) {
- this.inputFormatDataSet = inputFormatDataSet;
- }
-
-
-
- @Override
- public synchronized Iterator<Pair<K, V>> iterator() {
- if (recordsList == null) {
- recordsList = new ArrayList<>();
- for (final Pair<K,V> keyValue : inputFormatDataSet) {
- recordsList.add(keyValue);
- }
- }
- return recordsList.iterator();
- }}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatDataSet.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatDataSet.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatDataSet.java
deleted file mode 100644
index 1791c2a..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatDataSet.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import javax.inject.Inject;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.Counters.Counter;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.microsoft.reef.annotations.audience.TaskSide;
-import com.microsoft.reef.io.data.loading.api.DataSet;
-import com.microsoft.reef.io.network.util.Pair;
-
-/**
- * An implementation of {@link DataSet} that reads records using a RecordReader
- * encoded inside an InputSplit.
- * <p/>
- * The input split is injected through an external constructor by deserializing
- * the input split assigned to this evaluator.
- *
- * @param <K>
- * @param <V>
- */
-@TaskSide
-public final class
- InputFormatDataSet<K extends WritableComparable<K>, V extends Writable>
- implements DataSet<K, V> {
-
- private final DummyReporter dummyReporter = new DummyReporter();
- private RecordReader lastRecordReader = null;
-
- private final JobConf jobConf;
- private final InputFormat<K, V> inputFormat;
- private final InputSplit split;
-
- @Inject
- public InputFormatDataSet(final InputSplit split, final JobConf jobConf) {
- this.jobConf = jobConf;
- this.inputFormat = this.jobConf.getInputFormat();
- this.split = split;
- }
-
- @Override
- public Iterator<Pair<K, V>> iterator() {
- try {
-
- final RecordReader newRecordReader =
- this.inputFormat.getRecordReader(this.split, this.jobConf, this.dummyReporter);
-
- if (newRecordReader == this.lastRecordReader) {
- throw new RuntimeException("Received the same record reader again. This isn't supported.");
- }
-
- this.lastRecordReader = newRecordReader;
- return new RecordReaderIterator(newRecordReader);
-
- } catch (final IOException ex) {
- throw new RuntimeException("Can't instantiate iterator.", ex);
- }
- }
-
- private final class RecordReaderIterator implements Iterator<Pair<K, V>> {
-
- private final RecordReader<K, V> recordReader;
- private Pair<K, V> recordPair;
- private boolean hasNext;
-
- RecordReaderIterator(final RecordReader<K, V> recordReader) {
- this.recordReader = recordReader;
- fetchRecord();
- }
-
- @Override
- public boolean hasNext() {
- return this.hasNext;
- }
-
- @Override
- public Pair<K, V> next() {
- final Pair<K, V> prevRecordPair = this.recordPair;
- fetchRecord();
- return prevRecordPair;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("Remove is not supported on RecordReader iterator");
- }
-
- private void fetchRecord() {
- this.recordPair = new Pair<>(this.recordReader.createKey(), this.recordReader.createValue());
- try {
- this.hasNext = this.recordReader.next(this.recordPair.first, this.recordPair.second);
- } catch (final IOException ex) {
- throw new RuntimeException("Unable to get InputSplits using the specified InputFormat", ex);
- }
- }
- }
-
- private final class DummyReporter implements Reporter {
-
- @Override
- public void progress() {
- }
-
- @Override
- public Counter getCounter(final Enum<?> key) {
- return null;
- }
-
- @Override
- public Counter getCounter(final String group, final String name) {
- return null;
- }
-
- @Override
- public InputSplit getInputSplit() throws UnsupportedOperationException {
- throw new UnsupportedOperationException("This is a Fake Reporter");
- }
-
- @Override
- public float getProgress() {
- return 0;
- }
-
- @Override
- public void incrCounter(final Enum<?> key, final long amount) {
- }
-
- @Override
- public void incrCounter(final String group, final String counter, final long amount) {
- }
-
- @Override
- public void setStatus(final String status) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatExternalConstructor.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatExternalConstructor.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatExternalConstructor.java
deleted file mode 100644
index 7247731..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatExternalConstructor.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import javax.inject.Inject;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.JobConf;
-
-import com.microsoft.reef.annotations.audience.DriverSide;
-import com.microsoft.tang.ExternalConstructor;
-
-
-/**
- * A Tang External Constructor to inject the required
- * InputFormat
- */
-@DriverSide
-public class InputFormatExternalConstructor implements ExternalConstructor<InputFormat<?,?>> {
-
- private final JobConf jobConf;
- private final InputFormat<?,?> inputFormat;
-
- @Inject
- public InputFormatExternalConstructor(final JobConf jobConf) {
- this.jobConf = jobConf;
- inputFormat = jobConf.getInputFormat();
- }
-
- @Override
- public InputFormat<?,?> newInstance() {
- return inputFormat;
- }
-
-}