You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2014/10/23 02:03:46 UTC

[15/51] [abbrv] [partial] Initial merge of Wake, Tang and REEF into one repository and project

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendClientControl.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendClientControl.java b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendClientControl.java
new file mode 100644
index 0000000..ee4860c
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendClientControl.java
@@ -0,0 +1,89 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.suspend;
+
+import org.apache.reef.client.RunningJob;
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.wake.EStage;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.impl.ThreadPoolStage;
+import org.apache.reef.wake.remote.impl.ObjectSerializableCodec;
+import org.apache.reef.wake.remote.impl.TransportEvent;
+import org.apache.reef.wake.remote.transport.Transport;
+import org.apache.reef.wake.remote.transport.netty.NettyMessagingTransport;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * (Wake) listener to get suspend/resume commands from Control process.
+ */
+public class SuspendClientControl implements AutoCloseable {
+
+  private static final Logger LOG = Logger.getLogger(Control.class.getName());
+  private static final ObjectSerializableCodec<byte[]> CODEC = new ObjectSerializableCodec<>();
+  private final transient Transport transport;
+  private transient RunningJob runningJob;
+
+  @Inject
+  public SuspendClientControl(
+      final @Parameter(SuspendClientControl.Port.class) int port) throws IOException {
+
+    LOG.log(Level.INFO, "Listen to control port {0}", port);
+
+    final EStage<TransportEvent> stage = new ThreadPoolStage<>(
+        "suspend-control-server", new ControlMessageHandler(), 1, new EventHandler<Throwable>() {
+      @Override
+      public void onNext(final Throwable throwable) {
+        throw new RuntimeException(throwable);
+      }
+    });
+
+    this.transport = new NettyMessagingTransport("localhost", port, stage, stage, 1, 10000);
+  }
+
+  public synchronized void setRunningJob(final RunningJob job) {
+    this.runningJob = job;
+  }
+
+  @Override
+  public void close() throws Exception {
+    this.transport.close();
+  }
+
+  @NamedParameter(doc = "Port for suspend/resume control commands",
+      short_name = "port", default_value = "7008")
+  public static final class Port implements Name<Integer> {
+  }
+
+  /**
+   * Forward remote message to the job driver.
+   */
+  private class ControlMessageHandler implements EventHandler<TransportEvent> {
+    @Override
+    public synchronized void onNext(final TransportEvent msg) {
+      LOG.log(Level.INFO, "Control message: {0} destination: {1}",
+          new Object[]{CODEC.decode(msg.getData()), runningJob});
+      if (runningJob != null) {
+        runningJob.send(msg.getData());
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendDriver.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendDriver.java b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendDriver.java
new file mode 100644
index 0000000..519e5dc
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendDriver.java
@@ -0,0 +1,337 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.suspend;
+
+import org.apache.reef.driver.client.JobMessageObserver;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.context.ContextConfiguration;
+import org.apache.reef.driver.evaluator.AllocatedEvaluator;
+import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
+import org.apache.reef.driver.evaluator.EvaluatorRequest;
+import org.apache.reef.driver.evaluator.EvaluatorRequestor;
+import org.apache.reef.driver.task.*;
+import org.apache.reef.io.checkpoint.fs.FSCheckPointServiceConfiguration;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.JavaConfigurationBuilder;
+import org.apache.reef.tang.Tang;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.tang.annotations.Unit;
+import org.apache.reef.tang.exceptions.BindException;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.remote.impl.ObjectSerializableCodec;
+import org.apache.reef.wake.time.event.StartTime;
+import org.apache.reef.wake.time.event.StopTime;
+
+import javax.inject.Inject;
+import javax.xml.bind.DatatypeConverter;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Suspend/resume example job driver. Execute a simple task in all evaluators,
+ * and sendEvaluatorControlMessage suspend/resume events properly.
+ */
+@Unit
+public class SuspendDriver {
+
+  /**
+   * Standard Java logger.
+   */
+  private static final Logger LOG = Logger.getLogger(SuspendDriver.class.getName());
+
+  /**
+   * Number of evaluators to request
+   */
+  private static final int NUM_EVALUATORS = 2;
+
+  /**
+   * String codec is used to encode the results driver sends to the client.
+   */
+  private static final ObjectSerializableCodec<String> CODEC_STR = new ObjectSerializableCodec<>();
+
+  /**
+   * Integer codec is used to decode the results driver gets from the tasks.
+   */
+  private static final ObjectSerializableCodec<Integer> CODEC_INT = new ObjectSerializableCodec<>();
+
+  /**
+   * Job observer on the client.
+   * We use it to send results from the driver back to the client.
+   */
+  private final JobMessageObserver jobMessageObserver;
+
+  /**
+   * Job driver uses EvaluatorRequestor to request Evaluators that will run the Tasks.
+   */
+  private final EvaluatorRequestor evaluatorRequestor;
+
+  /**
+   * TANG Configuration of the Task.
+   */
+  private final Configuration contextConfig;
+
+  /**
+   * Map from task ID (a string) to the TaskRuntime instance (that can be suspended).
+   */
+  private final Map<String, RunningTask> runningTasks =
+      Collections.synchronizedMap(new HashMap<String, RunningTask>());
+
+  /**
+   * Map from task ID (a string) to the SuspendedTask instance (that can be resumed).
+   */
+  private final Map<String, SuspendedTask> suspendedTasks = new HashMap<>();
+
+  /**
+   * Job driver constructor.
+   * All parameters are injected from TANG automatically.
+   *
+   * @param evaluatorRequestor is used to request Evaluators.
+   * @param numCycles          number of cycles to run in the task.
+   * @param delay              delay in seconds between cycles in the task.
+   */
+  @Inject
+  SuspendDriver(
+      final JobMessageObserver jobMessageObserver,
+      final EvaluatorRequestor evaluatorRequestor,
+      final @Parameter(Launch.Local.class) boolean isLocal,
+      final @Parameter(Launch.NumCycles.class) int numCycles,
+      final @Parameter(Launch.Delay.class) int delay) {
+
+    this.jobMessageObserver = jobMessageObserver;
+    this.evaluatorRequestor = evaluatorRequestor;
+
+    try {
+
+      final Configuration checkpointServiceConfig = FSCheckPointServiceConfiguration.CONF
+          .set(FSCheckPointServiceConfiguration.IS_LOCAL, Boolean.toString(isLocal))
+          .set(FSCheckPointServiceConfiguration.PATH, "/tmp")
+          .set(FSCheckPointServiceConfiguration.PREFIX, "reef-checkpoint-")
+          .set(FSCheckPointServiceConfiguration.REPLICATION_FACTOR, "3")
+          .build();
+
+      final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder()
+          .bindNamedParameter(Launch.NumCycles.class, Integer.toString(numCycles))
+          .bindNamedParameter(Launch.Delay.class, Integer.toString(delay));
+
+      cb.addConfiguration(checkpointServiceConfig);
+      this.contextConfig = cb.build();
+
+    } catch (final BindException ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+
+  /**
+   * Receive notification that the Task is ready to run.
+   */
+  final class RunningTaskHandler implements EventHandler<RunningTask> {
+    @Override
+    public final void onNext(final RunningTask task) {
+      LOG.log(Level.INFO, "Running task: {0}", task.getId());
+      runningTasks.put(task.getId(), task);
+      jobMessageObserver.sendMessageToClient(CODEC_STR.encode("start task: " + task.getId()));
+    }
+  }
+
+  /**
+   * Receive notification that the Task has completed successfully.
+   */
+  final class CompletedTaskHandler implements EventHandler<CompletedTask> {
+    @Override
+    public final void onNext(final CompletedTask task) {
+
+      final EvaluatorDescriptor e = task.getActiveContext().getEvaluatorDescriptor();
+      final String msg = "Task completed " + task.getId() + " on node " + e;
+      LOG.info(msg);
+
+      jobMessageObserver.sendMessageToClient(CODEC_STR.encode(msg));
+      runningTasks.remove(task.getId());
+      task.getActiveContext().close();
+
+      final boolean noTasks;
+
+      synchronized (suspendedTasks) {
+        LOG.log(Level.INFO, "Tasks running: {0} suspended: {1}", new Object[]{
+            runningTasks.size(), suspendedTasks.size()});
+        noTasks = runningTasks.isEmpty() && suspendedTasks.isEmpty();
+      }
+
+      if (noTasks) {
+        LOG.info("All tasks completed; shutting down.");
+      }
+    }
+  }
+
+  /**
+   * Receive notification that the Task has been suspended.
+   */
+  final class SuspendedTaskHandler implements EventHandler<SuspendedTask> {
+    @Override
+    public final void onNext(final SuspendedTask task) {
+
+      final String msg = "Task suspended: " + task.getId();
+      LOG.info(msg);
+
+      synchronized (suspendedTasks) {
+        suspendedTasks.put(task.getId(), task);
+        runningTasks.remove(task.getId());
+      }
+
+      jobMessageObserver.sendMessageToClient(CODEC_STR.encode(msg));
+    }
+  }
+
+  /**
+   * Receive message from the Task.
+   */
+  final class TaskMessageHandler implements EventHandler<TaskMessage> {
+    @Override
+    public void onNext(final TaskMessage message) {
+      final int result = CODEC_INT.decode(message.get());
+      final String msg = "Task message " + message.getId() + ": " + result;
+      LOG.info(msg);
+      jobMessageObserver.sendMessageToClient(CODEC_STR.encode(msg));
+    }
+  }
+
+  /**
+   * Receive notification that an Evaluator had been allocated,
+   * and submitTask a new Task in that Evaluator.
+   */
+  final class AllocatedEvaluatorHandler implements EventHandler<AllocatedEvaluator> {
+    @Override
+    public void onNext(final AllocatedEvaluator eval) {
+      try {
+
+        LOG.log(Level.INFO, "Allocated Evaluator: {0}", eval.getId());
+
+        final Configuration thisContextConfiguration = ContextConfiguration.CONF.set(
+            ContextConfiguration.IDENTIFIER, eval.getId() + "_context").build();
+
+        eval.submitContext(Tang.Factory.getTang()
+            .newConfigurationBuilder(thisContextConfiguration, contextConfig).build());
+
+      } catch (final BindException ex) {
+        throw new RuntimeException(ex);
+      }
+    }
+  }
+
+  /**
+   * Receive notification that a new Context is available.
+   * Submit a new Task to that Context.
+   */
+  final class ActiveContextHandler implements EventHandler<ActiveContext> {
+    @Override
+    public synchronized void onNext(final ActiveContext context) {
+      LOG.log(Level.INFO, "Active Context: {0}", context.getId());
+      try {
+        context.submitTask(TaskConfiguration.CONF
+            .set(TaskConfiguration.IDENTIFIER, context.getId() + "_task")
+            .set(TaskConfiguration.TASK, SuspendTestTask.class)
+            .set(TaskConfiguration.ON_SUSPEND, SuspendTestTask.SuspendHandler.class)
+            .build());
+      } catch (final BindException ex) {
+        LOG.log(Level.SEVERE, "Bad Task configuration for context: " + context.getId(), ex);
+        throw new RuntimeException(ex);
+      }
+    }
+  }
+
+  /**
+   * Handle notifications from the client.
+   */
+  final class ClientMessageHandler implements EventHandler<byte[]> {
+    @Override
+    public void onNext(final byte[] message) {
+
+      final String commandStr = CODEC_STR.decode(message);
+      LOG.log(Level.INFO, "Client message: {0}", commandStr);
+
+      final String[] split = commandStr.split("\\s+", 2);
+      if (split.length != 2) {
+        throw new IllegalArgumentException("Bad command: " + commandStr);
+      } else {
+
+        final String command = split[0].toLowerCase().intern();
+        final String taskId = split[1];
+
+        switch (command) {
+
+          case "suspend": {
+            final RunningTask task = runningTasks.get(taskId);
+            if (task != null) {
+              task.suspend();
+            } else {
+              throw new IllegalArgumentException("Suspend: Task not found: " + taskId);
+            }
+            break;
+          }
+
+          case "resume": {
+            final SuspendedTask suspendedTask;
+            synchronized (suspendedTasks) {
+              suspendedTask = suspendedTasks.remove(taskId);
+            }
+            if (suspendedTask != null) {
+              try {
+                suspendedTask.getActiveContext().submitTask(TaskConfiguration.CONF
+                    .set(TaskConfiguration.IDENTIFIER, taskId)
+                    .set(TaskConfiguration.MEMENTO,
+                        DatatypeConverter.printBase64Binary(suspendedTask.get()))
+                    .build());
+              } catch (final BindException e) {
+                throw new RuntimeException(e);
+              }
+            } else {
+              throw new IllegalArgumentException("Resume: Task not found: " + taskId);
+            }
+            break;
+          }
+
+          default:
+            throw new IllegalArgumentException("Bad command: " + command);
+        }
+      }
+    }
+  }
+
+  /**
+   * Job Driver is ready and the clock is set up: request the evaluators.
+   */
+  final class StartHandler implements EventHandler<StartTime> {
+    @Override
+    public void onNext(final StartTime time) {
+      LOG.log(Level.INFO, "StartTime: {0}", time);
+      evaluatorRequestor.submit(EvaluatorRequest.newBuilder()
+          .setMemory(128).setNumberOfCores(1).setNumber(NUM_EVALUATORS).build());
+    }
+  }
+
+  /**
+   * Shutting down the job driver: close the evaluators.
+   */
+  final class StopHandler implements EventHandler<StopTime> {
+    @Override
+    public void onNext(final StopTime time) {
+      LOG.log(Level.INFO, "StopTime: {0}", time);
+      jobMessageObserver.sendMessageToClient(CODEC_STR.encode("got StopTime"));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendTestTask.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendTestTask.java b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendTestTask.java
new file mode 100644
index 0000000..0acbac1
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/suspend/SuspendTestTask.java
@@ -0,0 +1,176 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.suspend;
+
+import org.apache.reef.io.checkpoint.CheckpointID;
+import org.apache.reef.io.checkpoint.CheckpointService;
+import org.apache.reef.io.checkpoint.CheckpointService.CheckpointReadChannel;
+import org.apache.reef.io.checkpoint.CheckpointService.CheckpointWriteChannel;
+import org.apache.reef.io.checkpoint.fs.FSCheckpointID;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.tang.annotations.Unit;
+import org.apache.reef.task.Task;
+import org.apache.reef.task.TaskMessage;
+import org.apache.reef.task.TaskMessageSource;
+import org.apache.reef.task.events.SuspendEvent;
+import org.apache.reef.util.Optional;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.remote.impl.ObjectSerializableCodec;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Simple do-nothing task that can send messages to the Driver and can be suspended/resumed.
+ */
+@Unit
+public class SuspendTestTask implements Task, TaskMessageSource {
+
+  /**
+   * Standard java logger.
+   */
+  private static final Logger LOG = Logger.getLogger(SuspendTestTask.class.getName());
+  private final CheckpointService checkpointService;
+  /**
+   * number of cycles to run in the task.
+   */
+  private final int numCycles;
+  /**
+   * delay in milliseconds between cycles in the task.
+   */
+  private final int delay;
+  /**
+   * Codec to serialize/deserialize counter values for the updates.
+   */
+  private final ObjectSerializableCodec<Integer> codecInt = new ObjectSerializableCodec<>();
+  /**
+   * Codec to serialize/deserialize checkpoint IDs for suspend/resume.
+   */
+  private final ObjectWritableCodec<CheckpointID> codecCheckpoint =
+      new ObjectWritableCodec<CheckpointID>(FSCheckpointID.class);
+  /**
+   * Current value of the counter.
+   */
+  private int counter = 0;
+  /**
+   * True if the suspend message has been received, false otherwise.
+   */
+  private boolean suspended = false;
+
+  /**
+   * Task constructor: invoked by TANG.
+   *
+   * @param numCycles number of cycles to run in the task.
+   * @param delay     delay in seconds between cycles in the task.
+   */
+  @Inject
+  public SuspendTestTask(
+      final CheckpointService checkpointService,
+      @Parameter(Launch.NumCycles.class) final int numCycles,
+      @Parameter(Launch.Delay.class) final int delay) {
+    this.checkpointService = checkpointService;
+    this.numCycles = numCycles;
+    this.delay = delay * 1000;
+  }
+
+  /**
+   * Main method of the task: run cycle from 0 to numCycles,
+   * and sleep for delay seconds on each cycle.
+   *
+   * @param memento serialized version of the counter.
+   *                Empty array for initial run, but can contain value for resumed job.
+   * @return serialized version of the counter.
+   */
+  @Override
+  public synchronized byte[] call(final byte[] memento) throws IOException, InterruptedException {
+
+    LOG.log(Level.INFO, "Start: {0} counter: {1}/{2}",
+        new Object[]{this, this.counter, this.numCycles});
+
+    if (memento != null && memento.length > 0) {
+      this.restore(memento);
+    }
+
+    this.suspended = false;
+    for (; this.counter < this.numCycles && !this.suspended; ++this.counter) {
+      try {
+        LOG.log(Level.INFO, "Run: {0} counter: {1}/{2} sleep: {3}",
+            new Object[]{this, this.counter, this.numCycles, this.delay});
+        this.wait(this.delay);
+      } catch (final InterruptedException ex) {
+        LOG.log(Level.INFO, "{0} interrupted. counter: {1}: {2}",
+            new Object[]{this, this.counter, ex});
+      }
+    }
+
+    return this.suspended ? this.save() : this.codecInt.encode(this.counter);
+  }
+
+  /**
+   * Update driver on current state of the task.
+   *
+   * @return serialized version of the counter.
+   */
+  @Override
+  public synchronized Optional<TaskMessage> getMessage() {
+    LOG.log(Level.INFO, "Message from Task {0} to the Driver: counter: {1}",
+        new Object[]{this, this.counter});
+    return Optional.of(TaskMessage.from(SuspendTestTask.class.getName(), this.codecInt.encode(this.counter)));
+  }
+
+  /**
+   * Save current state of the task in the checkpoint.
+   *
+   * @return checkpoint ID (serialized)
+   */
+  private synchronized byte[] save() throws IOException, InterruptedException {
+    try (final CheckpointWriteChannel channel = this.checkpointService.create()) {
+      channel.write(ByteBuffer.wrap(this.codecInt.encode(this.counter)));
+      return this.codecCheckpoint.encode(this.checkpointService.commit(channel));
+    }
+  }
+
+  /**
+   * Restore the task state from the given checkpoint.
+   *
+   * @param memento serialized checkpoint ID
+   */
+  private synchronized void restore(final byte[] memento) throws IOException, InterruptedException {
+    final CheckpointID checkpointId = this.codecCheckpoint.decode(memento);
+    try (final CheckpointReadChannel channel = this.checkpointService.open(checkpointId)) {
+      final ByteBuffer buffer = ByteBuffer.wrap(this.codecInt.encode(this.counter));
+      channel.read(buffer);
+      this.counter = this.codecInt.decode(buffer.array());
+    }
+    this.checkpointService.delete(checkpointId);
+  }
+
+  public class SuspendHandler implements EventHandler<SuspendEvent> {
+
+    @Override
+    public void onNext(SuspendEvent suspendEvent) {
+      final byte[] message = suspendEvent.get().get();
+      LOG.log(Level.INFO, "Suspend: {0} with: {1} bytes; counter: {2}",
+          new Object[]{this, message.length, SuspendTestTask.this.counter});
+      SuspendTestTask.this.suspended = true;
+      SuspendTestTask.this.notify();
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/suspend/package-info.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/suspend/package-info.java b/reef-examples/src/main/java/org/apache/reef/examples/suspend/package-info.java
new file mode 100644
index 0000000..a38fee9
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/suspend/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * suspend/resume demo.
+ */
+package org.apache.reef.examples.suspend;

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/BlockingEventHandler.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/BlockingEventHandler.java b/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/BlockingEventHandler.java
new file mode 100644
index 0000000..ec6526b
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/BlockingEventHandler.java
@@ -0,0 +1,60 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.utils.wake;
+
+import org.apache.reef.wake.EventHandler;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An EventHandler that blocks until a set number of Events has been received.
+ * Once they have been received, the downstream event handler is called with an
+ * Iterable of the events spooled.
+ *
+ * @param <T>
+ */
+public final class BlockingEventHandler<T> implements EventHandler<T> {
+
+  private final int expectedSize;
+  private final EventHandler<Iterable<T>> destination;
+  private List<T> events = new ArrayList<>();
+
+  public BlockingEventHandler(final int expectedSize, final EventHandler<Iterable<T>> destination) {
+    this.expectedSize = expectedSize;
+    this.destination = destination;
+  }
+
+  @Override
+  public final void onNext(final T event) {
+    if (this.isComplete()) {
+      throw new IllegalStateException("Received more Events than expected");
+    }
+    this.events.add(event);
+    if (this.isComplete()) {
+      this.destination.onNext(events);
+      this.reset();
+    }
+  }
+
+  private boolean isComplete() {
+    return this.events.size() >= expectedSize;
+  }
+
+  private void reset() {
+    this.events = new ArrayList<>();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/LoggingEventHandler.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/LoggingEventHandler.java b/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/LoggingEventHandler.java
new file mode 100644
index 0000000..ce87c6e
--- /dev/null
+++ b/reef-examples/src/main/java/org/apache/reef/examples/utils/wake/LoggingEventHandler.java
@@ -0,0 +1,55 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.utils.wake;
+
+import org.apache.reef.wake.EventHandler;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * An EventHandler that logs its events before handing it to a downstream
+ * EventHandler.
+ *
+ * @param <T>
+ */
+public class LoggingEventHandler<T> implements EventHandler<T> {
+
+  private final EventHandler<T> downstreamEventHandler;
+  private final String prefix;
+  private final String suffix;
+
+  /**
+   * @param prefix                 to be logged before the event
+   * @param downstreamEventHandler the event handler to hand the event to
+   * @param suffix                 to be logged after the event
+   */
+  public LoggingEventHandler(final String prefix, EventHandler<T> downstreamEventHandler, final String suffix) {
+    this.downstreamEventHandler = downstreamEventHandler;
+    this.prefix = prefix;
+    this.suffix = suffix;
+  }
+
+  public LoggingEventHandler(final EventHandler<T> downstreamEventHandler) {
+    this("", downstreamEventHandler, "");
+  }
+
+  @Override
+  public void onNext(final T value) {
+    Logger.getLogger(LoggingEventHandler.class.getName()).log(Level.INFO, prefix + value.toString() + suffix);
+    this.downstreamEventHandler.onNext(value);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/test/java/com/microsoft/reef/examples/hello/HelloHttpTest.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/test/java/com/microsoft/reef/examples/hello/HelloHttpTest.java b/reef-examples/src/test/java/com/microsoft/reef/examples/hello/HelloHttpTest.java
deleted file mode 100644
index f7cc788..0000000
--- a/reef-examples/src/test/java/com/microsoft/reef/examples/hello/HelloHttpTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.microsoft.reef.examples.hello;
-
-import com.microsoft.reef.client.LauncherStatus;
-import com.microsoft.reef.examples.hellohttp.HelloREEFHttp;
-import com.microsoft.reef.runtime.local.client.LocalRuntimeConfiguration;
-import com.microsoft.tang.Configuration;
-import com.microsoft.tang.exceptions.BindException;
-import com.microsoft.tang.exceptions.InjectionException;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class HelloHttpTest {
-  @Test
-  public void testHttpServer() throws BindException, InjectionException {
-
-    final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
-        .set(LocalRuntimeConfiguration.NUMBER_OF_THREADS, 2)
-        .build();
-
-    final LauncherStatus status = HelloREEFHttp.runHelloReef(runtimeConfiguration, 10 * 1000);
-    Assert.assertEquals(LauncherStatus.FORCE_CLOSED, status); // must be force closed by timeout
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/test/java/com/microsoft/reef/examples/suspend/ObjectWritableCodecTest.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/test/java/com/microsoft/reef/examples/suspend/ObjectWritableCodecTest.java b/reef-examples/src/test/java/com/microsoft/reef/examples/suspend/ObjectWritableCodecTest.java
deleted file mode 100644
index 98739cb..0000000
--- a/reef-examples/src/test/java/com/microsoft/reef/examples/suspend/ObjectWritableCodecTest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.examples.suspend;
-
-import com.microsoft.reef.io.checkpoint.CheckpointID;
-import com.microsoft.reef.io.checkpoint.fs.FSCheckpointID;
-import org.apache.hadoop.fs.Path;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class ObjectWritableCodecTest {
-
-  private static ObjectWritableCodec<CheckpointID> codec;
-
-  /**
-   * Test class setup - create the codec.
-   */
-  @BeforeClass
-  public static void setUpClass() {
-    codec = new ObjectWritableCodec<CheckpointID>(FSCheckpointID.class);
-  }
-
-  /**
-   * After the encode/decode cycle result equals to the original object.
-   */
-  @Test
-  public void testFSCheckpointIdCodec() {
-    final CheckpointID checkpoint1 = new FSCheckpointID(new Path("path"));
-    final byte[] serialized = codec.encode(checkpoint1);
-    final CheckpointID checkpoint2 = codec.decode(serialized);
-    Assert.assertEquals(checkpoint1, checkpoint2);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/test/java/org/apache/reef/examples/hello/HelloHttpTest.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/test/java/org/apache/reef/examples/hello/HelloHttpTest.java b/reef-examples/src/test/java/org/apache/reef/examples/hello/HelloHttpTest.java
new file mode 100644
index 0000000..eb874c7
--- /dev/null
+++ b/reef-examples/src/test/java/org/apache/reef/examples/hello/HelloHttpTest.java
@@ -0,0 +1,39 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.reef.examples.hello;
+
+import org.apache.reef.client.LauncherStatus;
+import org.apache.reef.examples.hellohttp.HelloREEFHttp;
+import org.apache.reef.runtime.local.client.LocalRuntimeConfiguration;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.exceptions.BindException;
+import org.apache.reef.tang.exceptions.InjectionException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HelloHttpTest {
+  @Test
+  public void testHttpServer() throws BindException, InjectionException {
+
+    final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
+        .set(LocalRuntimeConfiguration.NUMBER_OF_THREADS, 2)
+        .build();
+
+    final LauncherStatus status = HelloREEFHttp.runHelloReef(runtimeConfiguration, 10 * 1000);
+    Assert.assertEquals(LauncherStatus.FORCE_CLOSED, status); // must be force closed by timeout
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-examples/src/test/java/org/apache/reef/examples/suspend/ObjectWritableCodecTest.java
----------------------------------------------------------------------
diff --git a/reef-examples/src/test/java/org/apache/reef/examples/suspend/ObjectWritableCodecTest.java b/reef-examples/src/test/java/org/apache/reef/examples/suspend/ObjectWritableCodecTest.java
new file mode 100644
index 0000000..6e98be9
--- /dev/null
+++ b/reef-examples/src/test/java/org/apache/reef/examples/suspend/ObjectWritableCodecTest.java
@@ -0,0 +1,47 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.examples.suspend;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.reef.io.checkpoint.CheckpointID;
+import org.apache.reef.io.checkpoint.fs.FSCheckpointID;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ObjectWritableCodecTest {
+
+  private static ObjectWritableCodec<CheckpointID> codec;
+
+  /**
+   * Test class setup - create the codec.
+   */
+  @BeforeClass
+  public static void setUpClass() {
+    codec = new ObjectWritableCodec<CheckpointID>(FSCheckpointID.class);
+  }
+
+  /**
+   * After the encode/decode cycle result equals to the original object.
+   */
+  @Test
+  public void testFSCheckpointIdCodec() {
+    final CheckpointID checkpoint1 = new FSCheckpointID(new Path("path"));
+    final byte[] serialized = codec.encode(checkpoint1);
+    final CheckpointID checkpoint2 = codec.decode(serialized);
+    Assert.assertEquals(checkpoint1, checkpoint2);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/pom.xml
----------------------------------------------------------------------
diff --git a/reef-io/pom.xml b/reef-io/pom.xml
index 0554149..1b83e11 100644
--- a/reef-io/pom.xml
+++ b/reef-io/pom.xml
@@ -1,8 +1,9 @@
 <?xml version="1.0"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
-        <groupId>com.microsoft.reef</groupId>
+        <groupId>org.apache.reef</groupId>
         <artifactId>reef-project</artifactId>
         <version>0.10-SNAPSHOT</version>
     </parent>
@@ -48,6 +49,7 @@
                         <configuration>
                             <sources>
                                 <source>target/generated-sources/proto</source>
+                                <source>target/generated-sources/avro</source>
                             </sources>
                         </configuration>
                     </execution>

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/avro/nameservice.avsc
----------------------------------------------------------------------
diff --git a/reef-io/src/main/avro/nameservice.avsc b/reef-io/src/main/avro/nameservice.avsc
index b4f7a26..2b5bb24 100644
--- a/reef-io/src/main/avro/nameservice.avsc
+++ b/reef-io/src/main/avro/nameservice.avsc
@@ -15,7 +15,7 @@
  */
  [
 {
-    "namespace":"com.microsoft.io.network.naming.avro",
+    "namespace":"org.apache.reef.io.network.naming.avro",
     "type":"record",
     "name":"AvroNamingLookupRequest",
     "fields":[
@@ -23,7 +23,7 @@
     ]
 },
 {
-    "namespace":"com.microsoft.io.network.naming.avro",
+    "namespace":"org.apache.reef.io.network.naming.avro",
     "type":"record",
     "name":"AvroNamingAssignment",
     "fields":[
@@ -33,7 +33,7 @@
     ]
 },
 {
-    "namespace":"com.microsoft.io.network.naming.avro",
+    "namespace":"org.apache.reef.io.network.naming.avro",
     "type":"record",
     "name":"AvroNamingLookupResponse",
     "fields":[
@@ -41,7 +41,7 @@
     ]
 },
 {
-    "namespace":"com.microsoft.io.network.naming.avro",
+    "namespace":"org.apache.reef.io.network.naming.avro",
     "type":"record",
     "name":"AvroNamingRegisterRequest",
     "fields":[
@@ -51,7 +51,7 @@
     ]
 },
 {
-    "namespace":"com.microsoft.io.network.naming.avro",
+    "namespace":"org.apache.reef.io.network.naming.avro",
     "type":"record",
     "name":"AvroNamingUnRegisterRequest",
     "fields":[

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoader.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoader.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoader.java
deleted file mode 100644
index 5537e05..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoader.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.microsoft.reef.annotations.audience.DriverSide;
-import com.microsoft.reef.driver.context.ContextConfiguration;
-import com.microsoft.reef.driver.evaluator.AllocatedEvaluator;
-import com.microsoft.reef.driver.evaluator.EvaluatorRequest;
-import com.microsoft.reef.driver.evaluator.EvaluatorRequestor;
-import com.microsoft.reef.driver.evaluator.FailedEvaluator;
-import com.microsoft.reef.io.data.loading.impl.EvaluatorRequestSerializer;
-import com.microsoft.reef.io.network.util.Pair;
-import com.microsoft.tang.Configuration;
-import com.microsoft.tang.annotations.Parameter;
-import com.microsoft.tang.annotations.Unit;
-import com.microsoft.tang.exceptions.BindException;
-import com.microsoft.wake.EventHandler;
-import com.microsoft.wake.impl.SingleThreadStage;
-import com.microsoft.wake.time.Clock;
-import com.microsoft.wake.time.event.Alarm;
-import com.microsoft.wake.time.event.StartTime;
-
-/**
- * The driver component for the DataLoadingService
- * Also acts as the central point for resource requests
- * All the allocated evaluators pass through this and
- * the ones that need data loading have a context stacked
- * that enables a task to get access to Data via the
- * {@link DataSet}.
- *
- * TODO: Add timeouts
- */
-@DriverSide
-@Unit
-public class DataLoader {
-
-  private static final Logger LOG = Logger.getLogger(DataLoader.class.getName());
-
-  private final ConcurrentMap<String, Pair<Configuration, Configuration>> submittedDataEvalConfigs = new ConcurrentHashMap<>();
-  private final ConcurrentMap<String, Configuration> submittedComputeEvalConfigs = new ConcurrentHashMap<>();
-  private final BlockingQueue<Configuration> failedComputeEvalConfigs = new LinkedBlockingQueue<>();
-  private final BlockingQueue<Pair<Configuration, Configuration>> failedDataEvalConfigs = new LinkedBlockingQueue<>();
-
-  private final AtomicInteger numComputeRequestsToSubmit = new AtomicInteger(0);
-
-  private final DataLoadingService dataLoadingService;
-  private final int dataEvalMemoryMB;
-  private final int dataEvalCore;
-  private final EvaluatorRequest computeRequest;
-  private final SingleThreadStage<EvaluatorRequest> resourceRequestStage;
-  private final ResourceRequestHandler resourceRequestHandler;
-  private final int computeEvalMemoryMB;
-  private final int computeEvalCore;
-  private final EvaluatorRequestor requestor;
-
-  @Inject
-  public DataLoader(
-      final Clock clock,
-      final EvaluatorRequestor requestor,
-      final DataLoadingService dataLoadingService,
-      final @Parameter(DataLoadingRequestBuilder.DataLoadingEvaluatorMemoryMB.class) int dataEvalMemoryMB,
-      final @Parameter(DataLoadingRequestBuilder.DataLoadingEvaluatorNumberOfCores.class) int dataEvalCore,
-      final @Parameter(DataLoadingRequestBuilder.DataLoadingComputeRequest.class) String serializedComputeRequest) {
-
-    // FIXME: Issue #855: We need this alarm to look busy for REEF.
-    clock.scheduleAlarm(30000, new EventHandler<Alarm>() {
-      @Override
-      public void onNext(final Alarm time) {
-        LOG.log(Level.FINE, "Received Alarm: {0}", time);
-      }
-    });
-
-    this.requestor = requestor;
-    this.dataLoadingService = dataLoadingService;
-    this.dataEvalMemoryMB = dataEvalMemoryMB;
-    this.dataEvalCore = dataEvalCore;
-    this.resourceRequestHandler = new ResourceRequestHandler(requestor);
-    this.resourceRequestStage = new SingleThreadStage<>(this.resourceRequestHandler, 2);
-
-    if (serializedComputeRequest.equals("NULL")) {
-      this.computeRequest = null;
-      this.computeEvalMemoryMB = -1;
-      computeEvalCore = 1;
-    } else {
-      this.computeRequest = EvaluatorRequestSerializer.deserialize(serializedComputeRequest);
-      this.computeEvalMemoryMB = this.computeRequest.getMegaBytes();
-      this.computeEvalCore = this.computeRequest.getNumberOfCores();
-      this.numComputeRequestsToSubmit.set(this.computeRequest.getNumber());
-
-      this.resourceRequestStage.onNext(this.computeRequest);
-    }
-
-    this.resourceRequestStage.onNext(getDataLoadingRequest());
-  }
-
-  private EvaluatorRequest getDataLoadingRequest() {
-    return EvaluatorRequest.newBuilder()
-        .setNumber(this.dataLoadingService.getNumberOfPartitions())
-        .setMemory(this.dataEvalMemoryMB)
-        .setNumberOfCores(this.dataEvalCore)
-        .build();
-  }
-
-  public class StartHandler implements EventHandler<StartTime> {
-    @Override
-    public void onNext(final StartTime startTime) {
-      LOG.log(Level.INFO, "StartTime: {0}", startTime);
-      resourceRequestHandler.releaseResourceRequestGate();
-    }
-  }
-
-  public class EvaluatorAllocatedHandler implements EventHandler<AllocatedEvaluator> {
-
-    @Override
-    public void onNext(final AllocatedEvaluator allocatedEvaluator) {
-
-      final String evalId = allocatedEvaluator.getId();
-      LOG.log(Level.FINEST, "Allocated evaluator: {0}", evalId);
-
-      if (!failedComputeEvalConfigs.isEmpty()) {
-        LOG.log(Level.FINE, "Failed Compute requests need to be satisfied for {0}", evalId);
-        final Configuration conf = failedComputeEvalConfigs.poll();
-        if (conf != null) {
-          LOG.log(Level.FINE, "Satisfying failed configuration for {0}", evalId);
-          allocatedEvaluator.submitContext(conf);
-          submittedComputeEvalConfigs.put(evalId, conf);
-          return;
-        }
-      }
-
-      if (!failedDataEvalConfigs.isEmpty()) {
-        LOG.log(Level.FINE, "Failed Data requests need to be satisfied for {0}", evalId);
-        final Pair<Configuration, Configuration> confPair = failedDataEvalConfigs.poll();
-        if (confPair != null) {
-          LOG.log(Level.FINE, "Satisfying failed configuration for {0}", evalId);
-          allocatedEvaluator.submitContextAndService(confPair.first, confPair.second);
-          submittedDataEvalConfigs.put(evalId, confPair);
-          return;
-        }
-      }
-
-      final int evaluatorsForComputeRequest = numComputeRequestsToSubmit.decrementAndGet();
-      LOG.log(Level.FINE, "Evaluators for compute request: {0}", evaluatorsForComputeRequest);
-
-      if (evaluatorsForComputeRequest >= 0) {
-        try {
-          final Configuration idConfiguration = ContextConfiguration.CONF
-              .set(ContextConfiguration.IDENTIFIER,
-                  dataLoadingService.getComputeContextIdPrefix() + evaluatorsForComputeRequest)
-              .build();
-          LOG.log(Level.FINE, "Submitting Compute Context to {0}", evalId);
-          allocatedEvaluator.submitContext(idConfiguration);
-          submittedComputeEvalConfigs.put(allocatedEvaluator.getId(), idConfiguration);
-          if (evaluatorsForComputeRequest == 0) {
-            LOG.log(Level.FINE, "All Compute requests satisfied. Releasing gate");
-            resourceRequestHandler.releaseResourceRequestGate();
-          }
-        } catch (final BindException e) {
-          throw new RuntimeException("Unable to bind context id for Compute request", e);
-        }
-
-      } else {
-
-        final Pair<Configuration, Configuration> confPair = new Pair<>(
-            dataLoadingService.getContextConfiguration(allocatedEvaluator),
-            dataLoadingService.getServiceConfiguration(allocatedEvaluator));
-
-        LOG.log(Level.FINE, "Submitting data loading context to {0}", evalId);
-        allocatedEvaluator.submitContextAndService(confPair.first, confPair.second);
-        submittedDataEvalConfigs.put(allocatedEvaluator.getId(), confPair);
-      }
-    }
-  }
-
-  public class EvaluatorFailedHandler implements EventHandler<FailedEvaluator> {
-    @Override
-    public void onNext(final FailedEvaluator failedEvaluator) {
-
-      final String evalId = failedEvaluator.getId();
-
-      final Configuration computeConfig = submittedComputeEvalConfigs.remove(evalId);
-      if (computeConfig != null) {
-
-        LOG.log(Level.INFO, "Received failed compute evaluator: {0}", evalId);
-        failedComputeEvalConfigs.add(computeConfig);
-
-        requestor.submit(EvaluatorRequest.newBuilder()
-            .setMemory(computeEvalMemoryMB).setNumber(1).setNumberOfCores(computeEvalCore).build());
-
-      } else {
-
-        final Pair<Configuration, Configuration> confPair = submittedDataEvalConfigs.remove(evalId);
-        if (confPair != null) {
-
-          LOG.log(Level.INFO, "Received failed data evaluator: {0}", evalId);
-          failedDataEvalConfigs.add(confPair);
-
-          requestor.submit(EvaluatorRequest.newBuilder()
-              .setMemory(dataEvalMemoryMB).setNumber(1).setNumberOfCores(dataEvalCore).build());
-
-        } else {
-
-          LOG.log(Level.SEVERE, "Received unknown failed evaluator " + evalId,
-              failedEvaluator.getEvaluatorException());
-
-          throw new RuntimeException("Received failed evaluator that I did not submit: " + evalId);
-        }
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingDriverConfiguration.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingDriverConfiguration.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingDriverConfiguration.java
deleted file mode 100644
index fbacfaa..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingDriverConfiguration.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import com.microsoft.reef.client.DriverConfiguration;
-import com.microsoft.reef.driver.parameters.EvaluatorAllocatedHandlers;
-import com.microsoft.tang.formats.ConfigurationModule;
-import com.microsoft.tang.formats.ConfigurationModuleBuilder;
-import com.microsoft.wake.time.Clock;
-
-public final class DataLoadingDriverConfiguration extends ConfigurationModuleBuilder {
-  public static final ConfigurationModule CONF = new DataLoadingDriverConfiguration()
-      .merge(DriverConfiguration.CONF)
-      .bindSetEntry(Clock.StartHandler.class, DataLoader.StartHandler.class)
-      .bindSetEntry(EvaluatorAllocatedHandlers.class, DataLoader.EvaluatorAllocatedHandler.class)
-      .build();
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingRequestBuilder.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingRequestBuilder.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingRequestBuilder.java
deleted file mode 100644
index bb380e4..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingRequestBuilder.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.microsoft.reef.client.DriverConfiguration;
-import com.microsoft.reef.driver.evaluator.EvaluatorRequest;
-import com.microsoft.reef.io.data.loading.impl.EvaluatorRequestSerializer;
-import com.microsoft.reef.io.data.loading.impl.InputFormatExternalConstructor;
-import com.microsoft.reef.io.data.loading.impl.InputFormatLoadingService;
-import com.microsoft.reef.io.data.loading.impl.JobConfExternalConstructor;
-import com.microsoft.tang.Configuration;
-import com.microsoft.tang.JavaConfigurationBuilder;
-import com.microsoft.tang.Tang;
-import com.microsoft.tang.annotations.Name;
-import com.microsoft.tang.annotations.NamedParameter;
-import com.microsoft.tang.exceptions.BindException;
-import com.microsoft.tang.formats.ConfigurationModule;
-
-/**
- * Builder to create a request to the DataLoadingService.
- */
-public final class DataLoadingRequestBuilder
-    implements com.microsoft.reef.util.Builder<Configuration> {
-
-  @NamedParameter(short_name = "num_splits", default_value = "0")
-  public static final class NumberOfDesiredSplits implements Name<Integer> {
-  }
-
-  @NamedParameter(short_name = "dataLoadingEvaluatorMemoryMB", default_value = "4096")
-  public static final class DataLoadingEvaluatorMemoryMB implements Name<Integer> {
-  }
-
-  @NamedParameter(short_name = "dataLoadingEvaluatorCore", default_value = "1")
-  public static final class DataLoadingEvaluatorNumberOfCores implements Name<Integer> {
-  }
-
-  @NamedParameter(default_value = "NULL")
-  public static final class DataLoadingComputeRequest implements Name<String> {
-  }
-
-  @NamedParameter(default_value = "false")
-  public static final class LoadDataIntoMemory implements Name<Boolean> {
-  }
-
-  private int memoryMB = -1;
-  private int numberOfCores = -1;
-  private int numberOfDesiredSplits = -1;
-  private EvaluatorRequest computeRequest = null;
-  private boolean inMemory = false;
-  private boolean renewFailedEvaluators = true;
-  private ConfigurationModule driverConfigurationModule = null;
-  private String inputFormatClass;
-  private String inputPath;
-
-  public DataLoadingRequestBuilder setNumberOfDesiredSplits(final int numberOfDesiredSplits) {
-    this.numberOfDesiredSplits = numberOfDesiredSplits;
-    return this;
-  }
-
-  /**
-   * Set the memory to be used for Evaluator allocated.
-   *
-   * @param memoryMB the amount of memory in MB
-   * @return this
-   */
-  public DataLoadingRequestBuilder setMemoryMB(final int memoryMB) {
-    this.memoryMB = memoryMB;
-    return this;
-  }
-
-  /**
-   * Set the core number to be used for Evaluator allocated.
-   *
-   * @param numberOfCores the number of cores
-   * @return this
-   */
-  public DataLoadingRequestBuilder setNumberOfCores(final int numberOfCores) {
-    this.numberOfCores = numberOfCores;
-    return this;
-  }
-
-  public DataLoadingRequestBuilder setComputeRequest(final EvaluatorRequest computeRequest) {
-    this.computeRequest = computeRequest;
-    return this;
-  }
-
-  public DataLoadingRequestBuilder loadIntoMemory(final boolean inMemory) {
-    this.inMemory = inMemory;
-    return this;
-  }
-
-  public DataLoadingRequestBuilder renewFailedEvaluators(final boolean renewFailedEvaluators) {
-    this.renewFailedEvaluators = renewFailedEvaluators;
-    return this;
-  }
-
-  public DataLoadingRequestBuilder setDriverConfigurationModule(
-      final ConfigurationModule driverConfigurationModule) {
-    this.driverConfigurationModule = driverConfigurationModule;
-    return this;
-  }
-
-  public DataLoadingRequestBuilder setInputFormatClass(
-      final Class<? extends InputFormat> inputFormatClass) {
-    this.inputFormatClass = inputFormatClass.getName();
-    return this;
-  }
-
-  public DataLoadingRequestBuilder setInputPath(final String inputPath) {
-    this.inputPath = inputPath;
-    return this;
-  }
-
-  @Override
-  public Configuration build() throws BindException {
-    if (this.driverConfigurationModule == null) {
-      throw new BindException("Driver Configuration Module is a required parameter.");
-    }
-
-    if(this.inputPath == null) {
-      throw new BindException("InputPath is a required parameter.");
-    }
-
-    if(this.inputFormatClass == null) {
-      this.inputFormatClass = TextInputFormat.class.getName();
-    }
-
-    final Configuration driverConfiguration;
-    if (renewFailedEvaluators) {
-      driverConfiguration = this.driverConfigurationModule
-              .set(DriverConfiguration.ON_DRIVER_STARTED, DataLoader.StartHandler.class)
-              .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, DataLoader.EvaluatorAllocatedHandler.class)
-              .set(DriverConfiguration.ON_EVALUATOR_FAILED, DataLoader.EvaluatorFailedHandler.class)
-              .build();
-    } else {
-      driverConfiguration = this.driverConfigurationModule
-              .set(DriverConfiguration.ON_DRIVER_STARTED, DataLoader.StartHandler.class)
-              .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, DataLoader.EvaluatorAllocatedHandler.class)
-              .build();
-    }
-
-    final JavaConfigurationBuilder jcb =
-        Tang.Factory.getTang().newConfigurationBuilder(driverConfiguration);
-
-    if (this.numberOfDesiredSplits > 0) {
-      jcb.bindNamedParameter(NumberOfDesiredSplits.class, "" + this.numberOfDesiredSplits);
-    }
-
-    if (this.memoryMB > 0) {
-      jcb.bindNamedParameter(DataLoadingEvaluatorMemoryMB.class, "" + this.memoryMB);
-    }
-
-    if (this.numberOfCores > 0) {
-      jcb.bindNamedParameter(DataLoadingEvaluatorNumberOfCores.class, "" + this.numberOfCores);
-    }
-
-    if (this.computeRequest != null) {
-      jcb.bindNamedParameter(DataLoadingComputeRequest.class,
-          EvaluatorRequestSerializer.serialize(this.computeRequest));
-    }
-
-    return jcb
-        .bindNamedParameter(LoadDataIntoMemory.class, Boolean.toString(this.inMemory))
-        .bindConstructor(InputFormat.class, InputFormatExternalConstructor.class)
-        .bindConstructor(JobConf.class, JobConfExternalConstructor.class)
-        .bindNamedParameter(JobConfExternalConstructor.InputFormatClass.class, inputFormatClass)
-        .bindNamedParameter(JobConfExternalConstructor.InputPath.class, inputPath)
-        .bindImplementation(DataLoadingService.class, InputFormatLoadingService.class)
-        .build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingService.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingService.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingService.java
deleted file mode 100644
index f49c7e0..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataLoadingService.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import com.microsoft.reef.annotations.audience.DriverSide;
-import com.microsoft.reef.driver.context.ActiveContext;
-import com.microsoft.reef.driver.evaluator.AllocatedEvaluator;
-import com.microsoft.tang.Configuration;
-
-/**
- * All data loading services should implement this interface.
- */
-@DriverSide
-public interface DataLoadingService {
-
-  /**
-   * Access to the number of partitions suggested by this DataSource.
-   *
-   * @return the number of partitions suggested by this DataSource.
-   */
-  int getNumberOfPartitions();
-
-  /**
-   * @return the context configuration for the given Evaluator.
-   */
-  Configuration getContextConfiguration(final AllocatedEvaluator allocatedEvaluator);
-
-  /**
-   * @return the service configuration for the given Evaluator.
-   */
-  Configuration getServiceConfiguration(final AllocatedEvaluator allocatedEvaluator);
-
-  /**
-   * @return Return the prefix to be used to enumerate
-   * context ids for compute requests fired other than
-   * the data load contexts.
-   */
-  String getComputeContextIdPrefix();
-
-  /**
-   * Distinguishes data loaded contexts from compute contexts.
-   *
-   * @return true if this context has been loaded with data.
-   */
-  boolean isDataLoadedContext(ActiveContext context);
-
-  /**
-   * @return true if this is a computation context,
-   * false otherwise. (e.g. this is a data loading context).
-   */
-  boolean isComputeContext(ActiveContext context);
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataSet.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataSet.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataSet.java
deleted file mode 100644
index 2ac1c44..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/DataSet.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import com.microsoft.reef.annotations.audience.TaskSide;
-import com.microsoft.reef.io.network.util.Pair;
-
-/**
- * A view of the data set to be loaded
- * at an evaluator as an iterable of
- * key value pairs.
- * 
- * Implementations need not materialize
- * and clients should not assume that the
- * data is materialized. Any such thing
- * is left as a post-processing step.
- * 
- * Client also can't assume that the iterator
- * returned here can be restarted
- * 
- * @param <K>
- * @param <V>
- */
-@TaskSide
-public interface DataSet<K,V> extends Iterable<Pair<K,V>>{
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/ResourceRequestHandler.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/ResourceRequestHandler.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/ResourceRequestHandler.java
deleted file mode 100644
index e5390ca..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/api/ResourceRequestHandler.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.api;
-
-import com.microsoft.reef.driver.evaluator.EvaluatorRequest;
-import com.microsoft.reef.driver.evaluator.EvaluatorRequestor;
-import com.microsoft.wake.EventHandler;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class ResourceRequestHandler implements EventHandler<EvaluatorRequest> {
-
-  private static final Logger LOG = Logger.getLogger(ResourceRequestHandler.class.getName());
-
-  private final EvaluatorRequestor requestor;
-
-  private CountDownLatch resourceRequestGate = new CountDownLatch(1);
-
-  public ResourceRequestHandler(final EvaluatorRequestor requestor) {
-    this.requestor = requestor;
-  }
-
-  public void releaseResourceRequestGate() {
-    LOG.log(Level.FINE, "Releasing Gate");
-    this.resourceRequestGate.countDown();
-  }
-
-  @Override
-  public void onNext(final EvaluatorRequest request) {
-    try {
-
-      LOG.log(Level.FINE,
-          "Processing a request with count: {0} - Waiting for gate to be released",
-          request.getNumber());
-
-      this.resourceRequestGate.await();
-
-      LOG.log(Level.FINE, "Gate released. Submitting request: {0}", request);
-      this.resourceRequestGate = new CountDownLatch(1);
-      this.requestor.submit(request);
-
-    } catch (final InterruptedException ex) {
-      LOG.log(Level.FINEST, "Interrupted", ex);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorRequestSerializer.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorRequestSerializer.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorRequestSerializer.java
deleted file mode 100644
index faf1bc7..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorRequestSerializer.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.commons.codec.binary.Base64;
-
-import com.microsoft.reef.driver.evaluator.EvaluatorRequest;
-
-/**
- * Serialize and deserialize EvaluatorRequest objects
- * Currently only supports number & memory
- * Does not take care of Resource Descriptor
- */
-public class EvaluatorRequestSerializer {
-  public static String serialize(EvaluatorRequest request){
-    try(ByteArrayOutputStream baos = new ByteArrayOutputStream()){
-      try(DataOutputStream daos = new DataOutputStream(baos)){
-        
-        daos.writeInt(request.getNumber());
-        daos.writeInt(request.getMegaBytes());
-        daos.writeInt(request.getNumberOfCores());
-
-      } catch (IOException e) {
-        throw e;
-      }
-      
-      return Base64.encodeBase64String(baos.toByteArray());
-    } catch (IOException e1) {
-      throw new RuntimeException("Unable to serialize compute request", e1);
-    }
-  }
-  
-  public static EvaluatorRequest deserialize(String serializedRequest){
-    try(ByteArrayInputStream bais =  new ByteArrayInputStream(Base64.decodeBase64(serializedRequest))){
-      try(DataInputStream dais = new DataInputStream(bais)){
-        return EvaluatorRequest.newBuilder()
-            .setNumber(dais.readInt())
-            .setMemory(dais.readInt())
-            .setNumberOfCores(dais.readInt())
-            .build();
-      }
-    } catch (IOException e) {
-      throw new RuntimeException("Unable to de-serialize compute request", e);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorToPartitionMapper.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorToPartitionMapper.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorToPartitionMapper.java
deleted file mode 100644
index 88f3343..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/EvaluatorToPartitionMapper.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-
-import com.microsoft.reef.annotations.audience.DriverSide;
-import com.microsoft.reef.io.data.loading.api.DataLoadingService;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Class that tracks the mapping between
- * evaluators & the data partition assigned
- * to those evaluators. Its part of the
- * implementation of a {@link DataLoadingService}
- * that uses the Hadoop {@link InputFormat} to
- * partition the data and request resources
- * accordingly
- * 
- * This is an online version which satisfies
- * requests in a greedy way.
- * @param <V>
- */
-@DriverSide
-public class EvaluatorToPartitionMapper<V extends InputSplit> {
-  private static final Logger LOG = Logger
-      .getLogger(EvaluatorToPartitionMapper.class.getName());
-
-  private final ConcurrentMap<String, BlockingQueue<NumberedSplit<V>>> locationToSplits = new ConcurrentHashMap<>();
-  private final ConcurrentMap<String, NumberedSplit<V>> evaluatorToSplits = new ConcurrentHashMap<>();
-  private final BlockingQueue<NumberedSplit<V>> unallocatedSplits = new LinkedBlockingQueue<>();
-
-  /**
-   * Initializes the locations of splits mapping
-   * @param splits
-   */
-  public EvaluatorToPartitionMapper(V[] splits) {
-    try {
-      for (int splitNum = 0; splitNum < splits.length; splitNum++) {
-        LOG.log(Level.FINE,"Processing split: " + splitNum);
-        final V split = splits[splitNum];
-        final String[] locations = split.getLocations();
-        final NumberedSplit<V> numberedSplit = new NumberedSplit<V>(split, splitNum);
-        unallocatedSplits.add(numberedSplit);
-        for (final String location : locations) {
-          BlockingQueue<NumberedSplit<V>> newSplitQue = new LinkedBlockingQueue<NumberedSplit<V>>();
-          final BlockingQueue<NumberedSplit<V>> splitQue = locationToSplits.putIfAbsent(location,
-              newSplitQue);
-          if (splitQue != null) {
-            newSplitQue = splitQue;
-          }
-          newSplitQue.add(numberedSplit);
-        }
-      }
-      for (Map.Entry<String, BlockingQueue<NumberedSplit<V>>> locSplit : locationToSplits.entrySet()) {
-        LOG.log(Level.FINE,locSplit.getKey() + ": " + locSplit.getValue().toString());
-      }
-    } catch (IOException e) {
-      throw new RuntimeException(
-          "Unable to get InputSplits using the specified InputFormat", e);
-    }
-  }
-
-  /**
-   * Get an input split to be assigned to this
-   * evaluator
-   * 
-   * Allocates one if its not already allocated
-   * @param evaluatorId
-   * @return
-   */
-  public NumberedSplit<V> getInputSplit(final String hostName, final String evaluatorId) {
-    synchronized (evaluatorToSplits) {
-      if (evaluatorToSplits.containsKey(evaluatorId)) {
-        LOG.log(Level.FINE,"Found an already allocated partition");
-        LOG.log(Level.FINE,evaluatorToSplits.toString());
-        return evaluatorToSplits.get(evaluatorId);
-      }
-    }
-    LOG.log(Level.FINE,"allocated partition not found");
-    if (locationToSplits.containsKey(hostName)) {
-      LOG.log(Level.FINE,"Found partitions possibly hosted for " + evaluatorId + " at " + hostName);
-      final NumberedSplit<V> split = allocateSplit(evaluatorId, locationToSplits.get(hostName));
-      LOG.log(Level.FINE,evaluatorToSplits.toString());
-      if (split != null) {
-        return split;
-      }
-    }
-    //pick random
-    LOG.log(
-        Level.FINE,
-        hostName
-            + " does not host any partitions or someone else took partitions hosted here. Picking a random one");
-    final NumberedSplit<V> split = allocateSplit(evaluatorId, unallocatedSplits);
-    LOG.log(Level.FINE,evaluatorToSplits.toString());
-    if (split != null) {
-      return split;
-    }
-    throw new RuntimeException("Unable to find an input partition to evaluator " + evaluatorId);
-  }
-
-  private NumberedSplit<V> allocateSplit(final String evaluatorId,
-                                         final BlockingQueue<NumberedSplit<V>> value) {
-    if (value == null) {
-      LOG.log(Level.FINE,"Queue of splits can't be empty. Returning null");
-      return null;
-    }
-    while(true){
-      final NumberedSplit<V> split = value.poll();
-      if(split==null)
-        return null;
-      if (value == unallocatedSplits || unallocatedSplits.remove(split)) {
-        LOG.log(Level.FINE,"Found split-" + split.getIndex() + " in the queue");
-        final NumberedSplit<V> old = evaluatorToSplits.putIfAbsent(evaluatorId, split);
-        if (old != null) {
-          final String msg = "Trying to assign different partitions to the same evaluator " +
-              "is not supported";
-          LOG.severe(msg);
-          throw new RuntimeException(msg);
-        } else {
-          LOG.log(Level.FINE,"Returning " + split.getIndex());
-          return split;
-        }
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InMemoryInputFormatDataSet.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InMemoryInputFormatDataSet.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InMemoryInputFormatDataSet.java
deleted file mode 100644
index 480c71c..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InMemoryInputFormatDataSet.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.inject.Inject;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.microsoft.reef.io.data.loading.api.DataSet;
-import com.microsoft.reef.io.network.util.Pair;
-
-public class InMemoryInputFormatDataSet<K extends WritableComparable<K>,V extends Writable> 
-              implements DataSet<K,V>  {
-  
-  private final InputFormatDataSet<K, V> inputFormatDataSet;
-  private List<Pair<K, V>> recordsList = null;
-
-  @Inject
-  public InMemoryInputFormatDataSet(InputFormatDataSet<K,V> inputFormatDataSet) {
-    this.inputFormatDataSet = inputFormatDataSet;
-  }
-
-
-
-  @Override
-  public synchronized Iterator<Pair<K, V>> iterator() {
-    if (recordsList == null) {
-      recordsList = new ArrayList<>();
-      for (final Pair<K,V> keyValue : inputFormatDataSet) {
-        recordsList.add(keyValue);
-      }
-    }
-    return recordsList.iterator();
-  }}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatDataSet.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatDataSet.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatDataSet.java
deleted file mode 100644
index 1791c2a..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatDataSet.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import javax.inject.Inject;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.Counters.Counter;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.microsoft.reef.annotations.audience.TaskSide;
-import com.microsoft.reef.io.data.loading.api.DataSet;
-import com.microsoft.reef.io.network.util.Pair;
-
-/**
- * An implementation of {@link DataSet} that reads records using a RecordReader
- * encoded inside an InputSplit.
- * <p/>
- * The input split is injected through an external constructor by deserializing
- * the input split assigned to this evaluator.
- *
- * @param <K>
- * @param <V>
- */
-@TaskSide
-public final class
-    InputFormatDataSet<K extends WritableComparable<K>, V extends Writable>
-    implements DataSet<K, V> {
-
-  private final DummyReporter dummyReporter = new DummyReporter();
-  private RecordReader lastRecordReader = null;
-
-  private final JobConf jobConf;
-  private final InputFormat<K, V> inputFormat;
-  private final InputSplit split;
-
-  @Inject
-  public InputFormatDataSet(final InputSplit split, final JobConf jobConf) {
-    this.jobConf = jobConf;
-    this.inputFormat = this.jobConf.getInputFormat();
-    this.split = split;
-  }
-
-  @Override
-  public Iterator<Pair<K, V>> iterator() {
-    try {
-
-      final RecordReader newRecordReader =
-          this.inputFormat.getRecordReader(this.split, this.jobConf, this.dummyReporter);
-
-      if (newRecordReader == this.lastRecordReader) {
-        throw new RuntimeException("Received the same record reader again. This isn't supported.");
-      }
-
-      this.lastRecordReader = newRecordReader;
-      return new RecordReaderIterator(newRecordReader);
-
-    } catch (final IOException ex) {
-      throw new RuntimeException("Can't instantiate iterator.", ex);
-    }
-  }
-
-  private final class RecordReaderIterator implements Iterator<Pair<K, V>> {
-
-    private final RecordReader<K, V> recordReader;
-    private Pair<K, V> recordPair;
-    private boolean hasNext;
-
-    RecordReaderIterator(final RecordReader<K, V> recordReader) {
-      this.recordReader = recordReader;
-      fetchRecord();
-    }
-
-    @Override
-    public boolean hasNext() {
-      return this.hasNext;
-    }
-
-    @Override
-    public Pair<K, V> next() {
-      final Pair<K, V> prevRecordPair = this.recordPair;
-      fetchRecord();
-      return prevRecordPair;
-    }
-
-    @Override
-    public void remove() {
-      throw new UnsupportedOperationException("Remove is not supported on RecordReader iterator");
-    }
-
-    private void fetchRecord() {
-      this.recordPair = new Pair<>(this.recordReader.createKey(), this.recordReader.createValue());
-      try {
-        this.hasNext = this.recordReader.next(this.recordPair.first, this.recordPair.second);
-      } catch (final IOException ex) {
-        throw new RuntimeException("Unable to get InputSplits using the specified InputFormat", ex);
-      }
-    }
-  }
-
-  private final class DummyReporter implements Reporter {
-
-    @Override
-    public void progress() {
-    }
-
-    @Override
-    public Counter getCounter(final Enum<?> key) {
-      return null;
-    }
-
-    @Override
-    public Counter getCounter(final String group, final String name) {
-      return null;
-    }
-
-    @Override
-    public InputSplit getInputSplit() throws UnsupportedOperationException {
-      throw new UnsupportedOperationException("This is a Fake Reporter");
-    }
-
-    @Override
-    public float getProgress() {
-      return 0;
-    }
-
-    @Override
-    public void incrCounter(final Enum<?> key, final long amount) {
-    }
-
-    @Override
-    public void incrCounter(final String group, final String counter, final long amount) {
-    }
-
-    @Override
-    public void setStatus(final String status) {
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatExternalConstructor.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatExternalConstructor.java b/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatExternalConstructor.java
deleted file mode 100644
index 7247731..0000000
--- a/reef-io/src/main/java/com/microsoft/reef/io/data/loading/impl/InputFormatExternalConstructor.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.io.data.loading.impl;
-
-import javax.inject.Inject;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.JobConf;
-
-import com.microsoft.reef.annotations.audience.DriverSide;
-import com.microsoft.tang.ExternalConstructor;
-
-
-/**
- * A Tang External Constructor to inject the required
- * InputFormat
- */
-@DriverSide
-public class InputFormatExternalConstructor implements ExternalConstructor<InputFormat<?,?>> {
-
-  private final JobConf jobConf;
-  private final InputFormat<?,?> inputFormat;
-
-  @Inject
-  public InputFormatExternalConstructor(final JobConf jobConf) {
-    this.jobConf = jobConf;
-    inputFormat = jobConf.getInputFormat();
-  }
-
-  @Override
-  public InputFormat<?,?> newInstance() {
-    return inputFormat;
-  }
-
-}