You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2014/10/23 02:03:40 UTC
[09/51] [abbrv] [partial] Initial merge of Wake,
Tang and REEF into one repository and project
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/naming/serialization/NamingUnregisterRequestCodec.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/naming/serialization/NamingUnregisterRequestCodec.java b/reef-io/src/main/java/org/apache/reef/io/network/naming/serialization/NamingUnregisterRequestCodec.java
new file mode 100644
index 0000000..8548a09
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/naming/serialization/NamingUnregisterRequestCodec.java
@@ -0,0 +1,69 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.network.naming.serialization;
+
+import org.apache.reef.io.network.naming.avro.AvroNamingUnRegisterRequest;
+import org.apache.reef.io.network.naming.exception.NamingRuntimeException;
+import org.apache.reef.wake.IdentifierFactory;
+import org.apache.reef.wake.remote.Codec;
+
+import javax.inject.Inject;
+
+/**
+ * Naming un-registration request codec
+ */
+public final class NamingUnregisterRequestCodec implements Codec<NamingUnregisterRequest> {
+
+ private final IdentifierFactory factory;
+
+ /**
+ * Constructs a naming un-registration request codec
+ *
+ * @param factory the identifier factory
+ */
+ @Inject
+ public NamingUnregisterRequestCodec(final IdentifierFactory factory) {
+ this.factory = factory;
+ }
+
+ /**
+ * Encodes the naming un-registration request to bytes
+ *
+ * @param obj the naming un-registration request
+ * @return a byte array
+ */
+ @Override
+ public byte[] encode(NamingUnregisterRequest obj) {
+ final AvroNamingUnRegisterRequest result = AvroNamingUnRegisterRequest.newBuilder()
+ .setId(obj.getIdentifier().toString())
+ .build();
+ return AvroUtils.toBytes(result, AvroNamingUnRegisterRequest.class);
+ }
+
+ /**
+ * Decodes the bytes to a naming un-registration request
+ *
+ * @param buf the byte array
+ * @return a naming un-registration request
+ * @throws NamingRuntimeException
+ */
+ @Override
+ public NamingUnregisterRequest decode(byte[] buf) {
+ final AvroNamingUnRegisterRequest result = AvroUtils.fromBytes(buf, AvroNamingUnRegisterRequest.class);
+ return new NamingUnregisterRequest(factory.getNewInstance(result.getId().toString()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/naming/serialization/package-info.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/naming/serialization/package-info.java b/reef-io/src/main/java/org/apache/reef/io/network/naming/serialization/package-info.java
new file mode 100644
index 0000000..40ce819
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/naming/serialization/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Contains naming serialization codecs
+ */
+package org.apache.reef.io.network.naming.serialization;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/package-info.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/package-info.java b/reef-io/src/main/java/org/apache/reef/io/network/package-info.java
new file mode 100644
index 0000000..c7d2c18
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/package-info.java
@@ -0,0 +1,16 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.network;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/util/ListCodec.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/util/ListCodec.java b/reef-io/src/main/java/org/apache/reef/io/network/util/ListCodec.java
new file mode 100644
index 0000000..b3fed11
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/util/ListCodec.java
@@ -0,0 +1,80 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.network.util;
+
+import org.apache.reef.wake.remote.Codec;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public final class ListCodec<T> implements Codec<List<T>> {
+
+ private static final Logger LOG = Logger.getLogger(ListCodec.class.getName());
+
+ private final Codec<T> codec;
+
+ public ListCodec(final Codec<T> codec) {
+ super();
+ this.codec = codec;
+ }
+
+ public static void main(final String[] args) {
+ final List<String> arrList = Arrays.asList(
+ new String[]{"One", "Two", "Three", "Four", "Five"});
+ LOG.log(Level.FINEST, "Input: {0}", arrList);
+ final ListCodec<String> lstCodec = new ListCodec<>(new StringCodec());
+ final byte[] bytes = lstCodec.encode(arrList);
+ LOG.log(Level.FINEST, "Output: {0}", lstCodec.decode(bytes));
+ }
+
+ @Override
+ public byte[] encode(final List<T> list) {
+ try (final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ final DataOutputStream daos = new DataOutputStream(baos)) {
+ for (final T t : list) {
+ final byte[] tBytes = this.codec.encode(t);
+ daos.writeInt(tBytes.length);
+ daos.write(tBytes);
+ }
+ return baos.toByteArray();
+ } catch (final IOException ex) {
+ LOG.log(Level.WARNING, "Error in list encoding", ex);
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public List<T> decode(final byte[] list) {
+ final List<T> result = new ArrayList<>();
+ try (final DataInputStream dais = new DataInputStream(new ByteArrayInputStream(list))) {
+ while (dais.available() > 0) {
+ final int length = dais.readInt();
+ final byte[] tBytes = new byte[length];
+ dais.readFully(tBytes);
+ final T t = this.codec.decode(tBytes);
+ result.add(t);
+ }
+ return result;
+ } catch (final IOException ex) {
+ LOG.log(Level.WARNING, "Error in list decoding", ex);
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/util/Pair.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/util/Pair.java b/reef-io/src/main/java/org/apache/reef/io/network/util/Pair.java
new file mode 100644
index 0000000..2dc0ac9
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/util/Pair.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2013 Microsoft.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.network.util;
+
+import java.io.Serializable;
+
+public final class Pair<T1, T2> implements Serializable {
+
+ public final T1 first;
+ public final T2 second;
+
+ private String pairStr = null;
+
+ public Pair(final T1 first, final T2 second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ @Override
+ public String toString() {
+ if (this.pairStr == null) {
+ this.pairStr = "(" + this.first + "," + this.second + ")";
+ }
+ return this.pairStr;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/util/StringCodec.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/util/StringCodec.java b/reef-io/src/main/java/org/apache/reef/io/network/util/StringCodec.java
new file mode 100644
index 0000000..2a6fa6a
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/util/StringCodec.java
@@ -0,0 +1,39 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.network.util;
+
+import org.apache.reef.wake.remote.Codec;
+
+import javax.inject.Inject;
+
+
+public class StringCodec implements Codec<String> {
+
+ @Inject
+ public StringCodec() {
+ // Intentionally blank
+ }
+
+ @Override
+ public byte[] encode(String obj) {
+ return obj.getBytes();
+ }
+
+ @Override
+ public String decode(byte[] buf) {
+ return new String(buf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/util/StringIdentifier.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/util/StringIdentifier.java b/reef-io/src/main/java/org/apache/reef/io/network/util/StringIdentifier.java
new file mode 100644
index 0000000..a4567f9
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/util/StringIdentifier.java
@@ -0,0 +1,77 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.network.util;
+
+import org.apache.reef.wake.ComparableIdentifier;
+import org.apache.reef.wake.Identifier;
+
+/**
+ * String identifier
+ */
+public class StringIdentifier implements ComparableIdentifier {
+
+ private final String str;
+
+ /**
+ * Constructs a string identifier
+ *
+ * @param str a string
+ */
+ StringIdentifier(String str) {
+ this.str = str;
+ }
+
+ /**
+ * Returns a hash code for the object
+ *
+ * @return a hash code value for this object
+ */
+ public int hashCode() {
+ return str.hashCode();
+ }
+
+ /**
+ * Checks that another object is equal to this object
+ *
+ * @param o another object
+ * @return true if the object is the same as the object argument; false, otherwise
+ */
+ public boolean equals(Object o) {
+ return str.equals(((StringIdentifier) o).toString());
+ }
+
+ /**
+ * Returns a string representation of the object
+ *
+ * @return a string representation of the object
+ */
+ public String toString() {
+ return str;
+ }
+
+ @Override
+ public int compareTo(Identifier o) {
+ if (o == null) {
+ if (str == null)
+ return 0;
+ return 1;
+ } else {
+ if (str == null)
+ return -1;
+ return str.compareTo(o.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/util/StringIdentifierFactory.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/util/StringIdentifierFactory.java b/reef-io/src/main/java/org/apache/reef/io/network/util/StringIdentifierFactory.java
new file mode 100644
index 0000000..2c65e27
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/util/StringIdentifierFactory.java
@@ -0,0 +1,43 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.network.util;
+
+import org.apache.reef.wake.Identifier;
+import org.apache.reef.wake.IdentifierFactory;
+
+import javax.inject.Inject;
+
+/**
+ * Factory that creates StringIdentifier
+ */
+public class StringIdentifierFactory implements IdentifierFactory {
+
+ @Inject
+ public StringIdentifierFactory() {
+ }
+
+ /**
+ * Creates a StringIdentifier object
+ *
+ * @param s a string
+ * @return a string identifier
+ */
+ @Override
+ public Identifier getNewInstance(String s) {
+ return new StringIdentifier(s);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/network/util/package-info.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/network/util/package-info.java b/reef-io/src/main/java/org/apache/reef/io/network/util/package-info.java
new file mode 100644
index 0000000..b393009
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/network/util/package-info.java
@@ -0,0 +1,16 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.network.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/FramingInputStream.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/FramingInputStream.java b/reef-io/src/main/java/org/apache/reef/io/storage/FramingInputStream.java
new file mode 100644
index 0000000..edb4963
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/FramingInputStream.java
@@ -0,0 +1,73 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage;
+
+import org.apache.reef.exception.evaluator.ServiceRuntimeException;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+
+public class FramingInputStream extends DataInputStream implements Iterable<byte[]> {
+
+ public FramingInputStream(InputStream in) {
+ super(in);
+ }
+
+ public byte[] readFrame() throws IOException {
+ int i = readInt();
+ if (i == -1) {
+ return null;
+ }
+ byte[] b = new byte[i];
+ readFully(b);
+ return b;
+ }
+
+ @Override
+ public Iterator<byte[]> iterator() {
+ try {
+ return new Iterator<byte[]>() {
+ byte[] cur = readFrame();
+
+ @Override
+ public boolean hasNext() {
+ return cur != null;
+ }
+
+ @Override
+ public byte[] next() {
+ byte[] ret = cur;
+ try {
+ cur = readFrame();
+ } catch (IOException e) {
+ throw new ServiceRuntimeException(e);
+ }
+ return ret;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ } catch (IOException e) {
+ throw new ServiceRuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/FramingOutputStream.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/FramingOutputStream.java b/reef-io/src/main/java/org/apache/reef/io/storage/FramingOutputStream.java
new file mode 100644
index 0000000..5d93669
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/FramingOutputStream.java
@@ -0,0 +1,131 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage;
+
+import org.apache.reef.exception.evaluator.ServiceException;
+import org.apache.reef.exception.evaluator.StorageException;
+import org.apache.reef.io.Accumulable;
+import org.apache.reef.io.Accumulator;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class FramingOutputStream extends OutputStream implements Accumulable<byte[]> {
+
+ private final ByteArrayOutputStream baos;
+ private final DataOutputStream o;
+ private long offset;
+ private boolean closed;
+
+ public FramingOutputStream(OutputStream o) {
+ if (!(o instanceof DataOutputStream)) {
+ this.o = new DataOutputStream(o);
+ } else {
+ this.o = (DataOutputStream) o;
+ }
+ this.baos = new ByteArrayOutputStream();
+ }
+
+ public void nextFrame() throws StorageException {
+ try {
+ o.writeInt(baos.size());
+ offset += 4;
+ baos.writeTo(o);
+ baos.reset();
+ } catch (IOException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ public long getCurrentOffset() {
+ return offset;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ baos.write(b);
+ offset++;
+ ;
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ baos.write(b);
+ offset += b.length;
+ }
+
+ @Override
+ public void write(byte[] b, int offset, int length) throws IOException {
+ baos.write(b, offset, length);
+ offset += length;
+ }
+
+ @Override
+ public void flush() {
+ // no-op.
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!closed) {
+ try {
+ if (this.offset > 0) nextFrame();
+ } catch (StorageException e) {
+ throw (IOException) e.getCause();
+ }
+ o.writeInt(-1);
+ o.close();
+ closed = true;
+ }
+ }
+
+ @Override
+ public Accumulator<byte[]> accumulator() throws StorageException {
+ @SuppressWarnings("resource")
+ final FramingOutputStream fos = this;
+ return new Accumulator<byte[]>() {
+
+ @Override
+ public void add(byte[] datum) throws ServiceException {
+ try {
+ o.writeInt(datum.length);
+ offset += 4;
+ o.write(datum);
+ offset += datum.length;
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+
+ }
+
+ @Override
+ public void close() throws ServiceException {
+ try {
+ o.writeInt(-1);
+ offset += 4;
+ o.close();
+ fos.closed = true;
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/FramingTupleDeserializer.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/FramingTupleDeserializer.java b/reef-io/src/main/java/org/apache/reef/io/storage/FramingTupleDeserializer.java
new file mode 100644
index 0000000..44dd965
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/FramingTupleDeserializer.java
@@ -0,0 +1,97 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage;
+
+import org.apache.reef.exception.evaluator.ServiceException;
+import org.apache.reef.exception.evaluator.ServiceRuntimeException;
+import org.apache.reef.io.Tuple;
+import org.apache.reef.io.serialization.Deserializer;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+public class FramingTupleDeserializer<K, V> implements
+ Deserializer<Tuple<K, V>, InputStream> {
+
+ private final Deserializer<K, InputStream> keyDeserializer;
+ private final Deserializer<V, InputStream> valDeserializer;
+
+ public FramingTupleDeserializer(Deserializer<K, InputStream> keyDeserializer,
+ Deserializer<V, InputStream> valDeserializer) {
+ this.keyDeserializer = keyDeserializer;
+ this.valDeserializer = valDeserializer;
+ }
+
+ @Override
+ public Iterable<Tuple<K, V>> create(InputStream ins) {
+ final DataInputStream in = new DataInputStream(ins);
+ final Iterable<K> keyItbl = keyDeserializer.create(in);
+ final Iterable<V> valItbl = valDeserializer.create(in);
+ return new Iterable<Tuple<K, V>>() {
+ @Override
+ public Iterator<Tuple<K, V>> iterator() {
+ final Iterator<K> keyIt = keyItbl.iterator();
+ final Iterator<V> valIt = valItbl.iterator();
+ try {
+ return new Iterator<Tuple<K, V>>() {
+
+ private int readFrameLength() throws ServiceException {
+ try {
+ return in.readInt();
+ } catch (IOException e) {
+ throw new ServiceRuntimeException(e);
+ }
+ }
+
+ int nextFrameLength = readFrameLength();
+
+ @Override
+ public boolean hasNext() {
+ return nextFrameLength != -1;
+ }
+
+ @Override
+ public Tuple<K, V> next() {
+ try {
+ if (nextFrameLength == -1) {
+ throw new NoSuchElementException();
+ }
+ K k = keyIt.next();
+ readFrameLength();
+ V v = valIt.next();
+ nextFrameLength = readFrameLength();
+ return new Tuple<>(k, v);
+ } catch (ServiceException e) {
+ throw new ServiceRuntimeException(e);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ } catch (ServiceException e) {
+ throw new ServiceRuntimeException(e);
+ }
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/FramingTupleSerializer.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/FramingTupleSerializer.java b/reef-io/src/main/java/org/apache/reef/io/storage/FramingTupleSerializer.java
new file mode 100644
index 0000000..4e745b0
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/FramingTupleSerializer.java
@@ -0,0 +1,83 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage;
+
+import org.apache.reef.exception.evaluator.ServiceException;
+import org.apache.reef.exception.evaluator.StorageException;
+import org.apache.reef.io.Accumulable;
+import org.apache.reef.io.Accumulator;
+import org.apache.reef.io.Tuple;
+import org.apache.reef.io.serialization.Serializer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class FramingTupleSerializer<K, V> implements
+ Serializer<Tuple<K, V>, OutputStream> {
+
+ private final Serializer<K, OutputStream> keySerializer;
+ private final Serializer<V, OutputStream> valSerializer;
+
+ public FramingTupleSerializer(
+ final Serializer<K, OutputStream> keySerializer,
+ final Serializer<V, OutputStream> valSerializer) {
+ this.keySerializer = keySerializer;
+ this.valSerializer = valSerializer;
+ }
+
+ @Override
+ public Accumulable<Tuple<K, V>> create(final OutputStream os) {
+ final FramingOutputStream faos = new FramingOutputStream(os);
+
+ return new Accumulable<Tuple<K, V>>() {
+
+ @Override
+ public Accumulator<Tuple<K, V>> accumulator() throws ServiceException {
+
+ final Accumulator<K> keyAccumulator = keySerializer.create(faos)
+ .accumulator();
+ final Accumulator<V> valAccumulator = valSerializer.create(faos)
+ .accumulator();
+ return new Accumulator<Tuple<K, V>>() {
+ boolean first = true;
+
+ @Override
+ public void add(Tuple<K, V> datum) throws ServiceException {
+ if (!first) {
+ faos.nextFrame();
+ }
+ first = false;
+ keyAccumulator.add(datum.getKey());
+ faos.nextFrame();
+ valAccumulator.add(datum.getValue());
+ }
+
+ @Override
+ public void close() throws ServiceException {
+ try {
+ keyAccumulator.close();
+ valAccumulator.close();
+ faos.close();
+ } catch (IOException e) {
+ throw new StorageException(e);
+ }
+ }
+ };
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/MergingIterator.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/MergingIterator.java b/reef-io/src/main/java/org/apache/reef/io/storage/MergingIterator.java
new file mode 100644
index 0000000..cef0405
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/MergingIterator.java
@@ -0,0 +1,59 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage;
+
+import org.apache.reef.io.Tuple;
+import org.apache.reef.io.storage.util.TupleKeyComparator;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.PriorityQueue;
+
+public class MergingIterator<T> implements Iterator<T> {
+ private final PriorityQueue<Tuple<T, Iterator<T>>> heap;
+
+ public MergingIterator(final Comparator<T> c, Iterator<T>[] its) {
+ this.heap = new PriorityQueue<Tuple<T, Iterator<T>>>(11,
+ new TupleKeyComparator<T, Iterator<T>>(c));
+
+ for (Iterator<T> it : its) {
+ T b = it.hasNext() ? it.next() : null;
+ if (b != null) {
+ heap.add(new Tuple<>(b, it));
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return heap.size() != 0;
+ }
+
+ @Override
+ public T next() {
+ Tuple<T, Iterator<T>> ret = heap.remove();
+ if (ret.getValue().hasNext()) {
+ heap.add(new Tuple<>(ret.getValue().next(), ret.getValue()));
+ }
+ return ret.getKey();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException(
+ "Cannot remove entires from MergingIterator!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/ScratchSpace.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/ScratchSpace.java b/reef-io/src/main/java/org/apache/reef/io/storage/ScratchSpace.java
new file mode 100644
index 0000000..5e747d2
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/ScratchSpace.java
@@ -0,0 +1,24 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage;
+
+public interface ScratchSpace {
+ long availableSpace();
+
+ long usedSpace();
+
+ void delete();
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/StorageService.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/StorageService.java b/reef-io/src/main/java/org/apache/reef/io/storage/StorageService.java
new file mode 100644
index 0000000..5adfad7
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/StorageService.java
@@ -0,0 +1,22 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage;
+
+public interface StorageService {
+
+ ScratchSpace getScratchSpace();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileAccumulable.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileAccumulable.java b/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileAccumulable.java
new file mode 100644
index 0000000..20d329b
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileAccumulable.java
@@ -0,0 +1,49 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.local;
+
+import org.apache.reef.exception.evaluator.StorageException;
+import org.apache.reef.io.Accumulable;
+import org.apache.reef.io.Accumulator;
+import org.apache.reef.io.serialization.Codec;
+
+import java.io.File;
+import java.io.IOException;
+
+public final class CodecFileAccumulable<T, C extends Codec<T>> implements Accumulable<T> {
+
+ private final File filename;
+ private final C codec;
+
+ public CodecFileAccumulable(final LocalStorageService s, final C codec) {
+ this.filename = s.getScratchSpace().newFile();
+ this.codec = codec;
+ }
+
+ public String getName() {
+ return this.filename.toString();
+ }
+
+ @Override
+ public Accumulator<T> accumulator() throws StorageException {
+ try {
+ return new CodecFileAccumulator<>(this.codec, this.filename);
+ } catch (final IOException e) {
+ throw new StorageException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileAccumulator.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileAccumulator.java b/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileAccumulator.java
new file mode 100644
index 0000000..ca73248
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileAccumulator.java
@@ -0,0 +1,55 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.local;
+
+import org.apache.reef.exception.evaluator.ServiceException;
+import org.apache.reef.exception.evaluator.StorageException;
+import org.apache.reef.io.Accumulator;
+import org.apache.reef.io.serialization.Codec;
+
+import java.io.*;
+
+final class CodecFileAccumulator<T> implements Accumulator<T> {
+
+ private final Codec<T> codec;
+ private final ObjectOutputStream out;
+
+ public CodecFileAccumulator(final Codec<T> codec, final File file) throws IOException {
+ this.codec = codec;
+ this.out = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
+ }
+
+ @Override
+ public void add(final T datum) throws ServiceException {
+ final byte[] buf = codec.encode(datum);
+ try {
+ this.out.writeInt(buf.length);
+ this.out.write(buf);
+ } catch (final IOException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
+ public void close() throws ServiceException {
+ try {
+ this.out.writeInt(-1);
+ this.out.close();
+ } catch (final IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileIterable.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileIterable.java b/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileIterable.java
new file mode 100644
index 0000000..5b7e7fe
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileIterable.java
@@ -0,0 +1,50 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.local;
+
+import org.apache.reef.exception.evaluator.ServiceRuntimeException;
+import org.apache.reef.exception.evaluator.StorageException;
+import org.apache.reef.io.serialization.Codec;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * A read-only spool implementation, based on files. Some other process needs to
+ * create the spool file for us.
+ */
+public class CodecFileIterable<T, C extends Codec<T>> implements Iterable<T> {
+ private final File filename;
+ private final C codec;
+
+ public CodecFileIterable(final File filename, final C codec) {
+ this.filename = filename;
+ this.codec = codec;
+ }
+
+ @SuppressWarnings("resource")
+ @Override
+ public Iterator<T> iterator() {
+ try {
+ return new CodecFileIterator<>(this.codec, this.filename);
+ } catch (final IOException e) {
+ throw new ServiceRuntimeException(new StorageException(e));
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileIterator.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileIterator.java b/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileIterator.java
new file mode 100644
index 0000000..7940a0c
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/local/CodecFileIterator.java
@@ -0,0 +1,79 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.local;
+
+import org.apache.reef.exception.evaluator.ServiceRuntimeException;
+import org.apache.reef.exception.evaluator.StorageException;
+import org.apache.reef.io.serialization.Codec;
+
+import java.io.*;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+final class CodecFileIterator<T> implements Iterator<T> {
+
+ private final Codec<T> codec;
+ private final ObjectInputStream in;
+ private int sz = 0;
+
+ CodecFileIterator(final Codec<T> codec, final File file) throws IOException {
+ this.in = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)));
+ this.codec = codec;
+ this.readNextSize();
+ }
+
+ private void readNextSize() throws IOException {
+ if (this.hasNext()) {
+ try {
+ this.sz = this.in.readInt();
+ if (this.sz == -1) {
+ this.in.close();
+ }
+ } catch (final IOException ex) {
+ this.sz = -1; // Don't read from that file again.
+ this.in.close();
+ throw ex;
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return this.sz != -1;
+ }
+
+ @Override
+ public T next() {
+ if (!this.hasNext()) {
+ throw new NoSuchElementException("Moving past the end of the file.");
+ }
+ try {
+ final byte[] buf = new byte[this.sz];
+ for (int rem = buf.length; rem > 0; ) {
+ rem -= this.in.read(buf, buf.length - rem, rem);
+ }
+ this.readNextSize();
+ return this.codec.decode(buf);
+ } catch (final IOException e) {
+ throw new ServiceRuntimeException(new StorageException(e));
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Attempt to remove value from read-only input file!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/local/LocalScratchSpace.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/local/LocalScratchSpace.java b/reef-io/src/main/java/org/apache/reef/io/storage/local/LocalScratchSpace.java
new file mode 100644
index 0000000..745d45a
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/local/LocalScratchSpace.java
@@ -0,0 +1,84 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.local;
+
+import org.apache.reef.io.storage.ScratchSpace;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+public class LocalScratchSpace implements ScratchSpace {
+
+ private final String jobName;
+ private final String evaluatorName;
+ private final Set<File> tempFiles = new ConcurrentSkipListSet<File>();
+ /**
+ * Zero denotes "unlimited"
+ */
+ private long quota;
+
+ public LocalScratchSpace(String jobName, String evaluatorName) {
+ this.jobName = jobName;
+ this.evaluatorName = evaluatorName;
+ this.quota = 0;
+ }
+
+ public LocalScratchSpace(String jobName, String evaluatorName, long quota) {
+ this.jobName = jobName;
+ this.evaluatorName = evaluatorName;
+ this.quota = quota;
+ }
+
+ public File newFile() {
+ final File ret;
+ try {
+ ret = File.createTempFile("reef-" + jobName + "-" + evaluatorName,
+ "tmp");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ tempFiles.add(ret);
+ return ret;
+ }
+
+ @Override
+ public long availableSpace() {
+ return quota;
+ }
+
+ @Override
+ public long usedSpace() {
+ long ret = 0;
+ for (File f : tempFiles) {
+ // TODO: Error handling...
+ ret += f.length();
+ }
+ return ret;
+ }
+
+ @Override
+ public void delete() {
+ // TODO: Error handling. Files.delete() would give us an exception. We
+ // should pass a set of Exceptions into a ReefRuntimeException.
+ for (File f : tempFiles) {
+ f.delete();
+ }
+ tempFiles.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/local/LocalStorageService.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/local/LocalStorageService.java b/reef-io/src/main/java/org/apache/reef/io/storage/local/LocalStorageService.java
new file mode 100644
index 0000000..f2ec3fb
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/local/LocalStorageService.java
@@ -0,0 +1,41 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.local;
+
+import org.apache.reef.io.storage.StorageService;
+
+
+public class LocalStorageService implements StorageService {
+ @SuppressWarnings("unused")
+ private final String jobName;
+ @SuppressWarnings("unused")
+ private final String evaluatorName;
+
+ private final LocalScratchSpace scratchSpace;
+
+ public LocalStorageService(String jobName, String evaluatorName) {
+ this.jobName = jobName;
+ this.evaluatorName = evaluatorName;
+ this.scratchSpace = new LocalScratchSpace(jobName, evaluatorName);
+ }
+
+ @Override
+ public LocalScratchSpace getScratchSpace() {
+ return scratchSpace;
+ }
+
+ ;
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/local/SerializerFileSpool.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/local/SerializerFileSpool.java b/reef-io/src/main/java/org/apache/reef/io/storage/local/SerializerFileSpool.java
new file mode 100644
index 0000000..992ed93
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/local/SerializerFileSpool.java
@@ -0,0 +1,98 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.local;
+
+import org.apache.reef.exception.evaluator.ServiceException;
+import org.apache.reef.exception.evaluator.ServiceRuntimeException;
+import org.apache.reef.io.Accumulable;
+import org.apache.reef.io.Accumulator;
+import org.apache.reef.io.Spool;
+import org.apache.reef.io.serialization.Deserializer;
+import org.apache.reef.io.serialization.Serializer;
+
+import java.io.*;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+
+/**
+ * A SpoolFile backed by the filesystem.
+ *
+ * @param <T>
+ */
+public final class SerializerFileSpool<T> implements Spool<T> {
+
+ private final File file;
+ private final Accumulator<T> accumulator;
+ private final Deserializer<T, InputStream> deserializer;
+ private boolean canAppend = true;
+ private boolean canGetAccumulator = true;
+
+ public SerializerFileSpool(final LocalStorageService service,
+ final Serializer<T, OutputStream> out, final Deserializer<T, InputStream> in)
+ throws ServiceException {
+ this.file = service.getScratchSpace().newFile();
+ Accumulable<T> accumulable;
+ try {
+ accumulable = out.create(new BufferedOutputStream(new FileOutputStream(
+ file)));
+ } catch (final FileNotFoundException e) {
+ throw new IllegalStateException(
+ "Unable to create temporary file:" + file, e);
+ }
+ this.deserializer = in;
+
+ final Accumulator<T> acc = accumulable.accumulator();
+ this.accumulator = new Accumulator<T>() {
+ @Override
+ public void add(final T datum) throws ServiceException {
+ if (!canAppend) {
+ throw new ConcurrentModificationException(
+ "Attempt to append after creating iterator!");
+ }
+ acc.add(datum);
+ }
+
+ @Override
+ public void close() throws ServiceException {
+ canAppend = false;
+ acc.close();
+ }
+ };
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ try {
+ if (canAppend) {
+ throw new IllegalStateException(
+ "Need to call close() on accumulator before calling iterator()!");
+ }
+ return deserializer.create(
+ new BufferedInputStream(new FileInputStream(file))).iterator();
+ } catch (final IOException e) {
+ throw new ServiceRuntimeException(e);
+ }
+ }
+
+ @Override
+ public Accumulator<T> accumulator() {
+ if (!canGetAccumulator) {
+ throw new UnsupportedOperationException("Can only getAccumulator() once!");
+ }
+ canGetAccumulator = false;
+ return this.accumulator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/ram/CodecRamMap.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/ram/CodecRamMap.java b/reef-io/src/main/java/org/apache/reef/io/storage/ram/CodecRamMap.java
new file mode 100644
index 0000000..715e9bc
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/ram/CodecRamMap.java
@@ -0,0 +1,81 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.ram;
+
+import org.apache.reef.io.ExternalMap;
+import org.apache.reef.io.serialization.Codec;
+import org.apache.reef.io.storage.util.GetAllIterable;
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+import org.apache.reef.tang.annotations.Parameter;
+
+import javax.inject.Inject;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+public class CodecRamMap<T> implements ExternalMap<T> {
+
+ private final Codec<T> c;
+ private final ConcurrentSkipListMap<CharSequence, byte[]> map;
+
+ @Inject
+ public CodecRamMap(RamStorageService ramStore,
+ @Parameter(RamMapCodec.class) final Codec<T> c) {
+ this.c = c;
+ this.map = new ConcurrentSkipListMap<CharSequence, byte[]>();
+ }
+
+ @Override
+ public boolean containsKey(final CharSequence key) {
+ return map.containsKey(key);
+ }
+
+ @Override
+ public T get(final CharSequence key) {
+ final byte[] ret = map.get(key);
+ return ret != null ? c.decode(ret) : null;
+ }
+
+ @Override
+ public T put(final CharSequence key, T value) {
+ final byte[] ret = map.put(key, c.encode(value));
+ return ret != null ? c.decode(ret) : null;
+ }
+
+ @Override
+ public T remove(final CharSequence key) {
+ final byte[] ret = map.remove(key);
+ return ret != null ? c.decode(ret) : null;
+ }
+
+ @Override
+ public void putAll(final Map<? extends CharSequence, ? extends T> m) {
+ for (final CharSequence x : m.keySet()) {
+ map.put(x, c.encode(m.get(x)));
+ }
+ }
+
+ @Override
+ public Iterable<Map.Entry<CharSequence, T>> getAll(
+ final Set<? extends CharSequence> keys) {
+ return new GetAllIterable<T>(keys, this);
+ }
+
+ @NamedParameter
+ static public class RamMapCodec implements Name<Codec<?>> {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamMap.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamMap.java b/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamMap.java
new file mode 100644
index 0000000..db89c2c
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamMap.java
@@ -0,0 +1,71 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.ram;
+
+import org.apache.reef.io.ExternalMap;
+import org.apache.reef.io.storage.util.GetAllIterable;
+
+import javax.inject.Inject;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+/**
+ * Simple in-memory ExternalMap implementation. This class does not require
+ * any codecs, and so is guaranteed to be instantiable. Therefore, it is the
+ * default ExternalMap provided by StorageManagerRam.
+ */
+public class RamMap<T> implements ExternalMap<T> {
+ private final ConcurrentSkipListMap<CharSequence, T> map
+ = new ConcurrentSkipListMap<CharSequence, T>();
+
+ @Inject
+ public RamMap(RamStorageService ramStore) {
+ //this.localStore = localStore;
+ }
+
+ @Override
+ public boolean containsKey(CharSequence key) {
+ return map.containsKey(key);
+ }
+
+ @Override
+ public T get(CharSequence key) {
+ return map.get(key);
+ }
+
+ @Override
+ public T put(CharSequence key, T value) {
+ return map.put(key, value);
+ }
+
+ @Override
+ public T remove(CharSequence key) {
+ return map.remove(key);
+ }
+
+ @Override
+ public void putAll(Map<? extends CharSequence, ? extends T> m) {
+ map.putAll(m);
+ }
+
+ @Override
+ public Iterable<Entry<CharSequence, T>> getAll(Set<? extends CharSequence> keys) {
+ return new GetAllIterable<>(keys, this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamSpool.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamSpool.java b/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamSpool.java
new file mode 100644
index 0000000..451178b
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamSpool.java
@@ -0,0 +1,69 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.ram;
+
+import org.apache.reef.io.Accumulator;
+import org.apache.reef.io.Spool;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A SpoolFile implementation that is backed by RAM.
+ * <p/>
+ * It uses an ArrayList to store the objects in.
+ */
+public final class RamSpool<T> implements Spool<T> {
+
+ private final List<T> backingStore = new ArrayList<T>();
+ private boolean canAppend = true;
+ private boolean canGetAccumulator = true;
+
+ @Inject
+ public RamSpool(RamStorageService ramStore) {
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ canAppend = false;
+ return backingStore.iterator();
+ }
+
+ @Override
+ public Accumulator<T> accumulator() {
+ if (!canGetAccumulator) {
+ throw new UnsupportedOperationException("Can only getAccumulator() once!");
+ }
+ canGetAccumulator = false;
+ return new Accumulator<T>() {
+ @Override
+ public void add(T datum) {
+ if (!canAppend) {
+ throw new ConcurrentModificationException("Attempt to append after creating iterator!");
+ }
+ backingStore.add(datum);
+ }
+
+ @Override
+ public void close() {
+ canAppend = false;
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamStorageService.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamStorageService.java b/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamStorageService.java
new file mode 100644
index 0000000..3339fab
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/ram/RamStorageService.java
@@ -0,0 +1,35 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.ram;
+
+import org.apache.reef.io.storage.ScratchSpace;
+import org.apache.reef.io.storage.StorageService;
+
+import javax.inject.Inject;
+
+public class RamStorageService implements StorageService {
+ @Inject
+ public RamStorageService() {
+ }
+
+ // TODO move getScratchSpace into its own class, just like everything else.
+ // TODO add context object or something to StorageService
+ @Override
+ public ScratchSpace getScratchSpace() {
+ throw new UnsupportedOperationException(
+ "No temp space / tracking of temp space for main memory (yet).");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/ram/SortingRamSpool.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/ram/SortingRamSpool.java b/reef-io/src/main/java/org/apache/reef/io/storage/ram/SortingRamSpool.java
new file mode 100644
index 0000000..d5ff6d2
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/ram/SortingRamSpool.java
@@ -0,0 +1,87 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.ram;
+
+import org.apache.reef.exception.evaluator.StorageException;
+import org.apache.reef.io.Accumulator;
+import org.apache.reef.io.Spool;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.PriorityQueue;
+
+public class SortingRamSpool<T> implements Spool<T> {
+ private final PriorityQueue<T> heap;
+ private boolean ready = false;
+ private Accumulator<T> acc = new Accumulator<T>() {
+ @Override
+ public void add(T datum) throws StorageException {
+ if (ready)
+ throw new IllegalStateException("add called after close!");
+ heap.add(datum);
+ }
+
+ @Override
+ public void close() throws StorageException {
+ ready = true;
+ }
+ };
+ private Iterator<T> it = new Iterator<T>() {
+
+ @Override
+ public boolean hasNext() {
+ return !heap.isEmpty();
+ }
+
+ @Override
+ public T next() {
+ return heap.remove();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException(
+ "This iterator consumes the data it returns. remove() does not make any sense!");
+ }
+
+ };
+
+ public SortingRamSpool() {
+ heap = new PriorityQueue<>();
+ }
+
+ public SortingRamSpool(Comparator<T> c) {
+ heap = new PriorityQueue<>(11, c);
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ if (!ready) {
+ throw new IllegalStateException("Cannot call iterator() while accumulator is still open!");
+ }
+ Iterator<T> ret = it;
+ it = null;
+ return ret;
+ }
+
+ @Override
+ public Accumulator<T> accumulator() throws StorageException {
+ Accumulator<T> ret = acc;
+ acc = null;
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/util/GetAllIterable.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/util/GetAllIterable.java b/reef-io/src/main/java/org/apache/reef/io/storage/util/GetAllIterable.java
new file mode 100644
index 0000000..5f922bc
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/util/GetAllIterable.java
@@ -0,0 +1,92 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.util;
+
+import org.apache.reef.io.ExternalMap;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+public class GetAllIterable<T> implements
+ Iterable<Map.Entry<CharSequence, T>> {
+ private final Set<? extends CharSequence> keys;
+ private final ExternalMap<T> map;
+
+ public GetAllIterable(Set<? extends CharSequence> keys, ExternalMap<T> map) {
+ this.keys = keys;
+ this.map = map;
+ }
+
+ @Override
+ public Iterator<Map.Entry<CharSequence, T>> iterator() {
+ final Iterator<? extends CharSequence> k = keys.iterator();
+ return new Iterator<Map.Entry<CharSequence, T>>() {
+ CharSequence lastKey = null;
+ CharSequence curKey = findNextKey();
+
+ private CharSequence findNextKey() {
+ while (k.hasNext()) {
+ CharSequence next = k.next();
+ if (map.containsKey(next)) {
+ return next;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return curKey != null;
+ }
+
+ @Override
+ public Map.Entry<CharSequence, T> next() {
+ final CharSequence key = curKey;
+ curKey = findNextKey();
+ lastKey = key;
+ if (key == null)
+ throw new NoSuchElementException();
+
+ final T v = map.get(key);
+
+ return new Map.Entry<CharSequence, T>() {
+ @Override
+ public CharSequence getKey() {
+ return key;
+ }
+
+ @Override
+ public T getValue() {
+ return v;
+ }
+
+ @Override
+ public T setValue(T v) {
+ throw new UnsupportedOperationException(
+ "No support for mutating values via iterator");
+ }
+ };
+ }
+
+ @Override
+ public void remove() {
+ map.remove(lastKey);
+ }
+ };
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerCodec.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerCodec.java b/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerCodec.java
new file mode 100644
index 0000000..3b064bd
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerCodec.java
@@ -0,0 +1,32 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.util;
+
+import org.apache.reef.io.serialization.Codec;
+
+public class IntegerCodec implements Codec<Integer> {
+
+ @Override
+ public byte[] encode(Integer obj) {
+ return Integer.toString(obj).getBytes();
+ }
+
+ @Override
+ public Integer decode(byte[] buf) {
+ return Integer.decode(new String(buf));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerDeserializer.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerDeserializer.java b/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerDeserializer.java
new file mode 100644
index 0000000..9a6dcc3
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerDeserializer.java
@@ -0,0 +1,59 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.util;
+
+import org.apache.reef.exception.evaluator.ServiceRuntimeException;
+import org.apache.reef.io.serialization.Deserializer;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+
+public class IntegerDeserializer implements
+ Deserializer<Integer, InputStream> {
+ @Override
+ public Iterable<Integer> create(InputStream arg) {
+ final DataInputStream dis = new DataInputStream(arg);
+ return new Iterable<Integer>() {
+
+ @Override
+ public Iterator<Integer> iterator() {
+ return new Iterator<Integer>() {
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Integer next() {
+ try {
+ return dis.readInt();
+ } catch (IOException e) {
+ throw new ServiceRuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerSerializer.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerSerializer.java b/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerSerializer.java
new file mode 100644
index 0000000..5892740
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/util/IntegerSerializer.java
@@ -0,0 +1,59 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.util;
+
+import org.apache.reef.exception.evaluator.ServiceException;
+import org.apache.reef.io.Accumulable;
+import org.apache.reef.io.Accumulator;
+import org.apache.reef.io.serialization.Serializer;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class IntegerSerializer implements
+ Serializer<Integer, OutputStream> {
+ @Override
+ public Accumulable<Integer> create(OutputStream arg) {
+ final DataOutputStream dos = new DataOutputStream(arg);
+ return new Accumulable<Integer>() {
+
+ @Override
+ public Accumulator<Integer> accumulator() throws ServiceException {
+ return new Accumulator<Integer>() {
+ @Override
+ public void add(Integer datum) throws ServiceException {
+ try {
+ dos.writeInt(datum);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public void close() throws ServiceException {
+ try {
+ dos.close();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ };
+ }
+ };
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/util/StringDeserializer.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/util/StringDeserializer.java b/reef-io/src/main/java/org/apache/reef/io/storage/util/StringDeserializer.java
new file mode 100644
index 0000000..86b79e5
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/util/StringDeserializer.java
@@ -0,0 +1,63 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.util;
+
+import org.apache.reef.exception.evaluator.ServiceRuntimeException;
+import org.apache.reef.io.serialization.Deserializer;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+
+public class StringDeserializer implements
+ Deserializer<String, InputStream> {
+ @Override
+ public Iterable<String> create(InputStream arg) {
+ final DataInputStream dis = new DataInputStream(arg);
+ return new Iterable<String>() {
+
+ @Override
+ public Iterator<String> iterator() {
+ return new Iterator<String>() {
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String next() {
+ int len = 0;
+ try {
+ len = dis.readInt();
+ byte[] b = new byte[len];
+ dis.readFully(b);
+ return new String(b);
+ } catch (IOException e) {
+ throw new ServiceRuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/util/StringSerializer.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/util/StringSerializer.java b/reef-io/src/main/java/org/apache/reef/io/storage/util/StringSerializer.java
new file mode 100644
index 0000000..d6c5955
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/util/StringSerializer.java
@@ -0,0 +1,62 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.util;
+
+import org.apache.reef.exception.evaluator.ServiceException;
+import org.apache.reef.io.Accumulable;
+import org.apache.reef.io.Accumulator;
+import org.apache.reef.io.serialization.Serializer;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class StringSerializer implements
+ Serializer<String, OutputStream> {
+ @Override
+ public Accumulable<String> create(OutputStream arg) {
+ final DataOutputStream dos = new DataOutputStream(arg);
+ return new Accumulable<String>() {
+
+ @Override
+ public Accumulator<String> accumulator() throws ServiceException {
+ return new Accumulator<String>() {
+
+ @Override
+ public void add(String datum) throws ServiceException {
+ byte[] b = datum.getBytes();
+ try {
+ dos.writeInt(b.length);
+ dos.write(b);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+
+ }
+
+ @Override
+ public void close() throws ServiceException {
+ try {
+ dos.close();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+ };
+ }
+ };
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/java/org/apache/reef/io/storage/util/TupleKeyComparator.java
----------------------------------------------------------------------
diff --git a/reef-io/src/main/java/org/apache/reef/io/storage/util/TupleKeyComparator.java b/reef-io/src/main/java/org/apache/reef/io/storage/util/TupleKeyComparator.java
new file mode 100644
index 0000000..c6d0828
--- /dev/null
+++ b/reef-io/src/main/java/org/apache/reef/io/storage/util/TupleKeyComparator.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright (C) 2014 Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.reef.io.storage.util;
+
+import org.apache.reef.io.Tuple;
+
+import java.util.Comparator;
+
+public final class TupleKeyComparator<K, V> implements
+ Comparator<Tuple<K, V>> {
+ private final Comparator<K> c;
+
+ public TupleKeyComparator(Comparator<K> c) {
+ this.c = c;
+ }
+
+ @Override
+ public int compare(Tuple<K, V> o1, Tuple<K, V> o2) {
+ return c.compare(o1.getKey(), o2.getKey());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/main/proto/ns_protocol.proto
----------------------------------------------------------------------
diff --git a/reef-io/src/main/proto/ns_protocol.proto b/reef-io/src/main/proto/ns_protocol.proto
index fa5d9f2..6693327 100644
--- a/reef-io/src/main/proto/ns_protocol.proto
+++ b/reef-io/src/main/proto/ns_protocol.proto
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-option java_package = "com.microsoft.reef.io.network.proto";
+option java_package = "org.apache.reef.io.network.proto";
option java_outer_classname = "ReefNetworkServiceProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/50444bba/reef-io/src/test/java/com/microsoft/reef/services/network/NameClientTest.java
----------------------------------------------------------------------
diff --git a/reef-io/src/test/java/com/microsoft/reef/services/network/NameClientTest.java b/reef-io/src/test/java/com/microsoft/reef/services/network/NameClientTest.java
deleted file mode 100644
index 6f604ac..0000000
--- a/reef-io/src/test/java/com/microsoft/reef/services/network/NameClientTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Copyright (C) 2014 Microsoft Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.microsoft.reef.services.network;
-
-import com.microsoft.reef.io.network.naming.NameCache;
-import com.microsoft.reef.io.network.naming.NameClient;
-import com.microsoft.reef.io.network.naming.NameLookupClient;
-import com.microsoft.reef.io.network.naming.NameServer;
-import com.microsoft.reef.io.network.naming.exception.NamingException;
-import com.microsoft.reef.io.network.util.StringIdentifierFactory;
-import com.microsoft.tang.Tang;
-import com.microsoft.tang.exceptions.InjectionException;
-import com.microsoft.wake.Identifier;
-import com.microsoft.wake.IdentifierFactory;
-import com.microsoft.wake.remote.NetUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.net.InetSocketAddress;
-import java.util.concurrent.ExecutionException;
-
-public class NameClientTest {
-
- static int retryCount, retryTimeout;
- static{
- Tang tang = Tang.Factory.getTang();
- try {
- retryCount = tang.newInjector().getNamedInstance(NameLookupClient.RetryCount.class);
- retryTimeout = tang.newInjector().getNamedInstance(NameLookupClient.RetryTimeout.class);
- } catch (InjectionException e1) {
- throw new RuntimeException("Exception while trying to find default values for retryCount & Timeout", e1);
- }
- }
-
- /**
- * @throws java.lang.Exception
- */
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- }
-
- /**
- * @throws java.lang.Exception
- */
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- }
-
- /**
- * Test method for {@link com.microsoft.reef.io.network.naming.NameClient#close()}.
- *
- * @throws Exception
- */
- @Test
- public final void testClose() throws Exception {
- IdentifierFactory factory = new StringIdentifierFactory();
- try (NameServer server = new NameServer(0, factory)) {
- int serverPort = server.getPort();
- try (NameClient client = new NameClient(NetUtils.getLocalAddress(), serverPort, factory, retryCount, retryTimeout,
- new NameCache(10000))) {
- Identifier id = factory.getNewInstance("Task1");
- client.register(id, new InetSocketAddress(NetUtils.getLocalAddress(), 7001));
- client.unregister(id);
- Thread.sleep(100);
- }
- }
- }
-
- /**
- * Test method for {@link com.microsoft.reef.io.network.naming.NameClient#lookup()}.
- * To check caching behavior with expireAfterAccess & expireAfterWrite
- * Changing NameCache's pattern to expireAfterAccess causes this test to fail
- *
- * @throws Exception
- */
- @Test
- public final void testLookup() throws Exception {
- IdentifierFactory factory = new StringIdentifierFactory();
- try (NameServer server = new NameServer(0, factory)) {
- int serverPort = server.getPort();
- try (NameClient client = new NameClient(NetUtils.getLocalAddress(), serverPort, factory, retryCount, retryTimeout,
- new NameCache(150))) {
- Identifier id = factory.getNewInstance("Task1");
- client.register(id, new InetSocketAddress(NetUtils.getLocalAddress(), 7001));
- client.lookup(id);// caches the entry
- client.unregister(id);
- Thread.sleep(100);
- try {
- InetSocketAddress addr = client.lookup(id);
- Thread.sleep(100);
- //With expireAfterAccess, the previous lookup would reset expiry to 150ms
- //more and 100ms wait will not expire the item and will return the cached value
- //With expireAfterWrite, the extra wait of 100 ms will expire the item
- //resulting in NamingException and the test passes
- addr = client.lookup(id);
- Assert.assertNull("client.lookup(id)", addr);
- } catch (Exception e) {
- if (e instanceof ExecutionException) {
- Assert.assertTrue("Execution Exception cause is instanceof NamingException", e.getCause() instanceof NamingException);
- } else
- throw e;
- }
- }
- }
- }
-
-}