You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2015/01/23 00:47:06 UTC
[33/51] [partial] incubator-reef git commit: [REEF-93] Move java
sources to lang/java
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/EnvironmentUtils.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/EnvironmentUtils.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/EnvironmentUtils.java
new file mode 100644
index 0000000..9d19d35
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/EnvironmentUtils.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import org.apache.reef.tang.formats.ConfigurationModule;
+import org.apache.reef.tang.formats.OptionalParameter;
+import org.apache.reef.tang.formats.Param;
+
+import java.io.File;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public final class EnvironmentUtils {
+
+ private static final Logger LOG = Logger.getLogger(EnvironmentUtils.class.getName());
+
+ /**
+ * Get a set of all classpath entries EXCEPT of those under
+ * $JAVA_HOME, $YARN_HOME, and $HADOOP_HOME.
+ *
+ * @return A set of classpath entries as strings.
+ */
+ public static Set<String> getAllClasspathJars() {
+ return getAllClasspathJars(
+ "JAVA_HOME",
+ "YARN_HOME",
+ "HADOOP_HOME",
+ "HADOOP_YARN_HOME",
+ "HADOOP_HDFS_HOME",
+ "HADOOP_COMMON_HOME",
+ "HADOOP_MAPRED_HOME",
+ "YARN_CONF_DIR",
+ "HADOOP_CONF_DIR");
+ }
+
+ /**
+ * Get a set of all classpath entries EXCEPT of those under excludeEnv directories.
+ * Every excludeEnv entry is an environment variable name.
+ *
+ * @return A set of classpath entries as strings.
+ */
+ public static Set<String> getAllClasspathJars(final String... excludeEnv) {
+
+ final Set<String> jars = new HashSet<>();
+ final Set<Path> excludePaths = new HashSet<>();
+
+ for (final String env : excludeEnv) {
+ final String path = System.getenv(env);
+ if (null != path) {
+ final File file = new File(path);
+ if (file.exists()) {
+ excludePaths.add(file.toPath());
+ }
+ }
+ }
+
+ for (final String path : System.getProperty("java.class.path").split(File.pathSeparator)) {
+ try {
+ final File file = new File(path);
+ if (file.exists()) {
+ final Path absolutePath = file.toPath();
+ boolean toBeAdded = true;
+ for (final Path prefix : excludePaths) {
+ if (absolutePath.startsWith(prefix)) {
+ toBeAdded = false;
+ }
+ }
+ if (toBeAdded) {
+ jars.add(absolutePath.toString());
+ }
+ }
+ } catch (final InvalidPathException ex) {
+ LOG.log(Level.FINE, "Skip path: {0}: {1}", new Object[]{path, ex});
+ }
+ }
+
+ return jars;
+ }
+
+ /**
+ * @param config
+ * @param param
+ * @param values
+ * @param <P>
+ * @return
+ * @deprecated in 0.2 this really should be in Tang.
+ */
+ @Deprecated
+ public static <P extends Param> ConfigurationModule addAll(
+ ConfigurationModule config, final P param, final Iterable<String> values) {
+ for (final String val : values) {
+ config = config.set(param, val);
+ }
+ return config;
+ }
+
+ /**
+ * @param config
+ * @param param
+ * @return
+ * @deprecated Using this method is inherently non-deterministic as it depends on environment variables on your local
+ * machine.
+ */
+ @Deprecated
+ public static ConfigurationModule addClasspath(
+ final ConfigurationModule config, final OptionalParameter<String> param) {
+ return addAll(config, param, getAllClasspathJars());
+ }
+
+ /**
+ * Check whether assert() statements are evaluated.
+ *
+ * @return true, if assertions are enabled. False otherwise.
+ */
+ public static boolean areAssertionsEnabled() {
+ try {
+ assert false;
+ // If we got here, the assert above can't have thrown an exception. hence, asserts must be off.
+ return false;
+ } catch (final AssertionError assertionError) {
+ // The assert above threw an exception. Asserts must be enabled.
+ return true;
+ }
+ }
+
+ /**
+ * @param clazz
+ * @return the location (JAR or .class file) where the given class is located.
+ */
+ public static String getClassLocation(final Class<?> clazz) {
+ return clazz.getProtectionDomain().getCodeSource().getLocation().getFile();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/ExceptionHandlingEventHandler.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/ExceptionHandlingEventHandler.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/ExceptionHandlingEventHandler.java
new file mode 100644
index 0000000..00908a9
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/ExceptionHandlingEventHandler.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.wake.EventHandler;
+
+/**
+ * An ExceptionHandler that wraps another one, but catches all exceptions thrown by that one and forwards them
+ * to an ExceptionHandler.
+ *
+ * @param <T> the event type handled
+ */
+@Private
+public final class ExceptionHandlingEventHandler<T> implements EventHandler<T> {
+
+ private final EventHandler<T> wrapped;
+ private final EventHandler<Throwable> exceptionHandler;
+
+ public ExceptionHandlingEventHandler(final EventHandler<T> wrapped,
+ final EventHandler<Throwable> exceptionHandler) {
+ this.wrapped = wrapped;
+ this.exceptionHandler = exceptionHandler;
+ }
+
+ @Override
+ public void onNext(final T t) {
+ try {
+ this.wrapped.onNext(t);
+ } catch (final Throwable throwable) {
+ this.exceptionHandler.onNext(throwable);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/Exceptions.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/Exceptions.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/Exceptions.java
new file mode 100644
index 0000000..735f053
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/Exceptions.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+/**
+ * Utility class to deal with Exceptions
+ */
+public final class Exceptions {
+ private Exceptions() {
+ }
+
+ /**
+ * Walks the .getCause() chain till it hits the leaf node.
+ *
+ * @param throwable
+ * @return
+ */
+ public static Throwable getUltimateCause(final Throwable throwable) {
+ if (throwable.getCause() == null) {
+ return throwable;
+ } else {
+ return getUltimateCause(throwable.getCause());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/JARFileMaker.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/JARFileMaker.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/JARFileMaker.java
new file mode 100644
index 0000000..fb59c58
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/JARFileMaker.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import org.apache.commons.compress.utils.IOUtils;
+
+import java.io.*;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Helper class to create JAR files.
+ */
+public class JARFileMaker implements AutoCloseable {
+
+ private static final Logger LOG = Logger.getLogger(JARFileMaker.class.getName());
+
+ private final FileOutputStream fileOutputStream;
+ private final JarOutputStream jarOutputStream;
+ private String relativeStartCanonicalPath = null;
+
+ public JARFileMaker(final File outputFile, final Manifest manifest) throws IOException {
+ this.fileOutputStream = new FileOutputStream(outputFile);
+ this.jarOutputStream = new JarOutputStream(this.fileOutputStream, manifest);
+ }
+
+ public JARFileMaker(final File outputFile) throws IOException {
+ this.fileOutputStream = new FileOutputStream(outputFile);
+ this.jarOutputStream = new JarOutputStream(this.fileOutputStream);
+ }
+
+ /**
+ * Adds a file to the JAR. If inputFile is a folder, it will be added recursively.
+ *
+ * @param inputFile
+ * @throws IOException
+ */
+ public JARFileMaker add(final File inputFile) throws IOException {
+
+ final String fileNameInJAR = makeRelative(inputFile);
+ if (inputFile.isDirectory()) {
+ final JarEntry entry = new JarEntry(fileNameInJAR);
+ entry.setTime(inputFile.lastModified());
+ this.jarOutputStream.putNextEntry(entry);
+ this.jarOutputStream.closeEntry();
+ for (final File nestedFile : inputFile.listFiles()) {
+ add(nestedFile);
+ }
+ return this;
+ }
+
+ final JarEntry entry = new JarEntry(fileNameInJAR);
+ entry.setTime(inputFile.lastModified());
+ this.jarOutputStream.putNextEntry(entry);
+ try (final BufferedInputStream in = new BufferedInputStream(new FileInputStream(inputFile))) {
+ IOUtils.copy(in, this.jarOutputStream);
+ this.jarOutputStream.closeEntry();
+ } catch (final FileNotFoundException ex) {
+ LOG.log(Level.WARNING, "Skip the file: " + inputFile, ex);
+ }
+ return this;
+ }
+
+ public JARFileMaker addChildren(final File folder) throws IOException {
+ this.relativeStartCanonicalPath = folder.getCanonicalPath();
+ for (final File f : folder.listFiles()) {
+ this.add(f);
+ }
+ this.relativeStartCanonicalPath = null;
+ return this;
+ }
+
+ private String makeRelative(final File input) throws IOException {
+ final String result;
+ if (this.relativeStartCanonicalPath == null) {
+ result = input.getCanonicalPath();
+ } else {
+ result = input.getCanonicalPath()
+ .replace(this.relativeStartCanonicalPath, "") // Drop the absolute prefix
+ .substring(1); // drop the '/' at the beginning
+ }
+ if (input.isDirectory()) {
+ return result.replace("\\", "/") + "/";
+ } else {
+ return result.replace("\\", "/");
+ }
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.jarOutputStream.close();
+ this.fileOutputStream.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/MemoryUtils.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/MemoryUtils.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/MemoryUtils.java
new file mode 100644
index 0000000..5f9e6e7
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/MemoryUtils.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryPoolMXBean;
+import java.util.List;
+
+/**
+ * Utility class to report current and peak memory
+ * usage. Structured to be used while logging. Is
+ * useful for debugging memory issues
+ */
+public final class MemoryUtils {
+
+ private static final int MBs = 1024 * 1024;
+
+ private MemoryUtils() {
+ }
+
+ public static String memPoolNames() {
+ final List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
+ final StringBuilder output = new StringBuilder();
+ for (final MemoryPoolMXBean bean : memoryPoolMXBeans) {
+ output.append(bean.getName());
+ output.append(",");
+ }
+ output.deleteCharAt(output.length() - 1);
+ return output.toString();
+ }
+
+ public static long currentEdenMemoryUsageMB() {
+ return currentMemoryUsage("eden");
+ }
+
+ public static long currentOldMemoryUsageMB() {
+ return currentMemoryUsage("old");
+ }
+
+ public static long currentPermMemoryUsageMB() {
+ return currentMemoryUsage("perm");
+ }
+
+ private static long currentMemoryUsage(final String name) {
+ final List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
+ for (final MemoryPoolMXBean bean : memoryPoolMXBeans) {
+ if (bean.getName().toLowerCase().indexOf(name) != -1) {
+ return bean.getUsage().getUsed() / MBs;
+ }
+ }
+ return 0;
+ }
+
+ public static long peakEdenMemoryUsageMB() {
+ return peakMemoryUsage("eden");
+ }
+
+ public static long peakOldMemoryUsageMB() {
+ return peakMemoryUsage("old");
+ }
+
+ public static long peakPermMemoryUsageMB() {
+ return peakMemoryUsage("perm");
+ }
+
+ private static long peakMemoryUsage(final String name) {
+ final List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
+ for (final MemoryPoolMXBean bean : memoryPoolMXBeans) {
+ if (bean.getName().toLowerCase().indexOf(name) != -1) {
+ return bean.getPeakUsage().getUsed() / MBs;
+ }
+ }
+ return 0;
+ }
+
+ public static void resetPeakUsage() {
+ final List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
+ for (final MemoryPoolMXBean memoryPoolMXBean : memoryPoolMXBeans) {
+ memoryPoolMXBean.resetPeakUsage();
+ }
+ }
+
+ public static void main(final String[] args) {
+ System.out.println(memPoolNames());
+ {
+ final byte[] b = new byte[1 << 24];
+ System.out.println(currentEdenMemoryUsageMB()
+ + "," + currentOldMemoryUsageMB()
+ + "," + currentPermMemoryUsageMB());
+ }
+
+ System.gc();
+ System.out.println(currentEdenMemoryUsageMB()
+ + "," + currentOldMemoryUsageMB()
+ + "," + currentPermMemoryUsageMB());
+ System.out.println(peakEdenMemoryUsageMB()
+ + "," + peakOldMemoryUsageMB()
+ + "," + peakPermMemoryUsageMB());
+ resetPeakUsage();
+ System.out.println(peakEdenMemoryUsageMB()
+ + "," + peakOldMemoryUsageMB()
+ + "," + peakPermMemoryUsageMB());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/OSUtils.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/OSUtils.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/OSUtils.java
new file mode 100644
index 0000000..b6cd16e
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/OSUtils.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public final class OSUtils {
+ private static final Logger LOG = Logger.getLogger(OSUtils.class.getName());
+
+ private OSUtils() {
+ }
+
+ /**
+ * Determines whether the current JVM is running on the Windows OS.
+ *
+ * @return true, if the JVM is running on Windows. false, otherwise
+ */
+ public static boolean isWindows() {
+ return System.getProperty("os.name").toLowerCase().contains("windows");
+ }
+
+ /**
+ * Determines whether the current JVM is running on the Linux OS.
+ *
+ * @return true, if the JVM is running on Linux. false, otherwise
+ */
+ public static boolean isLinux() {
+ return System.getProperty("os.name").toLowerCase().contains("linux");
+ }
+
+ /**
+ * @return the process ID of the JVM, if running on Linux. This returns -1 for other OSs.
+ */
+ public static long getPID() {
+ if (isLinux()) {
+ try {
+ final Process process = new ProcessBuilder()
+ .command("bash", "-c", "echo $PPID")
+ .start();
+ final byte[] returnBytes = new byte[128];
+ process.getInputStream().read(returnBytes);
+ final Long result = Long.valueOf(new String(returnBytes).trim());
+ process.destroy();
+ return result;
+ } catch (final IOException e) {
+ LOG.log(Level.SEVERE, "Unable to determine PID", e);
+ return -1;
+ }
+
+ } else {
+ return -1;
+ }
+ }
+
+ /**
+ * Applies `kill -9` to the process.
+ *
+ * @param pid
+ * @throws IOException
+ */
+ public static void kill(final long pid) throws IOException, InterruptedException {
+ if (isLinux()) {
+ final Process process = new ProcessBuilder()
+ .command("bash", "-c", "kill", "-9", String.valueOf(pid))
+ .start();
+ final int returnCode = process.waitFor();
+ LOG.fine("Kill returned: " + returnCode);
+ } else {
+ throw new UnsupportedOperationException("Unable to execute kill on non-linux OS");
+ }
+ }
+
+ /**
+ * Formats the given variable for expansion by Windows (<code>%VARIABE%</code>) or Linux (<code>$VARIABLE</code>)
+ *
+ * @param variableName
+ * @return
+ */
+ public static String formatVariable(final String variableName) {
+ if (isWindows()) {
+ return "%" + variableName + "%";
+ } else {
+ return "$" + variableName;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/ObjectInstantiationLogger.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/ObjectInstantiationLogger.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/ObjectInstantiationLogger.java
new file mode 100644
index 0000000..5374e95
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/ObjectInstantiationLogger.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import javax.inject.Inject;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A utility class used as a default at times.
+ */
+public class ObjectInstantiationLogger {
+ @Inject
+ public ObjectInstantiationLogger() {
+ Logger.getLogger(this.getClass().getName()).log(Level.FINER, "Instantiated");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/REEFVersion.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/REEFVersion.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/REEFVersion.java
new file mode 100644
index 0000000..1b74e7f
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/REEFVersion.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Version information, retrieved from the pom (via a properties file reference)
+ */
+public final class REEFVersion {
+
+ private final static Logger LOG = Logger.getLogger(REEFVersion.class.getName());
+
+ private final static String FILENAME = "version.properties";
+ private final static String VERSION_KEY = "version";
+ private final static String VERSION_DEFAULT = "unknown";
+
+ private final String version;
+
+ @Inject
+ public REEFVersion() {
+ this.version = loadVersion();
+ }
+
+ /**
+ * Logs the version of REEF into the log Level INFO.
+ */
+ public void logVersion() {
+ this.logVersion(Level.INFO);
+ }
+
+ /**
+ * Logs the version of REEF into the given logLevel.
+ *
+ * @param logLevel The level to use in the log.
+ */
+ public void logVersion(final Level logLevel) {
+ LOG.log(logLevel, "REEF Version: {0}", this.version);
+ }
+
+ /**
+ * @return the version string for REEF.
+ */
+ public String getVersion() {
+ return version;
+ }
+
+ private static String loadVersion() {
+ String version;
+ try (final InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(FILENAME)) {
+ if (is == null) {
+ throw new IOException(FILENAME + " not found");
+ }
+ final Properties properties = new Properties();
+ properties.load(is);
+ version = properties.getProperty(VERSION_KEY, VERSION_DEFAULT);
+ } catch (IOException e) {
+ LOG.log(Level.WARNING, "Could not find REEF version");
+ version = VERSION_DEFAULT;
+ }
+ return version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/SetOnce.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/SetOnce.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/SetOnce.java
new file mode 100644
index 0000000..7408078
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/SetOnce.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+/**
+ * A reference to a value that can be set exactly once.
+ */
+public final class SetOnce<T> {
+
+ private Optional<T> value;
+
+ public SetOnce(final T value) {
+ this.set(value);
+ }
+
+ public SetOnce() {
+ this.value = Optional.empty();
+ }
+
+ public synchronized T get() {
+ return value.get();
+ }
+
+ public synchronized void set(final T value) {
+ if (this.value.isPresent()) {
+ throw new IllegalStateException("Trying to set new value " + value +
+ " while an old value was already present: " + this.value);
+ }
+ this.value = Optional.of(value);
+ }
+
+ public synchronized boolean isSet() {
+ return this.value.isPresent();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/SingletonAsserter.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/SingletonAsserter.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/SingletonAsserter.java
new file mode 100644
index 0000000..08ac456
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/SingletonAsserter.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A helper class that can be used to ensure that objects are only instantiated once.
+ */
+public final class SingletonAsserter {
+
+ private static final Set<Class> classes = Collections.synchronizedSet(new HashSet<Class>());
+
+ /**
+ * This class operates purely in static mode.
+ */
+ private SingletonAsserter() {
+ }
+
+ public static boolean assertSingleton(final Class clazz) {
+ return classes.add(clazz);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/ThreadLogger.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/ThreadLogger.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/ThreadLogger.java
new file mode 100644
index 0000000..9a47184
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/ThreadLogger.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util;
+
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Methods to log the currently active set of threads with their stack traces. This is useful to log abnormal
+ * process exit situations, for instance the Driver timeout in the tests.
+ */
+public final class ThreadLogger {
+
+ /**
+ * This is a utility class that shall not be instantiated.
+ */
+ private ThreadLogger() {
+ }
+
+ /**
+ * Same as <code>logThreads(logger, level, prefix, "\n\t", "\n\t\t")</code>
+ */
+ public static void logThreads(final Logger logger, final Level level, final String prefix) {
+ logThreads(logger, level, prefix, "\n\t", "\n\t\t");
+ }
+
+ /**
+ * Logs the currently active threads and their stack trace to the given Logger and Level.
+ *
+ * @param logger the Logger instance to log to.
+ * @param level the Level to log into.
+ * @param prefix a prefix of the log message.
+ * @param threadPrefix logged before each thread, e.g. "\n\t" to create an indented list.
+ * @param stackElementPrefix logged before each stack trace element, e.g. "\n\t\t" to create an indented list.
+ */
+ public static void logThreads(
+ final Logger logger, final Level level, final String prefix,
+ final String threadPrefix, final String stackElementPrefix) {
+ logger.log(level, getFormattedThreadList(prefix, threadPrefix, stackElementPrefix));
+ }
+
+ /**
+ * Produces a String representation of the currently running threads.
+ *
+ * @param prefix The prefix of the string returned.
+ * @param threadPrefix Printed before each thread, e.g. "\n\t" to create an indented list.
+ * @param stackElementPrefix Printed before each stack trace element, e.g. "\n\t\t" to create an indented list.
+ * @return a String representation of the currently running threads.
+ */
+ public static String getFormattedThreadList(
+ final String prefix, final String threadPrefix, final String stackElementPrefix) {
+ final StringBuilder message = new StringBuilder(prefix);
+ for (final Map.Entry<Thread, StackTraceElement[]> entry : Thread.getAllStackTraces().entrySet()) {
+ message.append(threadPrefix).append("Thread '").append(entry.getKey().getName()).append("':");
+ for (final StackTraceElement element : entry.getValue()) {
+ message.append(stackElementPrefix).append(element.toString());
+ }
+ }
+ return message.toString();
+ }
+
+ /**
+ * Same as <code>getFormattedThreadList(prefix, "\n\t", "\n\t\t")</code>
+ */
+ public static String getFormattedThreadList(final String prefix) {
+ return getFormattedThreadList(prefix, "\n\t", "\n\t\t");
+ }
+
+ /**
+ * An example how to use the above methods.
+ *
+ * @param args ignored.
+ */
+ public static void main(final String[] args) {
+ logThreads(Logger.getAnonymousLogger(), Level.INFO, "Threads active:");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/Config.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/Config.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/Config.java
new file mode 100644
index 0000000..b72ecf0
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/Config.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util.logging;
+
+import java.io.IOException;
+import java.util.logging.LogManager;
+
+public final class Config {
+
+ public Config() throws IOException {
+ LogManager.getLogManager().readConfiguration(
+ Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream("org/apache/reef/logging.properties"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LogLevelName.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LogLevelName.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LogLevelName.java
new file mode 100644
index 0000000..a25a578
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LogLevelName.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.util.logging;
+
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.annotations.NamedParameter;
+
+/**
+ * Name of a log Level as a string such as "INFO", "FINE"
+ */
+@NamedParameter(default_value = "FINE")
+public class LogLevelName implements Name<String> {
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LogParser.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LogParser.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LogParser.java
new file mode 100644
index 0000000..4f6ed04
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LogParser.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.util.logging;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+
+/**
+ * Parse logs for reporting
+ */
+public class LogParser {
+
+ public static String endIndicators[] = {
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.BRIDGE_SETUP,
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.EVALUATOR_SUBMIT,
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.EVALUATOR_BRIDGE_SUBMIT,
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.DRIVER_START,
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.EVALUATOR_LAUNCH,
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.EVALUATOR_ALLOCATED,
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.ACTIVE_CONTEXT,
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.HTTP_REQUEST,
+ LoggingScopeImpl.EXIT_PREFIX + LoggingScopeFactory.TASK_COMPLETE
+ };
+
+ public static String startIndicators[] = {
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.DRIVER_START,
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.BRIDGE_SETUP,
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.EVALUATOR_BRIDGE_SUBMIT,
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.EVALUATOR_SUBMIT,
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.EVALUATOR_ALLOCATED,
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.EVALUATOR_LAUNCH,
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.ACTIVE_CONTEXT,
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.HTTP_REQUEST,
+ LoggingScopeImpl.START_PREFIX + LoggingScopeFactory.TASK_COMPLETE
+ };
+
+ private LogParser() {
+ }
+
+ /**
+ * Get lines from a given file with a specified filter, trim the line by removing strings before removeBeforeToken and after removeAfterToken
+ * @param fileName
+ * @param filter
+ * @return
+ * @throws IOException
+ */
+ public static ArrayList<String> getFilteredLinesFromFile(final String fileName, final String filter, final String removeBeforeToken, final String removeAfterToken) throws IOException{
+ final ArrayList<String> filteredLines = new ArrayList<String>();
+ try (final FileReader fr = new FileReader(fileName)) {
+ try (final BufferedReader in = new BufferedReader(fr)) {
+ String line = "";
+ while ((line = in.readLine()) != null) {
+ if (line.trim().length() == 0) {
+ continue;
+ }
+ if (line.contains(filter)) {
+ String trimedLine;
+ if (removeBeforeToken != null) {
+ final String[] p = line.split(removeBeforeToken);
+ if (p.length > 1) {
+ trimedLine = p[p.length-1];
+ } else {
+ trimedLine = line.trim();
+ }
+ } else {
+ trimedLine = line.trim();
+ }
+ if (removeAfterToken != null) {
+ final String[] p = trimedLine.split(removeAfterToken);
+ if (p.length > 1) {
+ trimedLine = p[0];
+ }
+ }
+ filteredLines.add(trimedLine);
+ }
+ }
+ }
+ }
+ return filteredLines;
+ }
+
+ /**
+ * get lines from given file with specified filter
+ * @param fileName
+ * @param filter
+ * @return
+ * @throws IOException
+ */
+ public static ArrayList<String> getFilteredLinesFromFile(final String fileName, final String filter) throws IOException {
+ return getFilteredLinesFromFile(fileName, filter, null, null);
+ }
+
+ /**
+ * filter array list of lines and get the last portion of the line separated by the token, like ":::"
+ * @param original
+ * @param filter
+ * @return
+ */
+ public static ArrayList<String> filter(final ArrayList<String> original, final String filter, final String token) {
+ final ArrayList<String> result = new ArrayList<String>();
+ for (String line : original) {
+ if (line.contains(filter)) {
+ final String[] p = line.split(token);
+ if (p.length > 1) {
+ result.add(p[p.length-1]);
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * find lines that contain stage indicators. The stageIndicators must be in sequence which appear in the lines.
+ * @param lines
+ * @param stageIndicators
+ * @return
+ */
+ public static ArrayList<String> findStages(final ArrayList<String> lines, final String[] stageIndicators) {
+ ArrayList<String> stages = new ArrayList<String>();
+
+ int i = 0;
+ for (String line: lines) {
+ if (line.contains(stageIndicators[i])){
+ stages.add(stageIndicators[i]);
+ if (i < stageIndicators.length - 1) {
+ i++;
+ }
+ }
+ }
+ return stages;
+ }
+
+ public static ArrayList<String> mergeStages(ArrayList<String> startStages, ArrayList<String> endStages) {
+ ArrayList<String> mergeStage = new ArrayList<String>();
+ for (int i = 0; i < startStages.size(); i++) {
+ String end = startStages.get(i).replace(LoggingScopeImpl.START_PREFIX, LoggingScopeImpl.EXIT_PREFIX);
+ if (endStages.contains(end)) {
+ mergeStage.add(startStages.get(i) + " " + end);
+ } else {
+ mergeStage.add(startStages.get(i));
+ }
+ }
+ return mergeStage;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScope.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScope.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScope.java
new file mode 100644
index 0000000..56cb779
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScope.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.util.logging;
+
+/**
+ * Log time elapsed for a scope
+ */
+public interface LoggingScope extends AutoCloseable {
+
+ @Override
+ public void close();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScopeFactory.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScopeFactory.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScopeFactory.java
new file mode 100644
index 0000000..6f4bc54
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScopeFactory.java
@@ -0,0 +1,343 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.util.logging;
+
+import org.apache.reef.tang.JavaConfigurationBuilder;
+import org.apache.reef.tang.Tang;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.wake.time.event.StartTime;
+
+import javax.inject.Inject;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Create Logging scope objects
+ */
+public class LoggingScopeFactory {
+
+ private static final Logger LOG = Logger.getLogger(LoggingScopeFactory.class.getName());
+ public static final String DRIVER_START = "Driver Start Handler";
+ public static final String DRIVER_STOP = "Driver Stop Handler";
+ public static final String BRIDGE_SETUP = "Bridge setup";
+ public static final String LOAD_LIB = "Load libraries";
+ public static final String EVALUATOR_REQUESTOR = "Evaluator requestor passed to C#";
+ public static final String EVALUATOR_BRIDGE_SUBMIT = "Evaluator request submit cross bridge";
+ public static final String EVALUATOR_SUBMIT = "Evaluator submit";
+ public static final String EVALUATOR_LAUNCH = "Evaluator launch";
+ public static final String EVALUATOR_ALLOCATED = "Evaluator allocated";
+ public static final String EVALUATOR_COMPLETED = "Evaluator completed";
+ public static final String EVALUATOR_FAILED = "Evaluator failed";
+ public static final String ACTIVE_CONTEXT = "Active context created";
+ public static final String TASK_RUNNING = "Task running";
+ public static final String TASK_COMPLETE = "Task complete";
+ public static final String TASK_MESSAGE = "Task message";
+ public static final String CONTEXT_MESSAGE = "Context message";
+ public static final String CONTEXT_CLOSE = "Context close";
+ public static final String DRIVER_RESTART = "Driver restart";
+ public static final String DRIVER_RESTART_COMPLETE = "Driver restart complete";
+ public static final String DRIVER_RESTART_RUNNING_TASK = "Driver restart running task";
+ public static final String DRIVER_RESTART_ACTIVE_CONTEXT = "Driver restart active context";
+ public static final String TASK_SUSPEND = "Task suspend";
+ public static final String DRIVER_SUBMIT = "Driver submit";
+ public static final String REEF_SUBMIT = "Reef submit";
+ public static final String LOCAL_JOB_SUBMIT = "Local job submit";
+ public static final String HTTP_REQUEST = "Http request";
+ public static final String HTTP_SERVER = "Http server";
+
+ /**
+ * Log level. Client can set it through LogLevelName named parameter
+ */
+ private final Level logLevel;
+
+ /**
+ * User can inject a LoggingScopeFactory with injected log level as a string
+ */
+ @Inject
+ private LoggingScopeFactory(@Parameter(LogLevelName.class) final String logLevelName) {
+ this.logLevel = Level.parse(logLevelName);
+ }
+
+ /**
+ * Get a new instance of LoggingScope with specified log level
+ * @param logLevel
+ * @param msg
+ * @return
+ */
+ public static LoggingScope getNewLoggingScope(final Level logLevel, final String msg) {
+ return new LoggingScopeImpl(LOG, logLevel, msg);
+ }
+
+ /**
+ * Get a new instance of LoggingScope with injected LoggingScopeFactory instance
+ * @param msg
+ * @return
+ */
+ public LoggingScope getNewLoggingScope(final String msg) {
+ return new LoggingScopeImpl(LOG, logLevel, msg);
+ }
+
+ /**
+ * Get a new instance of LoggingScope with msg and params through new
+ * @param msg
+ * @param params
+ * @return
+ */
+ public LoggingScope getNewLoggingScope(final String msg, final Object params[]) {
+ return new LoggingScopeImpl(LOG, logLevel, msg, params);
+ }
+
+ /**
+ * The method is to measure the time used to start the driver. It can be inserted to the code between start driver till it is started
+ * @param startTime
+ * @return
+ */
+ public LoggingScope driverStart(final StartTime startTime) {
+ return new LoggingScopeImpl(LOG, logLevel, DRIVER_START + " :" + startTime);
+ }
+
+ /**
+ * The method is to measure the time used to stop the driver. It can be inserted to the code between start driver stop till it is stopped
+ * @param timeStamp
+ * @return
+ */
+ public LoggingScope driverStop(final long timeStamp) {
+ return new LoggingScopeImpl(LOG, logLevel, this.DRIVER_STOP + " :" + timeStamp);
+ }
+
+ /**
+ * The method is to measure the time used to set up Java CRL bridge. It can be inserted to the code between beginning of bridge set up and the end of it
+ * @return
+ */
+ public LoggingScope setupBridge() {
+ return new LoggingScopeImpl(LOG, logLevel, BRIDGE_SETUP);
+ }
+
+ /**
+ * The method is to measure the time used to load global files and libraries
+ * @return
+ */
+ public LoggingScope loadLib() {
+ return new LoggingScopeImpl(LOG, logLevel, LOAD_LIB);
+ }
+
+ /**
+ * The method is to measure the time used to pass EvaluatorRequestor from Java to .Net. It can be inserted to the code between beginning to send EvaluatorRequestor to CLR until it is returned.
+ * @return
+ */
+ public LoggingScope evaluatorRequestorPassToCs() {
+ return new LoggingScopeImpl(LOG, logLevel, EVALUATOR_REQUESTOR);
+ }
+
+ /**
+ * The method is to measure the time used to submit Evaluator request from CLR to Java driver. It can be inserted to evaluator submit() method.
+ * @param evaluatorsNumber
+ * @return
+ */
+ public LoggingScope evaluatorRequestSubmitToJavaDriver(final int evaluatorsNumber) {
+ return new LoggingScopeImpl(LOG, logLevel, EVALUATOR_BRIDGE_SUBMIT + ":" + evaluatorsNumber);
+ }
+
+ /**
+ * The method is to measure the time used to submit a Evaluator request at java side
+ * @param evaluatorNumber
+ * @return
+ */
+ public LoggingScope evaluatorSubmit(final int evaluatorNumber) {
+ return new LoggingScopeImpl(LOG, logLevel, EVALUATOR_SUBMIT + ":" + evaluatorNumber);
+ }
+
+ /**
+ * This is to measure the time on evaluatorAllocated handler
+ * @param evaluatorId
+ * @return
+ */
+ public LoggingScope evaluatorAllocated(final String evaluatorId) {
+ return new LoggingScopeImpl(LOG, logLevel, EVALUATOR_ALLOCATED + " :" + evaluatorId);
+ }
+
+ /**
+ * This is to measure the time to launch an evaluator
+ * @param evaluatorId
+ * @return
+ */
+ public LoggingScope evaluatorLaunch(final String evaluatorId) {
+ return new LoggingScopeImpl(LOG, logLevel, EVALUATOR_LAUNCH + " :" + evaluatorId);
+ }
+
+ /**
+ * This is to measure the time in calling evaluatorCompleted handler
+ * @param evaluatorId
+ * @return
+ */
+ public LoggingScope evaluatorCompleted(final String evaluatorId) {
+ return new LoggingScopeImpl(LOG, logLevel, EVALUATOR_COMPLETED + " :" + evaluatorId);
+ }
+
+ /**
+ * This is to measure the time in calling evaluatorFailed handler
+ * @param evaluatorId
+ * @return
+ */
+ public LoggingScope evaluatorFailed(final String evaluatorId) {
+ return new LoggingScopeImpl(LOG, logLevel, this.EVALUATOR_FAILED + " :" + evaluatorId);
+ }
+
+ /**
+ * This is to measure the time in calling activeContext handler
+ * @param contextId
+ * @return
+ */
+ public LoggingScope activeContextReceived(final String contextId) {
+ return new LoggingScopeImpl(LOG, logLevel, ACTIVE_CONTEXT + " :" + contextId);
+ }
+
+ /**
+ * This is to measure the time in calling closedContext handler
+ * @param contextId
+ * @return
+ */
+ public LoggingScope closedContext(final String contextId) {
+ return new LoggingScopeImpl(LOG, logLevel, this.CONTEXT_CLOSE + " :" + contextId);
+ }
+
+ /**
+ * This is to measure the time in calling runningTaskHandler
+ * @param taskId
+ * @return
+ */
+ public LoggingScope taskRunning(final String taskId) {
+ return new LoggingScopeImpl(LOG, logLevel, TASK_RUNNING + " :" + taskId);
+ }
+
+ /**
+ * This is to measure the time in calling taskCompletedHandler
+ * @param taskId
+ * @return
+ */
+ public LoggingScope taskCompleted(final String taskId) {
+ return new LoggingScopeImpl(LOG, logLevel, TASK_COMPLETE + " :" + taskId);
+ }
+
+ /**
+ * This is to measure the time in calling taskSuspendedHandler
+ * @param taskId
+ * @return
+ */
+ public LoggingScope taskSuspended(final String taskId) {
+ return new LoggingScopeImpl(LOG, logLevel, TASK_SUSPEND + " :" + taskId);
+ }
+
+ /**
+ * This is to measure the time in calling taskMessageReceivedHandler
+ * @param msg
+ * @return
+ */
+ public LoggingScope taskMessageReceived(final String msg) {
+ return new LoggingScopeImpl(LOG, logLevel, TASK_MESSAGE + " :" + msg);
+ }
+
+ /**
+ * This is to measure the time in calling contextMessageReceivedHandler
+ * @param msg
+ * @return
+ */
+ public LoggingScope contextMessageReceived(final String msg) {
+ return new LoggingScopeImpl(LOG, logLevel, CONTEXT_MESSAGE + " :" + msg);
+ }
+
+ /**
+ * This is to measure the time in calling driverRestartHandler
+ * @param startTime
+ * @return
+ */
+ public LoggingScope driverRestart(final StartTime startTime) {
+ return new LoggingScopeImpl(LOG, logLevel, DRIVER_RESTART + " :" + startTime);
+ }
+
+ /**
+ * This is to measure the time in calling driverRestartCompletedHandler
+ * @param timeStamp
+ * @return
+ */
+ public LoggingScope driverRestartCompleted(final long timeStamp) {
+ return new LoggingScopeImpl(LOG, logLevel, DRIVER_RESTART_COMPLETE + " :" + timeStamp);
+ }
+
+ /**
+ * This is to measure the time in calling driverRestartRunningTaskHandler
+ * @param taskId
+ * @return
+ */
+ public LoggingScope driverRestartRunningTask(final String taskId) {
+ return new LoggingScopeImpl(LOG, logLevel, DRIVER_RESTART_RUNNING_TASK + " :" + taskId);
+ }
+
+ /**
+ * This is to measure the time in calling driverRestartActiveContextReceivedHandler
+ * @param contextId
+ * @return
+ */
+ public LoggingScope driverRestartActiveContextReceived(final String contextId) {
+ return new LoggingScopeImpl(LOG, logLevel, DRIVER_RESTART_ACTIVE_CONTEXT + " :" + contextId);
+ }
+
+ /**
+ * This is to measure the time in handling a http request
+ * @param uri
+ * @return
+ */
+ public LoggingScope httpRequest(final String uri) {
+ return new LoggingScopeImpl(LOG, logLevel, this.HTTP_REQUEST + " :" + uri);
+ }
+
+ /**
+ * This is to measure the time used to create HttpServer
+ * @return
+ */
+ public LoggingScope httpServer() {
+ return new LoggingScopeImpl(LOG, logLevel, this.HTTP_SERVER);
+ }
+
+ /**
+ * This is to measure the time to submit a driver
+ * @param submitDriver
+ * @return
+ */
+ public LoggingScope driverSubmit(final Boolean submitDriver) {
+ return new LoggingScopeImpl(LOG, logLevel, DRIVER_SUBMIT + " :" + submitDriver);
+ }
+
+ /**
+ * This is to measure the time to call Reef.Submit
+ * @return
+ */
+ public LoggingScope reefSubmit() {
+ return new LoggingScopeImpl(LOG, logLevel, this.REEF_SUBMIT);
+ }
+
+ /**
+ * This is to measure the time for a job submission
+ * @return
+ */
+ public LoggingScope localJobSubmission() {
+ return new LoggingScopeImpl(LOG, logLevel, this.LOCAL_JOB_SUBMIT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScopeImpl.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScopeImpl.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScopeImpl.java
new file mode 100644
index 0000000..8475532
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingScopeImpl.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.util.logging;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.reef.util.Optional;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Log time and duration for a scope
+ */
+public class LoggingScopeImpl implements LoggingScope {
+ public static final String TOKEN = ":::";
+ public static final String START_PREFIX = "START" + TOKEN;
+ public static final String EXIT_PREFIX = "EXIT" + TOKEN;
+ public static final String DURATION = " Duration = ";
+
+ private final StopWatch stopWatch = new StopWatch();
+
+ private final Logger logger;
+
+ private final String msg;
+
+ private final Object[] params;
+
+ private final Optional<Object[]> optionalParams;
+
+ private final Level logLevel;
+
+ /**
+ * A constructor of ReefLoggingScope. It starts the timer and logs the msg
+ *
+ * @param logger
+ * @param msg
+ * @param params
+ */
+ LoggingScopeImpl(final Logger logger, final Level logLevel, final String msg, final Object params[]) {
+ this.logger = logger;
+ this.logLevel = logLevel;
+ this.msg = msg;
+ this.params = params;
+ stopWatch.start();
+ this.optionalParams = Optional.ofNullable(params);
+
+ if (logger.isLoggable(logLevel)) {
+ final StringBuilder sb = new StringBuilder();
+ log(sb.append(START_PREFIX).append(msg).toString());
+ }
+ }
+
+ /**
+ * A constructor of ReefLoggingScope. It starts the timer and and logs the msg.
+ *
+ * @param logger
+ * @param msg
+ */
+ LoggingScopeImpl(final Logger logger, final Level logLevel, final String msg) {
+ this(logger, logLevel, msg, null);
+ }
+
+ /**
+ * The close() will be called when the object is to deleted. It stops the timer and logs the time elapsed.
+ */
+ @Override
+ public void close() {
+ stopWatch.stop();
+
+ if (logger.isLoggable(logLevel)) {
+ final StringBuilder sb = new StringBuilder();
+ log(sb.append(EXIT_PREFIX).append(msg).append(DURATION).append(stopWatch.getTime()).toString());
+ }
+ }
+
+ /**
+ * log massage
+ * @param msg
+ */
+ private void log(final String msg) {
+ if (this.optionalParams.isPresent()) {
+ logger.log(logLevel, msg, params);
+ } else {
+ logger.log(logLevel, msg);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingSetup.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingSetup.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingSetup.java
new file mode 100644
index 0000000..7d683f0
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/LoggingSetup.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util.logging;
+
+/**
+ * Configure Commons Logging
+ */
+public final class LoggingSetup {
+
+ private LoggingSetup() {
+ }
+
+ /**
+ * Redirect the commons logging to the JDK logger.
+ */
+ public static void setupCommonsLogging() {
+ if (System.getProperty("org.apache.commons.logging.Log") == null) {
+ System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.Jdk14Logger");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/ThreadLogFormatter.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/ThreadLogFormatter.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/ThreadLogFormatter.java
new file mode 100644
index 0000000..36cfde6
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/logging/ThreadLogFormatter.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.util.logging;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.logging.Formatter;
+import java.util.logging.LogManager;
+import java.util.logging.LogRecord;
+
+/**
+ * A denser logging format for REEF that is similar to the standard SimpleFormatter.
+ * <p/>
+ * The following config properties are available:
+ * <p/>
+ * * `org.apache.reef.util.logging.ThreadLogFormatter.format`
+ * is a format string for String.format() that takes same arguments and in
+ * the same order as the standard SimpleFormatter, plus the thread name:
+ * 1. date
+ * 2. class and method name
+ * 3. logger name
+ * 4. logging level
+ * 5. message
+ * 6. stack trace
+ * 7. thread name
+ * <p/>
+ * * `org.apache.reef.util.logging.ThreadLogFormatter.dropPrefix`
+ * contains a comma-separated list of package name prefixes that should be
+ * removed from the class name for logging. e.g. value `com.microsoft.,org.apache.`
+ * will have the formatter write class `org.apache.reef.util.logging.Config` as
+ * `reef.util.logging.Config`. (Note the dot at the end of the prefix).
+ */
+public final class ThreadLogFormatter extends Formatter {
+
+ private static final String DEFAULT_FORMAT = "%1$tF %1$tT,%1$tL %4$s %2$s %7$s | %5$s%6$s%n";
+
+ private final List<String> dropPrefix = new ArrayList<>();
+ private final Date date = new Date();
+ private final String logFormat;
+
+ public ThreadLogFormatter() {
+
+ super();
+ final LogManager logManager = LogManager.getLogManager();
+ final String className = this.getClass().getName();
+
+ final String format = logManager.getProperty(className + ".format");
+ this.logFormat = format != null ? format : DEFAULT_FORMAT;
+
+ final String rawDropStr = logManager.getProperty(className + ".dropPrefix");
+ if (rawDropStr != null) {
+ for (String prefix : rawDropStr.trim().split(",")) {
+ prefix = prefix.trim();
+ if (!prefix.isEmpty()) {
+ this.dropPrefix.add(prefix);
+ }
+ }
+ }
+ }
+
+ /**
+ * Format the log string. Internally, it uses `String.format()` that takes same
+ * arguments and in the same order as the standard SimpleFormatter, plus the thread name:
+ * 1. date
+ * 2. class and method name
+ * 3. logger name
+ * 4. logging level
+ * 5. message
+ * 6. stack trace
+ * 7. thread name
+ *
+ * @return string to be written to the log.
+ */
+ @Override
+ public String format(final LogRecord logRecord) {
+ this.date.setTime(System.currentTimeMillis());
+ return String.format(
+ this.logFormat,
+ this.date,
+ this.trimPrefix(logRecord.getSourceClassName()) + "." + logRecord.getSourceMethodName(),
+ logRecord.getLoggerName(),
+ logRecord.getLevel().getLocalizedName(),
+ formatMessage(logRecord),
+ this.getStackTrace(logRecord.getThrown()),
+ Thread.currentThread().getName());
+ }
+
+ /**
+ * Check if the class name starts with one of the prefixes specified in `dropPrefix`,
+ * and remove it. e.g. for class name `org.apache.reef.util.logging.Config` and
+ * prefix `com.microsoft.` (note the trailing dot), the result will be
+ * `reef.util.logging.Config`
+ */
+ private String trimPrefix(final String className) {
+ for (final String prefix : this.dropPrefix) {
+ if (className.startsWith(prefix)) {
+ return className.substring(prefix.length());
+ }
+ }
+ return className;
+ }
+
+ /**
+ * @return a string that contains stack trace of a given exception.
+ * if `error` is null, return an empty string.
+ */
+ private String getStackTrace(final Throwable error) {
+ if (error != null) {
+ try (final StringWriter sw = new StringWriter();
+ final PrintWriter pw = new PrintWriter(sw)) {
+ pw.println();
+ error.printStackTrace(pw);
+ return sw.toString();
+ } catch (final IOException ex) {
+ // should never happen
+ throw new RuntimeException("Unexpected error while logging stack trace", ex);
+ }
+ }
+ return "";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/java/org/apache/reef/util/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/util/package-info.java b/lang/java/reef-common/src/main/java/org/apache/reef/util/package-info.java
new file mode 100644
index 0000000..f96eef5
--- /dev/null
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/util/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Various utility classes.
+ */
+package org.apache.reef.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/proto/client_runtime.proto
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/proto/client_runtime.proto b/lang/java/reef-common/src/main/proto/client_runtime.proto
new file mode 100644
index 0000000..36c648b
--- /dev/null
+++ b/lang/java/reef-common/src/main/proto/client_runtime.proto
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+option java_package = "org.apache.reef.proto";
+option java_outer_classname = "ClientRuntimeProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+import "reef_service_protos.proto";
+
+// Messages from REEF Client -> Driver Runtime
+
+message JobSubmissionProto {
+ required string identifier = 1; // the job identifier
+ required string remote_id = 2; // the remote identifier
+ required string configuration = 5; // the runtime configuration
+ required string user_name = 6; // the user name
+
+ //optional SIZE driver_size = 7; // Removed in REEF 0.3 in favor of driver_memory below.
+ optional int32 driver_memory = 8;
+ optional int32 priority = 9;
+ optional string queue = 10;
+
+ repeated FileResourceProto global_file = 11; // files that should be placed on the driver and all subsequent evaluators
+ repeated FileResourceProto local_File = 12; // files that should be placed on the driver only
+
+}
+
+enum Signal {
+ SIG_TERMINATE = 1;
+ SIG_SUSPEND = 2;
+ SIG_RESUME = 3;
+}
+
+message JobControlProto {
+ required string identifier = 1;
+ optional Signal signal = 2;
+ optional bytes message = 3;
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/proto/driver_runtime.proto
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/proto/driver_runtime.proto b/lang/java/reef-common/src/main/proto/driver_runtime.proto
new file mode 100644
index 0000000..64f5dc1
--- /dev/null
+++ b/lang/java/reef-common/src/main/proto/driver_runtime.proto
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+option java_package = "org.apache.reef.proto";
+option java_outer_classname = "DriverRuntimeProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+
+import "reef_service_protos.proto";
+
+// Messages from Driver Runtime -> Driver Process
+
+message DriverProcessRegistrationProto {
+ required string remote_identifier = 1;
+}
+
+
+message NodeDescriptorProto {
+ required string identifier = 1;
+ required string host_name = 2; // e.g., IP address
+ required int32 port = 3; // e.g., IP port
+ required int32 memory_size = 4;
+ optional string rack_name = 5; // e.g., /default-rack
+}
+
+message ResourceAllocationProto {
+ required string identifier = 1; // e.g., the container id, or the thread id
+ required int32 resource_memory = 2; // megabytes
+ required string node_id = 3;
+ optional int32 virtual_cores = 4;
+}
+
+message ResourceStatusProto {
+ required string identifier = 1;
+ required State state = 2;
+ optional string diagnostics = 3;
+ optional int32 exit_code = 4;
+ optional bool is_from_previous_driver = 5;
+}
+
+message RuntimeStatusProto {
+ required string name = 1; // e.g., local, yarn21
+ required State state = 2;
+ optional RuntimeErrorProto error = 3; // runtime (e.g., YARN) error
+
+ optional int32 outstanding_container_requests = 5;
+ repeated string container_allocation = 6;
+}
+
+//////////////////////////////////////////////////////
+// Messages from Driver Process -> Driver Runtime
+
+message ResourceRequestProto {
+ // optional SIZE resource_size = 1; // Removed in REEF 0.3 in favor of memory_size.
+ optional int32 memory_size = 2; // Memory size of the evaluator in MB
+ optional int32 priority = 3;
+ optional int32 virtual_cores = 4;
+ required int32 resource_count = 5;
+ repeated string node_name = 6; // a list of specific nodes
+ repeated string rack_name = 7; // a list of specific racks
+
+ optional bool relax_locality = 10;
+}
+
+message ResourceReleaseProto {
+ required string identifier = 1;
+}
+
+message ResourceLaunchProto {
+ required string identifier = 1;
+ required string remote_id = 2;
+ required string evaluator_conf = 3;
+ required ProcessType type = 4;
+ repeated FileResourceProto file = 10;
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/proto/evaluator_runtime.proto
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/proto/evaluator_runtime.proto b/lang/java/reef-common/src/main/proto/evaluator_runtime.proto
new file mode 100644
index 0000000..ea07ea0
--- /dev/null
+++ b/lang/java/reef-common/src/main/proto/evaluator_runtime.proto
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+option java_package = "org.apache.reef.proto";
+option java_outer_classname = "EvaluatorRuntimeProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+import "reef_service_protos.proto";
+
+// Stop the evaluator
+message StopEvaluatorProto {
+}
+
+// Kill the evaluator
+message KillEvaluatorProto {
+}
+
+// Start a task
+message StartTaskProto {
+ required string context_id = 1;
+ required string configuration = 2;
+}
+
+message AddContextProto {
+ required string parent_context_id = 1;
+ required string context_configuration = 2;
+ optional string service_configuration = 3;
+}
+
+message RemoveContextProto {
+ required string context_id = 1;
+}
+
+// Stop the task
+message StopTaskProto {
+}
+
+// Suspend the task
+message SuspendTaskProto {
+}
+
+/////////////////////////////////////////
+// Message aggregators
+
+message ContextMessageProto {
+ required string context_id = 1;
+ required bytes message = 2;
+}
+
+message ContextControlProto {
+ optional bytes task_message = 1;
+ optional ContextMessageProto context_message = 2;
+
+ optional AddContextProto add_context = 5;
+ optional RemoveContextProto remove_context = 6;
+ optional StartTaskProto start_task = 7;
+ optional StopTaskProto stop_task = 8;
+ optional SuspendTaskProto suspend_task = 9;
+}
+
+message EvaluatorHeartbeatProto {
+ required int64 timestamp = 1;
+ required EvaluatorStatusProto evaluator_status = 2;
+ repeated ContextStatusProto context_status = 3;
+ optional TaskStatusProto task_status = 4;
+ optional bool recovery = 5;
+}
+
+message EvaluatorControlProto {
+ required int64 timestamp = 1;
+ required string identifier = 2;
+
+ optional ContextControlProto context_control = 3;
+ optional KillEvaluatorProto kill_evaluator = 4;
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/proto/reef_protocol.proto
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/proto/reef_protocol.proto b/lang/java/reef-common/src/main/proto/reef_protocol.proto
new file mode 100644
index 0000000..a8c793f
--- /dev/null
+++ b/lang/java/reef-common/src/main/proto/reef_protocol.proto
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import "client_runtime.proto";
+
+import "evaluator_runtime.proto";
+
+import "reef_service_protos.proto";
+
+
+option java_package = "org.apache.reef.proto";
+
+option java_generic_services = true;
+
+option java_generate_equals_and_hash = true;
+
+option java_outer_classname = "REEFProtocol";
+
+message REEFMessage {
+ // Messages defined in client_runtime.proto
+ optional JobSubmissionProto jobSubmission = 1;
+ optional JobControlProto jobControl = 2;
+ // Messages defined in reef_service_protos.proto
+ optional RuntimeErrorProto runtimeError = 3;
+ optional JobStatusProto jobStatus = 4;
+ // Messages from evaluator_runtime.proto
+ optional EvaluatorControlProto evaluatorControl = 5;
+ optional EvaluatorHeartbeatProto evaluatorHeartBeat = 6;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/proto/reef_service_protos.proto
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/proto/reef_service_protos.proto b/lang/java/reef-common/src/main/proto/reef_service_protos.proto
new file mode 100644
index 0000000..7494737
--- /dev/null
+++ b/lang/java/reef-common/src/main/proto/reef_service_protos.proto
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+option java_package = "org.apache.reef.proto";
+
+option java_outer_classname = "ReefServiceProtos";
+
+option java_generic_services = true;
+
+option java_generate_equals_and_hash = true;
+
+enum State {
+ INIT = 0;
+ RUNNING = 1;
+ DONE = 2;
+ SUSPEND = 3;
+ FAILED = 4;
+ KILLED = 5;
+}
+
+enum FileType {
+ PLAIN = 0;
+ LIB = 1;
+ ARCHIVE = 2;
+}
+
+// Removed in REEF 0.3 in favor of explicit memory sizes.
+// enum SIZE {
+// SMALL = 0;
+// MEDIUM = 1;
+// LARGE = 2;
+// XLARGE = 3;
+//}
+
+enum ProcessType {
+ JVM = 0;
+ CLR = 1;
+}
+
+message FileResourceProto {
+ required FileType type = 1;
+ required string name = 2;
+ required string path = 3;
+}
+
+message RuntimeErrorProto {
+ required string name = 1; // e.g., local, yarn21
+ required string message = 2;
+ optional bytes exception = 3;
+
+ optional string identifier = 5; // e.g., evaluator id
+}
+
+message JobStatusProto {
+ required string identifier = 1;
+ required State state = 2;
+ optional bytes message = 3;
+ optional bytes exception = 4;
+}
+
+message ContextStatusProto {
+ enum State {
+ READY = 0;
+ DONE = 1;
+ FAIL = 2;
+ }
+ required State context_state = 1;
+
+ required string context_id = 2;
+ optional string parent_id = 3;
+
+ optional bytes error = 5; // when creating the context
+
+ optional bool recovery = 6;
+ // Context messages
+ message ContextMessageProto {
+ required string source_id = 1;
+ required bytes message = 2;
+ }
+ repeated ContextMessageProto context_message = 7;
+}
+
+message TaskStatusProto {
+ required string task_id = 1;
+ required string context_id = 2;
+ required State state = 3;
+ optional bytes result = 4; // e.g., return value from Task.call()
+ optional bool recovery = 5;
+
+ // TaskMessageSource messages
+ message TaskMessageProto {
+ required string source_id = 1;
+ required bytes message = 2;
+ }
+ repeated TaskMessageProto task_message = 6;
+}
+
+message EvaluatorStatusProto {
+ required string evaluator_id = 1;
+ required State state = 2;
+ optional bytes error = 3;
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/53ea32cc/lang/java/reef-common/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/resources/log4j.properties b/lang/java/reef-common/src/main/resources/log4j.properties
new file mode 100644
index 0000000..f80a941
--- /dev/null
+++ b/lang/java/reef-common/src/main/resources/log4j.properties
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Set root logger level to DEBUG and its only appender to stderr.
+log4j.rootLogger=DEBUG, stderr
+
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+
+# Pattern to output the caller's file name and line number.
+log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %p %C %M %t %m%n