You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2015/05/29 02:59:10 UTC
incubator-reef git commit: [REEF-348] Ignore and log changed
timestamps on global.jar
Repository: incubator-reef
Updated Branches:
refs/heads/master 429dc5e07 -> 8c4bb098c
[REEF-348] Ignore and log changed timestamps on global.jar
This change checks the time stamp on the `global.jar` for each Evaluator
launch. If it changed, the change will be logged as a WARNING.
*Note:* This doesn't address the root cause of [REEF-348], but it should
make the code more reliable. Also, the logging will allow us to debug
this further.
JIRA:
[REEF-348](https://issues.apache.org/jira/browse/REEF-348)
Pull Request:
This closes #196
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/8c4bb098
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/8c4bb098
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/8c4bb098
Branch: refs/heads/master
Commit: 8c4bb098c248a484e4f1e6cf1fbb742f7a6fd7fe
Parents: 429dc5e
Author: afchung <af...@gmail.com>
Authored: Thu May 28 16:03:59 2015 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Thu May 28 17:54:38 2015 -0700
----------------------------------------------------------------------
.../runtime/yarn/driver/GlobalJarUploader.java | 52 +++++++++++++++-----
1 file changed, 40 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/8c4bb098/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java
index c6ee91e..a0f4f81 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/GlobalJarUploader.java
@@ -29,6 +29,8 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Callable;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* Utility class that creates the JAR file with the global files on the driver and then uploads it to the job folder on
@@ -36,22 +38,34 @@ import java.util.concurrent.Callable;
*/
final class GlobalJarUploader implements Callable<Map<String, LocalResource>> {
+ private static final Logger LOG = Logger.getLogger(GlobalJarUploader.class.getName());
+
/**
* Used for the file system constants.
*/
private final REEFFileNames fileNames;
- /**
- * This will hold the actuall map to be used as the "global" resources when submitting Evaluators.
- */
- private final Map<String, LocalResource> globalResources = new HashMap<>(1);
+
/**
* Utility to actually perform the update.
*/
private final UploaderToJobFolder uploader;
+
/**
- * True, if globalResources contains the valid information which is cached after the first call to call().
+ * True, if the global JAR has already been uploaded.
*/
- private boolean isDone;
+ private boolean isUploaded;
+
+ /**
+ * Path to the uploaded global JAR.
+ */
+ private Path pathToGlobalJar;
+
+ /**
+ * The cached LocalResource for global JAR. The latest global JAR
+ * is still updated due to REEF-348.
+ * This is primarily used to detect change in JAR timestamps.
+ */
+ private LocalResource globalJarResource;
@Inject
GlobalJarUploader(final REEFFileNames fileNames,
@@ -69,13 +83,27 @@ final class GlobalJarUploader implements Callable<Map<String, LocalResource>> {
*/
@Override
public synchronized Map<String, LocalResource> call() throws IOException {
- if (!this.isDone) {
- final Path pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar());
- globalResources.put(this.fileNames.getGlobalFolderPath(),
- this.uploader.makeLocalResourceForJarFile(pathToGlobalJar));
- this.isDone = true;
+ final Map<String, LocalResource> globalResources = new HashMap<>(1);
+ if (!this.isUploaded){
+ this.pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar());
+ this.isUploaded = true;
+ }
+
+ LocalResource updatedGlobalJarResource = this.uploader.makeLocalResourceForJarFile(this.pathToGlobalJar);
+
+ if (this.globalJarResource != null
+ && this.globalJarResource.getTimestamp() != updatedGlobalJarResource.getTimestamp()) {
+ LOG.log(Level.WARNING,
+ "The global JAR LocalResource timestamp has been changed from "
+ + this.globalJarResource.getTimestamp() + " to " + updatedGlobalJarResource.getTimestamp());
}
- return this.globalResources;
+
+ this.globalJarResource = updatedGlobalJarResource;
+
+ // For now, always rewrite the information due to REEF-348
+ globalResources.put(this.fileNames.getGlobalFolderPath(), updatedGlobalJarResource);
+
+ return globalResources;
}
/**