You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by da...@apache.org on 2015/09/15 22:24:31 UTC
incubator-apex-core git commit: APEX-125 #resolve #comment add
long-lived option for app package
Repository: incubator-apex-core
Updated Branches:
refs/heads/devel-3 2cd917d9b -> cecdf4c73
APEX-125 #resolve #comment add long-lived option for app package
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/cecdf4c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/cecdf4c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/cecdf4c7
Branch: refs/heads/devel-3
Commit: cecdf4c7351eb3605acc7864bb50b1724993181d
Parents: 2cd917d
Author: siyuan <si...@datatorrent.com>
Authored: Tue Sep 15 13:10:38 2015 -0700
Committer: siyuan <si...@datatorrent.com>
Committed: Tue Sep 15 13:10:38 2015 -0700
----------------------------------------------------------------------
.../datatorrent/stram/client/AppPackage.java | 47 ++++++++++++++++++--
.../stram/client/AppPackageTest.java | 13 ++++++
2 files changed, 57 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/cecdf4c7/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java b/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java
index 5698807..f67860d 100644
--- a/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java
+++ b/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java
@@ -36,7 +36,7 @@ import org.slf4j.LoggerFactory;
*
* @since 1.0.3
*/
-public class AppPackage extends JarFile implements Closeable
+public class AppPackage extends JarFile
{
public static final String ATTRIBUTE_DT_ENGINE_VERSION = "DT-Engine-Version";
public static final String ATTRIBUTE_DT_APP_PACKAGE_NAME = "DT-App-Package-Name";
@@ -63,6 +63,7 @@ public class AppPackage extends JarFile implements Closeable
private final Set<String> configs = new TreeSet<String>();
private final File resourcesDirectory;
+ private final boolean cleanOnClose;
public static class AppInfo
{
@@ -97,14 +98,27 @@ public class AppPackage extends JarFile implements Closeable
* If app directory is to be processed, there may be resource leak in the class loader. Only pass true for short-lived
* applications
*
+ * If contentFolder is not null, it will try to create the contentFolder, file will be retained on disk after App Package is closed
+ * If contentFolder is null, temp folder will be created and will be cleaned on close()
+ *
* @param file
+ * @param contentFolder the folder that the app package will be extracted to
* @param processAppDirectory
* @throws java.io.IOException
* @throws net.lingala.zip4j.exception.ZipException
*/
- public AppPackage(File file, boolean processAppDirectory) throws IOException, ZipException
+ public AppPackage(File file, File contentFolder, boolean processAppDirectory) throws IOException, ZipException
{
super(file);
+
+ if (contentFolder != null) {
+ FileUtils.forceMkdir(contentFolder);
+ cleanOnClose = false;
+ } else {
+ cleanOnClose = true;
+ contentFolder = new File("/tmp/dt-appPackage-" + Long.toString(System.nanoTime()));
+ }
+
Manifest manifest = getManifest();
if (manifest == null) {
throw new IOException("Not a valid app package. MANIFEST.MF is not present.");
@@ -120,7 +134,7 @@ public class AppPackage extends JarFile implements Closeable
throw new IOException("Not a valid app package. Class-Path is missing from MANIFEST.MF");
}
classPath.addAll(Arrays.asList(StringUtils.split(classPathString, " ")));
- directory = new File("/tmp/dt-appPackage-" + Long.toString(System.nanoTime()));
+ directory = contentFolder;
extractToDirectory(directory, file);
if (processAppDirectory) {
processAppDirectory(new File(directory, "app"));
@@ -148,6 +162,25 @@ public class AppPackage extends JarFile implements Closeable
}
}
+ /**
+ * Creates an App Package object.
+ *
+ * If app directory is to be processed, there may be resource leak in the class loader. Only pass true for short-lived
+ * applications
+ *
+ * Files in app package will be extracted to tmp folder and will be cleaned on close()
+ * The close() method could be explicitly called or implicitly called by GC finalize()
+ *
+ * @param file
+ * @param processAppDirectory
+ * @throws java.io.IOException
+ * @throws net.lingala.zip4j.exception.ZipException
+ */
+ public AppPackage(File file, boolean processAppDirectory) throws IOException, ZipException
+ {
+ this(file, null, processAppDirectory);
+ }
+
public static void extractToDirectory(File directory, File appPackageFile) throws ZipException
{
ZipFile zipFile = new ZipFile(appPackageFile);
@@ -177,7 +210,15 @@ public class AppPackage extends JarFile implements Closeable
public void close() throws IOException
{
super.close();
+ if (cleanOnClose) {
+ cleanContent();
+ }
+ }
+
+ public void cleanContent() throws IOException
+ {
FileUtils.deleteDirectory(directory);
+ LOG.debug("App Package {}-{} folder {} is removed", appPackageName, appPackageVersion, directory.getAbsolutePath());
}
public String getAppPackageName()
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/cecdf4c7/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java b/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java
index bf41270..6d70eeb 100644
--- a/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java
@@ -18,6 +18,7 @@ package com.datatorrent.stram.client;
import com.datatorrent.stram.support.StramTestSupport;
import com.datatorrent.stram.util.JSONSerializationProvider;
import net.lingala.zip4j.exception.ZipException;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -37,6 +38,8 @@ import org.junit.BeforeClass;
public class AppPackageTest
{
private static AppPackage ap;
+ //yet another app package which retains the files
+ private static AppPackage yap;
private static JSONSerializationProvider jomp;
private static JSONObject json;
@@ -49,6 +52,9 @@ public class AppPackageTest
File file = StramTestSupport.createAppPackageFile();
// Set up test instance
ap = new AppPackage(file, true);
+ // set up another instance
+ File testfolder = new File("target/testapp");
+ yap = new AppPackage(file, testfolder, false);
jomp = new JSONSerializationProvider();
json = new JSONObject(jomp.getContext(null).writeValueAsString(ap));
@@ -58,6 +64,9 @@ public class AppPackageTest
throw new RuntimeException(e);
} catch (JSONException e) {
throw new RuntimeException(e);
+ } finally {
+ IOUtils.closeQuietly(ap);
+ IOUtils.closeQuietly(yap);
}
}
@@ -82,6 +91,10 @@ public class AppPackageTest
JSONObject dag = application.getJSONObject("dag");
Assert.assertTrue("There is at least one stream", dag.getJSONArray("streams").length() >= 1);
Assert.assertEquals("There are two operator", 2, dag.getJSONArray("operators").length());
+
+ Assert.assertTrue("app package extraction folder should be retained", new File("target/testapp").exists());
+ yap.cleanContent();
+ Assert.assertTrue("app package extraction folder should be removed", !new File("target/testapp").exists());
}
@Test