You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2014/04/04 21:22:39 UTC
git commit: OOZIE-1761 Improve sharelib purging logic (puru via
rohini)
Repository: oozie
Updated Branches:
refs/heads/master 5045f89cf -> a015a45ee
OOZIE-1761 Improve sharelib purging logic (puru via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a015a45e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a015a45e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a015a45e
Branch: refs/heads/master
Commit: a015a45ee50a23add71ee84e8ef6a6bd00d8aa45
Parents: 5045f89
Author: Rohini Palaniswamy <ro...@yahoo-inc.com>
Authored: Fri Apr 4 12:22:38 2014 -0700
Committer: Rohini Palaniswamy <ro...@yahoo-inc.com>
Committed: Fri Apr 4 12:22:38 2014 -0700
----------------------------------------------------------------------
.../apache/oozie/service/ShareLibService.java | 45 +++++---
.../oozie/service/TestShareLibService.java | 113 ++++++++++++-------
release-log.txt | 1 +
3 files changed, 100 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/a015a45e/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index 89939f1..73ca492 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -92,6 +92,8 @@ public class ShareLibService implements Service, Instrumentable {
FileSystem fs;
+ final long retentionTime = 1000 * 60 * 60 * 24 * Services.get().getConf().getInt(LAUNCHERJAR_LIB_RETENTION, 7);
+
@Override
public void init(Services services) throws ServiceException {
this.services = services;
@@ -106,8 +108,9 @@ public class ShareLibService implements Service, Instrumentable {
updateShareLib();
//Only one server should purge sharelib
if (Services.get().get(JobsConcurrencyService.class).isFirstServer()) {
- purgeLibs(fs, LAUNCHER_PREFIX);
- purgeLibs(fs, SHARED_LIB_PREFIX);
+ final Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime();
+ purgeLibs(fs, LAUNCHER_PREFIX, current);
+ purgeLibs(fs, SHARED_LIB_PREFIX, current);
}
}
catch (Exception e) {
@@ -359,14 +362,26 @@ public class ShareLibService implements Service, Instrumentable {
* @throws IOException Signals that an I/O exception has occurred.
* @throws ParseException the parse exception
*/
- private void purgeLibs(FileSystem fs, final String prefix) throws IOException, ParseException {
- Configuration conf = services.getConf();
+ private void purgeLibs(FileSystem fs, final String prefix, final Date current) throws IOException, ParseException {
Path executorLibBasePath = services.get(WorkflowAppService.class).getSystemLibPath();
-
PathFilter directoryFilter = new PathFilter() {
@Override
public boolean accept(Path path) {
- return path.getName().startsWith(prefix);
+ if (path.getName().startsWith(prefix)) {
+ String name = path.getName().toString();
+ String time = name.substring(prefix.length());
+ Date d = null;
+ try {
+ d = dateFormat.parse(time);
+ }
+ catch (ParseException e) {
+ return false;
+ }
+ return (current.getTime() - d.getTime()) > retentionTime;
+ }
+ else {
+ return false;
+ }
}
};
FileStatus[] dirList = fs.listStatus(executorLibBasePath, directoryFilter);
@@ -376,20 +391,14 @@ public class ShareLibService implements Service, Instrumentable {
public int compare(FileStatus o1, FileStatus o2) {
return o2.getPath().getName().compareTo(o1.getPath().getName());
}
-
});
- Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime();
- // Always keep top two, so start counter from 3
- long retentionTime = 1000 * 60 * 60 * 24 * conf.getInt(LAUNCHERJAR_LIB_RETENTION, 7);
- for (int i = 2; i < dirList.length; i++) {
+
+ //Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days.
+ // refer OOZIE-1761
+ for (int i = 1; i < dirList.length; i++) {
Path dirPath = dirList[i].getPath();
- String name = dirPath.getName().toString();
- String time = name.substring(prefix.length());
- Date d = dateFormat.parse(time);
- if ((current.getTime() - d.getTime()) > retentionTime) {
- fs.delete(dirPath, true);
- LOG.info("Deleted old launcher jar lib directory {0}", dirPath.getName());
- }
+ fs.delete(dirPath, true);
+ LOG.info("Deleted old launcher jar lib directory {0}", dirPath.getName());
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a015a45e/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
index e7d7a26..e1c5c5f 100644
--- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Properties;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
@@ -50,6 +51,7 @@ public class TestShareLibService extends XFsTestCase {
Services services;
private static String testCaseDirPath;
String shareLibPath = "shareLibPath";
+ SimpleDateFormat dt = new SimpleDateFormat("yyyyMMddHHmmss");
@Override
protected void setUp() throws Exception {
@@ -253,32 +255,29 @@ public class TestShareLibService extends XFsTestCase {
Configuration conf = services.getConf();
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
FileSystem fs = getFileSystem();
+ long expiryTime = System.currentTimeMillis()
+ - TimeUnit.MILLISECONDS.convert(
+ services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS);
+
// for directory created 8 days back to be deleted
- int expire1 = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) + 1;
+ String expireTs = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
// for directory created 6 days back NOT to be deleted
- int noexpire = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) - 1;
+ String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
// for directory created 5 days back NOT to be deleted
- int noexpire1 = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) - 2;
-
- Date expireDate = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * expire1));
- Date noexpireDate = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * noexpire));
- Date noexpireDate1 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * noexpire1));
- String expireTs = new SimpleDateFormat("yyyyMMddHHmmss").format(expireDate);
- String noexpireTs = new SimpleDateFormat("yyyyMMddHHmmss").format(noexpireDate);
- String noexpireTs1 = new SimpleDateFormat("yyyyMMddHHmmss").format(noexpireDate1);
+ String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
+
Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
Path expirePath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + expireTs);
Path noexpirePath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + noexpireTs);
Path noexpirePath1 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + noexpireTs1);
- fs.mkdirs(expirePath);
- fs.mkdirs(noexpirePath);
- fs.mkdirs(noexpirePath1);
+
+ createDirs(fs, expirePath, noexpirePath, noexpirePath1);
try {
services.init();
- assertEquals(3, fs.listStatus(basePath).length);
+ assertEquals(4, fs.listStatus(basePath).length);
assertTrue(fs.exists(noexpirePath));
assertTrue(fs.exists(noexpirePath1));
- assertTrue(!fs.exists(expirePath));
+ assertTrue(fs.exists(expirePath));
}
finally {
services.destroy();
@@ -292,33 +291,66 @@ public class TestShareLibService extends XFsTestCase {
Configuration conf = services.getConf();
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
FileSystem fs = getFileSystem();
+
+ long expiryTime = System.currentTimeMillis()
+ - TimeUnit.MILLISECONDS.convert(
+ services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS);
+
// for directory created 8 days back to be deleted
- int expire1 = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) + 1;
+ String expireTs = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
// for directory created 6 days back NOT to be deleted
- int noexpire = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) - 1;
+ String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
// for directory created 5 days back NOT to be deleted
- int noexpire1 = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) - 2;
-
- Date expireDate = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * expire1));
- Date noexpireDate = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * noexpire));
- Date noexpireDate1 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * noexpire1));
- String expireTs = new SimpleDateFormat("yyyyMMddHHmmss").format(expireDate);
- String noexpireTs = new SimpleDateFormat("yyyyMMddHHmmss").format(noexpireDate);
- String noexpireTs1 = new SimpleDateFormat("yyyyMMddHHmmss").format(noexpireDate1);
+ String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
+
Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
Path expirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs);
Path noexpirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs);
Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs1);
+ createDirs(fs, expirePath, noexpirePath, noexpirePath1);
- fs.mkdirs(expirePath);
- fs.mkdirs(noexpirePath);
- fs.mkdirs(noexpirePath1);
services.init();
- assertEquals(3, fs.listStatus(basePath).length);
+ assertEquals(4, fs.listStatus(basePath).length);
assertTrue(fs.exists(noexpirePath));
assertTrue(fs.exists(noexpirePath1));
- assertTrue(!fs.exists(expirePath));
+ assertTrue(fs.exists(expirePath));
+ services.destroy();
+ }
+
+ // Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days.
+ // refer OOZIE-1761
+ @Test
+ public void testPurgeJar() throws Exception {
+ services = new Services();
+ setSystemProps();
+ Configuration conf = services.getConf();
+ conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+ FileSystem fs = getFileSystem();
+ // for directory created 8 days back to be deleted
+ long expiryTime = System.currentTimeMillis()
+ - TimeUnit.MILLISECONDS.convert(
+ services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS);
+
+ String expireTs = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
+ String expireTs1 = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
+ String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
+ String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
+ Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
+
+ Path expirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs);
+ Path expirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs1);
+ Path noexpirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs);
+ Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs1);
+
+ createDirs(fs, expirePath, expirePath1, noexpirePath, noexpirePath1);
+ services.init();
+ assertEquals(4, fs.listStatus(basePath).length);
+ assertTrue(fs.exists(noexpirePath));
+ assertTrue(fs.exists(noexpirePath1));
+ assertTrue(fs.exists(expirePath));
+ assertTrue(!fs.exists(expirePath1));
+
services.destroy();
}
@@ -355,21 +387,14 @@ public class TestShareLibService extends XFsTestCase {
Configuration conf = services.getConf();
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
FileSystem fs = getFileSystem();
-
- Date day1 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * 1));
- Date day2 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * 2));
- Date day3 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * 3));
- String dir1 = new SimpleDateFormat("yyyyMMddHHmmss").format(day1);
- String dir2 = new SimpleDateFormat("yyyyMMddHHmmss").format(day2);
- String dir3 = new SimpleDateFormat("yyyyMMddHHmmss").format(day3);
+ String dir1 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
+ String dir2 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
+ String dir3 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(3, TimeUnit.DAYS)));
Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
Path path1 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + dir1);
Path path2 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + dir2);
-
Path path3 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + dir3);
- fs.mkdirs(path1);
- fs.mkdirs(path2);
- fs.mkdirs(path3);
+ createDirs(fs, path1, path2, path3);
createFile(path1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
services.init();
ShareLibService shareLibService = Services.get().get(ShareLibService.class);
@@ -534,4 +559,10 @@ public class TestShareLibService extends XFsTestCase {
}
}
+ private void createDirs(FileSystem fs, Path... paths) throws IOException {
+ for (Path path : paths) {
+ fs.mkdirs(path);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a015a45e/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 9673bac..c241774 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1761 Improve sharelib purging logic (puru via rohini)
OOZIE-1725 add coord EL functions to be used in SLA tag (ryota)
OOZIE-1765 JMS Notifications for Workflows not always on the correct topic (rkanter)
OOZIE-1732 Sharelib instrumentation fails if sharelib.system.libpath is not created (ryota)