You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sr...@apache.org on 2013/04/26 17:51:01 UTC
[45/47] git commit: checkstyle related fixes for retention module.
checkstyle related fixes for retention module.
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/4e39c7a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/4e39c7a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/4e39c7a9
Branch: refs/heads/master
Commit: 4e39c7a960a7ed0dd2d4a934e462ca92be074dd7
Parents: 6a39baf
Author: venkatesh <ve...@hortonworks.com>
Authored: Tue Apr 23 13:27:39 2013 -0700
Committer: venkatesh <ve...@hortonworks.com>
Committed: Tue Apr 23 13:27:39 2013 -0700
----------------------------------------------------------------------
.../org/apache/falcon/retention/FeedEvictor.java | 51 ++++----
.../apache/falcon/retention/FeedEvictorTest.java | 101 ++++++---------
2 files changed, 65 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4e39c7a9/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
----------------------------------------------------------------------
diff --git a/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java b/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
index 14c2c1c..155c847 100644
--- a/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
+++ b/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -44,6 +44,7 @@ import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -53,27 +54,25 @@ import java.util.regex.Pattern;
*/
public class FeedEvictor extends Configured implements Tool {
- private static Logger LOG = Logger.getLogger(FeedEvictor.class);
+ private static final Logger LOG = Logger.getLogger(FeedEvictor.class);
- private static final ExpressionEvaluator EVALUATOR = new
- ExpressionEvaluatorImpl();
- private static final ExpressionHelper resolver = ExpressionHelper.get();
+ private static final ExpressionEvaluator EVALUATOR = new ExpressionEvaluatorImpl();
+ private static final ExpressionHelper RESOLVER = ExpressionHelper.get();
- static PrintStream stream = System.out;
+ public static final AtomicReference<PrintStream> OUT = new AtomicReference<PrintStream>(System.out);
+// static PrintStream stream = System.out;
- private static final String format = "yyyyMMddHHmm";
+ private static final String FORMAT = "yyyyMMddHHmm";
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Path confPath = new Path("file:///" + System.getProperty("oozie.action.conf.xml"));
- LOG.info(confPath + " found ? " +
- confPath.getFileSystem(conf).exists(confPath));
+ LOG.info(confPath + " found ? " + confPath.getFileSystem(conf).exists(confPath));
conf.addResource(confPath);
int ret = ToolRunner.run(conf, new FeedEvictor(), args);
if (ret != 0) {
- throw new Exception("Unable to perform eviction action args: " +
- Arrays.toString(args));
+ throw new Exception("Unable to perform eviction action args: " + Arrays.toString(args));
}
}
@@ -98,12 +97,12 @@ public class FeedEvictor extends Configured implements Tool {
evictor(path, retentionType, retentionLimit, timeZone, frequency, logFile);
}
- logInstancePaths(new Path(logFile), instancePaths.toString());
+ logInstancePaths(new Path(logFile));
int len = buffer.length();
if (len > 0) {
- stream.println("instances=" + buffer.substring(0, len - 1));
+ OUT.get().println("instances=" + buffer.substring(0, len - 1));
} else {
- stream.println("instances=NULL");
+ OUT.get().println("instances=NULL");
}
return 0;
}
@@ -124,7 +123,7 @@ public class FeedEvictor extends Configured implements Tool {
+ retentionType + ", Limit: " + retentionLimit + ", timezone: "
+ timeZone + ", frequency: " + frequency);
- DateFormat dateFormat = new SimpleDateFormat(format);
+ DateFormat dateFormat = new SimpleDateFormat(FORMAT);
dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
for (Path path : toBeDeleted) {
if (deleteInstance(path)) {
@@ -137,27 +136,26 @@ public class FeedEvictor extends Configured implements Tool {
}
- private void logInstancePaths(Path path, String instancePaths) throws IOException {
+ private void logInstancePaths(Path path) throws IOException {
LOG.info("Writing deleted instances to path " + path);
OutputStream out = fs.create(path);
- out.write(instancePaths.getBytes());
+ out.write(instancePaths.toString().getBytes());
out.close();
if (LOG.isDebugEnabled()) {
- debug(path, fs);
+ debug(path);
}
}
private Pair<Date, Date> getDateRange(String period) throws ELException {
Long duration = (Long) EVALUATOR.evaluate("${" + period + "}",
- Long.class, resolver, resolver);
+ Long.class, RESOLVER, RESOLVER);
Date end = new Date();
Date start = new Date(end.getTime() - duration);
return Pair.of(start, end);
}
private List<Path> discoverInstanceToDelete(String inPath, String timeZone,
- String dateMask, Date start)
- throws IOException {
+ String dateMask, Date start) throws IOException {
FileStatus[] files = findFilesForFeed(inPath);
if (files == null || files.length == 0) {
@@ -187,8 +185,7 @@ public class FeedEvictor extends Configured implements Tool {
.replaceAll(VARS.MINUTE.regex(), "mm");
}
- private FileStatus[] findFilesForFeed(String feedBasePath)
- throws IOException {
+ private FileStatus[] findFilesForFeed(String feedBasePath) throws IOException {
Matcher matcher = FeedDataPath.PATTERN.matcher(feedBasePath);
while (matcher.find()) {
@@ -234,7 +231,7 @@ public class FeedEvictor extends Configured implements Tool {
}
try {
- DateFormat dateFormat = new SimpleDateFormat(format.
+ DateFormat dateFormat = new SimpleDateFormat(FORMAT.
substring(0, date.length()));
dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
return dateFormat.parse(date);
@@ -268,7 +265,7 @@ public class FeedEvictor extends Configured implements Tool {
return fs.delete(path, true);
}
- private void debug(Path outPath, FileSystem fs) throws IOException {
+ private void debug(Path outPath) throws IOException {
ByteArrayOutputStream writer = new ByteArrayOutputStream();
InputStream instance = fs.open(outPath);
IOUtils.copyBytes(instance, writer, 4096, true);
@@ -276,8 +273,7 @@ public class FeedEvictor extends Configured implements Tool {
LOG.debug("Written " + writer);
}
- private CommandLine getCommand(String[] args)
- throws org.apache.commons.cli.ParseException {
+ private CommandLine getCommand(String[] args) throws org.apache.commons.cli.ParseException {
Options options = new Options();
Option opt;
opt = new Option("feedBasePath", true,
@@ -304,5 +300,4 @@ public class FeedEvictor extends Configured implements Tool {
options.addOption(opt);
return new GnuParser().parse(options, args);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4e39c7a9/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index 66e99a6..c5a2013 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -35,6 +35,9 @@ import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.TimeUnit;
+/**
+ * Test for FeedEvictor.
+ */
public class FeedEvictorTest {
private EmbeddedCluster cluster;
@@ -44,7 +47,7 @@ public class FeedEvictorTest {
@BeforeClass
public void start() throws Exception {
cluster = EmbeddedCluster.newCluster("test", false);
- FeedEvictor.stream = stream;
+ FeedEvictor.OUT.set(stream);
}
@AfterClass
@@ -58,12 +61,14 @@ public class FeedEvictorTest {
FeedEvictor.main(null);
Assert.fail("Expected an exception to be thrown");
} catch (Exception ignore) {
+ // ignore
}
try {
FeedEvictor.main(new String[]{"1", "2"});
Assert.fail("Expected an exception to be thrown");
} catch (Exception ignore) {
+ // ignore
}
}
@@ -71,7 +76,8 @@ public class FeedEvictorTest {
public void testEviction1() throws Exception {
try {
FeedEvictor.main(new String[]{"1", "2", "3", "4", "5", "6", "7"});
- } catch (Exception e) {
+ } catch (Exception ignore) {
+ // ignore
}
}
@@ -89,18 +95,15 @@ public class FeedEvictorTest {
String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
FeedEvictor.main(new String[]{
- "-feedBasePath", cluster.getConf().get("fs.default.name")
- + dataPath,
- "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency",
- "daily",
- "-logFile", logFile});
+ "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
+ "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency",
+ "daily", "-logFile", logFile, });
assertFailures(fs, pair);
compare(map.get("feed1"), stream.getBuffer());
Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
-
} catch (Exception e) {
Assert.fail("Unknown exception", e);
}
@@ -170,15 +173,12 @@ public class FeedEvictorTest {
Pair<List<String>, List<String>> pair;
pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
- String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/" +
- "?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
+ String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
FeedEvictor.main(new String[]{
- "-feedBasePath", cluster.getConf().get("fs.default.name")
- + dataPath,
- "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency",
- "hourly",
- "-logFile", logFile});
+ "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
+ "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency",
+ "hourly", "-logFile", logFile, });
assertFailures(fs, pair);
compare(map.get("feed2"), stream.getBuffer());
@@ -201,25 +201,19 @@ public class FeedEvictorTest {
Pair<List<String>, List<String>> pair;
pair = createTestData("/data");
- FeedEvictor.main(new String[]{
- "-feedBasePath",
- cluster.getConf().get("fs.default.name")
- + "/data/YYYY/feed3/dd/MM/"
- + "?{MONTH}/more/?{HOUR}", "-retentionType",
- "instance", "-retentionLimit", "months(5)", "-timeZone",
- "UTC", "-frequency", "hourly", "-logFile",
- "/falcon/staging/feed/2012-01-01-04-00"});
+ FeedEvictor.main(new String[] {
+ "-feedBasePath",
+ cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
+ "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
+ "UTC", "-frequency", "hourly", "-logFile", "/falcon/staging/feed/2012-01-01-04-00", });
Assert.assertEquals("instances=NULL", stream.getBuffer());
stream.clear();
- String dataPath = "/data/YYYY/feed4/dd/MM/" +
- "02/more/hello";
+ String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
- FeedEvictor.main(new String[]{"-feedBasePath",
- cluster.getConf().get("fs.default.name") + dataPath,
- "-retentionType", "instance", "-retentionLimit",
- "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
- "-logFile", logFile});
+ FeedEvictor.main(new String[] {"-feedBasePath",
+ cluster.getConf().get("fs.default.name") + dataPath, "-retentionType", "instance",
+ "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency", "hourly", "-logFile", logFile, });
Assert.assertEquals("instances=NULL", stream.getBuffer());
Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
@@ -243,26 +237,23 @@ public class FeedEvictorTest {
statsPair = createTestData("/stats");
metaPair = createTestData("/meta");
tmpPair = createTestData("/tmp");
- FeedEvictor.main(new String[]{
- "-feedBasePath",
- getFeedBasePath(cluster, "/data") + "#"
- + getFeedBasePath(cluster, "/stats") + "#"
- + getFeedBasePath(cluster, "/meta") + "#"
- + getFeedBasePath(cluster, "/tmp"),
- "-retentionType", "instance", "-retentionLimit",
- "months(5)", "-timeZone", "UTC", "-frequency", "hourly",
- "-logFile", "/falcon/staging/feed/2012-01-01-04-00"});
+ FeedEvictor.main(new String[] {"-feedBasePath",
+ getFeedBasePath("/data") + "#"
+ + getFeedBasePath("/stats") + "#"
+ + getFeedBasePath("/meta") + "#"
+ + getFeedBasePath("/tmp"),
+ "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
+ "UTC", "-frequency", "hourly", "-logFile", "/falcon/staging/feed/2012-01-01-04-00", });
Assert.assertEquals("instances=NULL", stream.getBuffer());
stream.clear();
- String dataPath = "/data/YYYY/feed4/dd/MM/" +
- "02/more/hello";
+ String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
FeedEvictor.main(new String[]{"-feedBasePath",
cluster.getConf().get("fs.default.name") + dataPath,
"-retentionType", "instance", "-retentionLimit",
"hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
- "-logFile", logFile});
+ "-logFile", logFile, });
Assert.assertEquals("instances=NULL", stream.getBuffer());
Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
@@ -295,12 +286,8 @@ public class FeedEvictorTest {
+ "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
- FeedEvictor.main(new String[]{
- "-feedBasePath",
- dataPath,
- "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency",
- "daily",
- "-logFile", logFile});
+ FeedEvictor.main(new String[] {"-feedBasePath", dataPath, "-retentionType", "instance",
+ "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency", "daily", "-logFile", logFile, });
assertFailures(fs, pair);
@@ -333,12 +320,9 @@ public class FeedEvictorTest {
return Pair.of(inRange, outOfRange);
}
- private Pair<List<String>, List<String>> createTestData(String feed,
- String mask,
- int period,
- TimeUnit timeUnit, String locationType)
- throws Exception {
-
+ private Pair<List<String>, List<String>> createTestData(String feed, String mask,
+ int period, TimeUnit timeUnit,
+ String locationType) throws Exception {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
@@ -370,12 +354,11 @@ public class FeedEvictorTest {
SimpleDateFormat(timeUnit == TimeUnit.HOURS ? "yyyyMMddHH" : "yyyyMMdd");
displayFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- StringBuffer buffer = new StringBuffer();
+ StringBuilder buffer = new StringBuilder();
for (long date = now;
date > now - timeUnit.toMillis(range + 6);
date -= timeUnit.toMillis(1)) {
- String path = locationType + "/YYYY/" + feed + "/mmHH/dd/MM/" +
- format.format(date);
+ String path = locationType + "/YYYY/" + feed + "/mmHH/dd/MM/" + format.format(date);
touch(fs, path);
if (date <= now && date > now - timeUnit.toMillis(range)) {
outOfRange.add(path);
@@ -393,7 +376,7 @@ public class FeedEvictorTest {
fs.create(new Path(path)).close();
}
- private String getFeedBasePath(EmbeddedCluster cluster, String locationType) {
+ private String getFeedBasePath(String locationType) {
return cluster.getConf().get("fs.default.name")
+ "/data/YYYY/feed3/dd/MM/"
+ "?{MONTH}/more/?{HOUR}";