You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sr...@apache.org on 2013/04/26 17:50:23 UTC
[07/47] Fixes for Checkstyle
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index fe329b5..66e99a6 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -18,12 +18,12 @@
package org.apache.falcon.retention;
+import org.apache.falcon.Pair;
+import org.apache.falcon.cluster.util.EmbeddedCluster;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
-import org.apache.falcon.Pair;
-import org.apache.falcon.cluster.util.EmbeddedCluster;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -37,380 +37,388 @@ import java.util.concurrent.TimeUnit;
public class FeedEvictorTest {
- private EmbeddedCluster cluster;
- private InMemoryWriter stream = new InMemoryWriter(System.out);
- private Map<String, String> map = new HashMap<String, String>();
-
- @BeforeClass
- public void start() throws Exception{
- cluster = EmbeddedCluster.newCluster("test", false);
- FeedEvictor.stream = stream;
- }
-
- @AfterClass
- public void close() throws Exception {
- cluster.shutdown();
- }
-
- @Test
- public void testBadArgs() throws Exception {
- try {
- FeedEvictor.main(null);
- Assert.fail("Expected an exception to be thrown");
- } catch (Exception ignore) { }
-
- try {
- FeedEvictor.main(new String[] {"1","2"});
- Assert.fail("Expected an exception to be thrown");
- } catch (Exception ignore) { }
- }
-
- @Test
- public void testEviction1() throws Exception {
- try {
- FeedEvictor.main(new String[]{"1", "2", "3", "4", "5","6","7"});
- } catch (Exception e) {
- }
- }
-
- @Test
- public void testEviction2() throws Exception {
- try {
- Configuration conf = cluster.getConf();
- FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path("/"), true);
- stream.clear();
-
- Pair<List<String>, List<String>> pair;
- pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS,"/data");
- String dataPath = "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
-
- FeedEvictor.main(new String[] {
- "-feedBasePath",cluster.getConf().get("fs.default.name")
- + dataPath,
- "-retentionType","instance", "-retentionLimit","days(10)", "-timeZone","UTC", "-frequency","daily",
- "-logFile",logFile});
-
- assertFailures(fs, pair);
- compare(map.get("feed1"), stream.getBuffer());
-
- Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
-
-
- } catch (Exception e) {
- Assert.fail("Unknown exception", e);
- }
- }
-
- private String getExpectedInstancePaths(String dataPath){
- StringBuffer newBuffer = new StringBuffer("instancePaths=");
- DateFormat format = new SimpleDateFormat("yyyyMMddHHmm");
- format.setTimeZone(TimeZone.getTimeZone("UTC"));
- String[] locs = dataPath.split("#");
- String [] instances = stream.getBuffer().split("instances=")[1].split(",");
- if(instances[0].equals("NULL")){
- return "instancePaths=";
- }
-
- for(int i=0;i<locs.length;i++){
- for (int j=0, k=i*instances.length/locs.length;j<instances.length/locs.length;j++) {
- String instancePath = locs[i].replaceAll("\\?\\{YEAR\\}", instances[j+k].substring(0,4));
- instancePath = instancePath.replaceAll("\\?\\{MONTH\\}", instances[j+k].substring(4,6));
- instancePath = instancePath.replaceAll("\\?\\{DAY\\}", instances[j+k].substring(6,8));
- instancePath = instancePath.replaceAll("\\?\\{HOUR\\}", instances[j+k].substring(8,10));
- instancePath = instancePath.replaceAll("\\?\\{MINUTE\\}", instances[j+k].substring(10,12));
- newBuffer.append(instancePath).append(',');
- }
- }
- return newBuffer.toString();
- }
-
- private String readLogFile(Path logFile) throws IOException {
- Configuration conf = cluster.getConf();
- FileSystem fs = FileSystem.get(conf);
- ByteArrayOutputStream writer = new ByteArrayOutputStream();
- InputStream date = fs.open(logFile);
- IOUtils.copyBytes(date, writer, 4096, true);
- return writer.toString();
- }
-
- private void compare(String str1, String str2) {
- String[] instances1 = str1.split("=")[1].split(",");
- String[] instances2 = str2.split("instances=")[1].split(",");
-
- Arrays.sort(instances1);
- Arrays.sort(instances2);
- Assert.assertEquals(instances1, instances2);
- }
-
- private void assertFailures(FileSystem fs, Pair<List<String>, List<String>> pair) throws IOException {
- for (String path : pair.second) {
- if (!fs.exists(new Path(path))) {
- Assert.fail("Expecting " + path + " to be present");
- }
- }
- for (String path : pair.first) {
- if (fs.exists(new Path(path))) {
- Assert.fail("Expecting " + path + " to be deleted");
- }
- }
- }
-
- @Test
- public void testEviction3() throws Exception {
- try {
- Configuration conf = cluster.getConf();
- FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path("/"), true);
- stream.clear();
-
- Pair<List<String>, List<String>> pair;
- pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS,"/data");
- String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/" +
- "?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
- FeedEvictor.main(new String[] {
- "-feedBasePath",cluster.getConf().get("fs.default.name")
- + dataPath,
- "-retentionType","instance", "-retentionLimit","hours(5)", "-timeZone","UTC", "-frequency","hourly",
- "-logFile",logFile});
- assertFailures(fs, pair);
-
- compare(map.get("feed2"), stream.getBuffer());
-
- Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
-
- } catch (Exception e) {
- Assert.fail("Unknown exception", e);
- }
- }
-
-
- @Test
- public void testEviction4() throws Exception {
- try {
- Configuration conf = cluster.getConf();
- FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path("/"), true);
- stream.clear();
-
- Pair<List<String>, List<String>> pair;
- pair = createTestData("/data");
- FeedEvictor.main(new String[] {
- "-feedBasePath",
- cluster.getConf().get("fs.default.name")
- + "/data/YYYY/feed3/dd/MM/"
- + "?{MONTH}/more/?{HOUR}", "-retentionType",
- "instance", "-retentionLimit", "months(5)", "-timeZone",
- "UTC", "-frequency", "hourly", "-logFile",
- "/falcon/staging/feed/2012-01-01-04-00" });
- Assert.assertEquals("instances=NULL", stream.getBuffer());
-
- stream.clear();
- String dataPath="/data/YYYY/feed4/dd/MM/" +
- "02/more/hello";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
- FeedEvictor.main(new String[] { "-feedBasePath",
- cluster.getConf().get("fs.default.name") + dataPath,
- "-retentionType", "instance", "-retentionLimit",
- "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
- "-logFile", logFile });
- Assert.assertEquals("instances=NULL", stream.getBuffer());
-
- Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
-
- assertFailures(fs, pair);
- } catch (Exception e) {
- Assert.fail("Unknown exception", e);
- }
- }
-
- @Test
- public void testEviction5() throws Exception {
- try {
- Configuration conf = cluster.getConf();
- FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path("/"), true);
- stream.clear();
-
- Pair<List<String>, List<String>> pair,statsPair,metaPair,tmpPair;
- pair = createTestData("/data");
- statsPair = createTestData("/stats");
- metaPair = createTestData("/meta");
- tmpPair = createTestData("/tmp");
- FeedEvictor.main(new String[] {
- "-feedBasePath",
- getFeedBasePath(cluster, "/data") + "#"
- + getFeedBasePath(cluster, "/stats") + "#"
- + getFeedBasePath(cluster, "/meta") + "#"
- + getFeedBasePath(cluster, "/tmp"),
- "-retentionType", "instance", "-retentionLimit",
- "months(5)", "-timeZone", "UTC", "-frequency", "hourly",
- "-logFile", "/falcon/staging/feed/2012-01-01-04-00" });
- Assert.assertEquals("instances=NULL", stream.getBuffer());
-
- stream.clear();
- String dataPath="/data/YYYY/feed4/dd/MM/" +
- "02/more/hello";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
- FeedEvictor.main(new String[] { "-feedBasePath",
- cluster.getConf().get("fs.default.name") + dataPath,
- "-retentionType", "instance", "-retentionLimit",
- "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
- "-logFile", logFile });
- Assert.assertEquals("instances=NULL", stream.getBuffer());
-
- Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
-
- assertFailures(fs, pair);
- } catch (Exception e) {
- Assert.fail("Unknown exception", e);
- }
- }
-
- @Test
- public void testEviction6() throws Exception {
- try {
- Configuration conf = cluster.getConf();
- FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path("/"), true);
- stream.clear();
-
- Pair<List<String>, List<String>> pair, statsPair,metaPair;
- pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS,"/data");
- statsPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS,"/stats");
- metaPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS,"/meta");
- String dataPath = cluster.getConf().get("fs.default.name")+"/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
- + "#"
- + cluster.getConf().get("fs.default.name")+"/stats/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
- + "#"
- + cluster.getConf().get("fs.default.name")+"/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
- String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
-
- FeedEvictor.main(new String[] {
- "-feedBasePath",
- dataPath,
- "-retentionType","instance", "-retentionLimit","days(10)", "-timeZone","UTC", "-frequency","daily",
- "-logFile",logFile});
-
- assertFailures(fs, pair);
-
- Assert.assertEquals(readLogFile(new Path(logFile)),
- getExpectedInstancePaths(dataPath.replaceAll(cluster
- .getConf().get("fs.default.name"), "")));
-
-
- } catch (Exception e) {
- Assert.fail("Unknown exception", e);
- }
- }
-
- private Pair<List<String>, List<String>> createTestData(String locationType) throws Exception {
- Configuration conf = cluster.getConf();
- FileSystem fs = FileSystem.get(conf);
-
- List<String> outOfRange = new ArrayList<String>();
- List<String> inRange = new ArrayList<String>();
-
- touch(fs, locationType+"/YYYY/feed3/dd/MM/02/more/hello");
- touch(fs, locationType+"/YYYY/feed4/dd/MM/02/more/hello");
- touch(fs, locationType+"/YYYY/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
- touch(fs, locationType+"/somedir/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
- outOfRange.add(locationType+"/YYYY/feed3/dd/MM/02/more/hello");
- outOfRange.add(locationType+"/YYYY/feed4/dd/MM/02/more/hello");
- outOfRange.add(locationType+"/YYYY/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
- outOfRange.add(locationType+"/somedir/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
-
- return Pair.of(inRange, outOfRange);
- }
-
- private Pair<List<String>, List<String>> createTestData(String feed,
- String mask,
- int period,
- TimeUnit timeUnit, String locationType)
- throws Exception {
-
- Configuration conf = cluster.getConf();
- FileSystem fs = FileSystem.get(conf);
-
- List<String> outOfRange = new ArrayList<String>();
- List<String> inRange = new ArrayList<String>();
-
- Pair<List<String>, List<String>> pair = createTestData(locationType);
- outOfRange.addAll(pair.second);
- inRange.addAll(pair.first);
-
- pair = generateInstances(fs, feed, mask, period, timeUnit,locationType);
- outOfRange.addAll(pair.second);
- inRange.addAll(pair.first);
- return Pair.of(inRange, outOfRange);
- }
-
- private Pair<List<String>, List<String>> generateInstances(
- FileSystem fs, String feed, String formatString,
- int range, TimeUnit timeUnit, String locationType) throws Exception {
-
- List<String> outOfRange = new ArrayList<String>();
- List<String> inRange = new ArrayList<String>();
-
- DateFormat format = new SimpleDateFormat(formatString);
- format.setTimeZone(TimeZone.getTimeZone("UTC"));
- long now = System.currentTimeMillis();
-
- DateFormat displayFormat = new
- SimpleDateFormat(timeUnit == TimeUnit.HOURS ? "yyyyMMddHH" : "yyyyMMdd");
- displayFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-
- StringBuffer buffer = new StringBuffer();
- for (long date = now;
- date > now - timeUnit.toMillis(range + 6);
- date -= timeUnit.toMillis(1)) {
- String path = locationType+"/YYYY/" + feed + "/mmHH/dd/MM/" +
- format.format(date);
- touch(fs, path);
- if (date <= now && date > now - timeUnit.toMillis(range)) {
- outOfRange.add(path);
- } else {
- inRange.add(path);
- buffer.append((displayFormat.format(date) + "0000").
- substring(0, 12)).append(',');
- }
- }
- map.put(feed, "instances=" + buffer.substring(0, buffer.length() -1));
- return Pair.of(inRange, outOfRange);
- }
-
- private void touch(FileSystem fs, String path) throws Exception {
- fs.create(new Path(path)).close();
- }
-
- private String getFeedBasePath(EmbeddedCluster cluster, String locationType){
- return cluster.getConf().get("fs.default.name")
- + "/data/YYYY/feed3/dd/MM/"
- + "?{MONTH}/more/?{HOUR}";
- }
-
- private static class InMemoryWriter extends PrintStream {
-
- private StringBuffer buffer = new StringBuffer();
-
- public InMemoryWriter(OutputStream out) {
- super(out);
- }
-
- @Override
- public void println(String x) {
- buffer.append(x);
- super.println(x);
- }
-
- public String getBuffer() {
- return buffer.toString();
- }
-
- public void clear() {
- buffer.delete(0, buffer.length());
- }
- }
+ private EmbeddedCluster cluster;
+ private InMemoryWriter stream = new InMemoryWriter(System.out);
+ private Map<String, String> map = new HashMap<String, String>();
+
+ @BeforeClass
+ public void start() throws Exception {
+ cluster = EmbeddedCluster.newCluster("test", false);
+ FeedEvictor.stream = stream;
+ }
+
+ @AfterClass
+ public void close() throws Exception {
+ cluster.shutdown();
+ }
+
+ @Test
+ public void testBadArgs() throws Exception {
+ try {
+ FeedEvictor.main(null);
+ Assert.fail("Expected an exception to be thrown");
+ } catch (Exception ignore) {
+ }
+
+ try {
+ FeedEvictor.main(new String[]{"1", "2"});
+ Assert.fail("Expected an exception to be thrown");
+ } catch (Exception ignore) {
+ }
+ }
+
+ @Test
+ public void testEviction1() throws Exception {
+ try {
+ FeedEvictor.main(new String[]{"1", "2", "3", "4", "5", "6", "7"});
+ } catch (Exception e) {
+ }
+ }
+
+ @Test
+ public void testEviction2() throws Exception {
+ try {
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path("/"), true);
+ stream.clear();
+
+ Pair<List<String>, List<String>> pair;
+ pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
+ String dataPath = "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
+ String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
+
+ FeedEvictor.main(new String[]{
+ "-feedBasePath", cluster.getConf().get("fs.default.name")
+ + dataPath,
+ "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency",
+ "daily",
+ "-logFile", logFile});
+
+ assertFailures(fs, pair);
+ compare(map.get("feed1"), stream.getBuffer());
+
+ Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
+
+
+ } catch (Exception e) {
+ Assert.fail("Unknown exception", e);
+ }
+ }
+
+ private String getExpectedInstancePaths(String dataPath) {
+ StringBuffer newBuffer = new StringBuffer("instancePaths=");
+ DateFormat format = new SimpleDateFormat("yyyyMMddHHmm");
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String[] locs = dataPath.split("#");
+ String[] instances = stream.getBuffer().split("instances=")[1].split(",");
+ if (instances[0].equals("NULL")) {
+ return "instancePaths=";
+ }
+
+ for (int i = 0; i < locs.length; i++) {
+ for (int j = 0, k = i * instances.length / locs.length; j < instances.length / locs.length; j++) {
+ String instancePath = locs[i].replaceAll("\\?\\{YEAR\\}", instances[j + k].substring(0, 4));
+ instancePath = instancePath.replaceAll("\\?\\{MONTH\\}", instances[j + k].substring(4, 6));
+ instancePath = instancePath.replaceAll("\\?\\{DAY\\}", instances[j + k].substring(6, 8));
+ instancePath = instancePath.replaceAll("\\?\\{HOUR\\}", instances[j + k].substring(8, 10));
+ instancePath = instancePath.replaceAll("\\?\\{MINUTE\\}", instances[j + k].substring(10, 12));
+ newBuffer.append(instancePath).append(',');
+ }
+ }
+ return newBuffer.toString();
+ }
+
+ private String readLogFile(Path logFile) throws IOException {
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+ ByteArrayOutputStream writer = new ByteArrayOutputStream();
+ InputStream date = fs.open(logFile);
+ IOUtils.copyBytes(date, writer, 4096, true);
+ return writer.toString();
+ }
+
+ private void compare(String str1, String str2) {
+ String[] instances1 = str1.split("=")[1].split(",");
+ String[] instances2 = str2.split("instances=")[1].split(",");
+
+ Arrays.sort(instances1);
+ Arrays.sort(instances2);
+ Assert.assertEquals(instances1, instances2);
+ }
+
+ private void assertFailures(FileSystem fs, Pair<List<String>, List<String>> pair) throws IOException {
+ for (String path : pair.second) {
+ if (!fs.exists(new Path(path))) {
+ Assert.fail("Expecting " + path + " to be present");
+ }
+ }
+ for (String path : pair.first) {
+ if (fs.exists(new Path(path))) {
+ Assert.fail("Expecting " + path + " to be deleted");
+ }
+ }
+ }
+
+ @Test
+ public void testEviction3() throws Exception {
+ try {
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path("/"), true);
+ stream.clear();
+
+ Pair<List<String>, List<String>> pair;
+ pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
+ String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/" +
+ "?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
+ String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+ FeedEvictor.main(new String[]{
+ "-feedBasePath", cluster.getConf().get("fs.default.name")
+ + dataPath,
+ "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency",
+ "hourly",
+ "-logFile", logFile});
+ assertFailures(fs, pair);
+
+ compare(map.get("feed2"), stream.getBuffer());
+
+ Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
+
+ } catch (Exception e) {
+ Assert.fail("Unknown exception", e);
+ }
+ }
+
+
+ @Test
+ public void testEviction4() throws Exception {
+ try {
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path("/"), true);
+ stream.clear();
+
+ Pair<List<String>, List<String>> pair;
+ pair = createTestData("/data");
+ FeedEvictor.main(new String[]{
+ "-feedBasePath",
+ cluster.getConf().get("fs.default.name")
+ + "/data/YYYY/feed3/dd/MM/"
+ + "?{MONTH}/more/?{HOUR}", "-retentionType",
+ "instance", "-retentionLimit", "months(5)", "-timeZone",
+ "UTC", "-frequency", "hourly", "-logFile",
+ "/falcon/staging/feed/2012-01-01-04-00"});
+ Assert.assertEquals("instances=NULL", stream.getBuffer());
+
+ stream.clear();
+ String dataPath = "/data/YYYY/feed4/dd/MM/" +
+ "02/more/hello";
+ String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+ FeedEvictor.main(new String[]{"-feedBasePath",
+ cluster.getConf().get("fs.default.name") + dataPath,
+ "-retentionType", "instance", "-retentionLimit",
+ "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
+ "-logFile", logFile});
+ Assert.assertEquals("instances=NULL", stream.getBuffer());
+
+ Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
+
+ assertFailures(fs, pair);
+ } catch (Exception e) {
+ Assert.fail("Unknown exception", e);
+ }
+ }
+
+ @Test
+ public void testEviction5() throws Exception {
+ try {
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path("/"), true);
+ stream.clear();
+
+ Pair<List<String>, List<String>> pair, statsPair, metaPair, tmpPair;
+ pair = createTestData("/data");
+ statsPair = createTestData("/stats");
+ metaPair = createTestData("/meta");
+ tmpPair = createTestData("/tmp");
+ FeedEvictor.main(new String[]{
+ "-feedBasePath",
+ getFeedBasePath(cluster, "/data") + "#"
+ + getFeedBasePath(cluster, "/stats") + "#"
+ + getFeedBasePath(cluster, "/meta") + "#"
+ + getFeedBasePath(cluster, "/tmp"),
+ "-retentionType", "instance", "-retentionLimit",
+ "months(5)", "-timeZone", "UTC", "-frequency", "hourly",
+ "-logFile", "/falcon/staging/feed/2012-01-01-04-00"});
+ Assert.assertEquals("instances=NULL", stream.getBuffer());
+
+ stream.clear();
+ String dataPath = "/data/YYYY/feed4/dd/MM/" +
+ "02/more/hello";
+ String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
+ FeedEvictor.main(new String[]{"-feedBasePath",
+ cluster.getConf().get("fs.default.name") + dataPath,
+ "-retentionType", "instance", "-retentionLimit",
+ "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
+ "-logFile", logFile});
+ Assert.assertEquals("instances=NULL", stream.getBuffer());
+
+ Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
+
+ assertFailures(fs, pair);
+ } catch (Exception e) {
+ Assert.fail("Unknown exception", e);
+ }
+ }
+
+ @Test
+ public void testEviction6() throws Exception {
+ try {
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path("/"), true);
+ stream.clear();
+
+ Pair<List<String>, List<String>> pair, statsPair, metaPair;
+ pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
+ statsPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/stats");
+ metaPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/meta");
+ String dataPath = cluster.getConf().get("fs.default.name")
+ + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
+ + "#"
+ + cluster.getConf().get("fs.default.name")
+ + "/stats/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
+ + "#"
+ + cluster.getConf().get("fs.default.name")
+ + "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
+ String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
+
+ FeedEvictor.main(new String[]{
+ "-feedBasePath",
+ dataPath,
+ "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency",
+ "daily",
+ "-logFile", logFile});
+
+ assertFailures(fs, pair);
+
+ Assert.assertEquals(readLogFile(new Path(logFile)),
+ getExpectedInstancePaths(dataPath.replaceAll(cluster
+ .getConf().get("fs.default.name"), "")));
+
+
+ } catch (Exception e) {
+ Assert.fail("Unknown exception", e);
+ }
+ }
+
+ private Pair<List<String>, List<String>> createTestData(String locationType) throws Exception {
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+
+ List<String> outOfRange = new ArrayList<String>();
+ List<String> inRange = new ArrayList<String>();
+
+ touch(fs, locationType + "/YYYY/feed3/dd/MM/02/more/hello");
+ touch(fs, locationType + "/YYYY/feed4/dd/MM/02/more/hello");
+ touch(fs, locationType + "/YYYY/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
+ touch(fs, locationType + "/somedir/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
+ outOfRange.add(locationType + "/YYYY/feed3/dd/MM/02/more/hello");
+ outOfRange.add(locationType + "/YYYY/feed4/dd/MM/02/more/hello");
+ outOfRange.add(locationType + "/YYYY/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
+ outOfRange.add(locationType + "/somedir/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
+
+ return Pair.of(inRange, outOfRange);
+ }
+
+ private Pair<List<String>, List<String>> createTestData(String feed,
+ String mask,
+ int period,
+ TimeUnit timeUnit, String locationType)
+ throws Exception {
+
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+
+ List<String> outOfRange = new ArrayList<String>();
+ List<String> inRange = new ArrayList<String>();
+
+ Pair<List<String>, List<String>> pair = createTestData(locationType);
+ outOfRange.addAll(pair.second);
+ inRange.addAll(pair.first);
+
+ pair = generateInstances(fs, feed, mask, period, timeUnit, locationType);
+ outOfRange.addAll(pair.second);
+ inRange.addAll(pair.first);
+ return Pair.of(inRange, outOfRange);
+ }
+
+ private Pair<List<String>, List<String>> generateInstances(
+ FileSystem fs, String feed, String formatString,
+ int range, TimeUnit timeUnit, String locationType) throws Exception {
+
+ List<String> outOfRange = new ArrayList<String>();
+ List<String> inRange = new ArrayList<String>();
+
+ DateFormat format = new SimpleDateFormat(formatString);
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long now = System.currentTimeMillis();
+
+ DateFormat displayFormat = new
+ SimpleDateFormat(timeUnit == TimeUnit.HOURS ? "yyyyMMddHH" : "yyyyMMdd");
+ displayFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ StringBuffer buffer = new StringBuffer();
+ for (long date = now;
+ date > now - timeUnit.toMillis(range + 6);
+ date -= timeUnit.toMillis(1)) {
+ String path = locationType + "/YYYY/" + feed + "/mmHH/dd/MM/" +
+ format.format(date);
+ touch(fs, path);
+ if (date <= now && date > now - timeUnit.toMillis(range)) {
+ outOfRange.add(path);
+ } else {
+ inRange.add(path);
+ buffer.append((displayFormat.format(date) + "0000").
+ substring(0, 12)).append(',');
+ }
+ }
+ map.put(feed, "instances=" + buffer.substring(0, buffer.length() - 1));
+ return Pair.of(inRange, outOfRange);
+ }
+
+ private void touch(FileSystem fs, String path) throws Exception {
+ fs.create(new Path(path)).close();
+ }
+
+ private String getFeedBasePath(EmbeddedCluster cluster, String locationType) {
+ return cluster.getConf().get("fs.default.name")
+ + "/data/YYYY/feed3/dd/MM/"
+ + "?{MONTH}/more/?{HOUR}";
+ }
+
+ private static class InMemoryWriter extends PrintStream {
+
+ private StringBuffer buffer = new StringBuffer();
+
+ public InMemoryWriter(OutputStream out) {
+ super(out);
+ }
+
+ @Override
+ public void println(String x) {
+ buffer.append(x);
+ super.println(x);
+ }
+
+ public String getBuffer() {
+ return buffer.toString();
+ }
+
+ public void clear() {
+ buffer.delete(0, buffer.length());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/test-util/src/main/java/org/apache/falcon/cluster/util/EmbeddedCluster.java
----------------------------------------------------------------------
diff --git a/test-util/src/main/java/org/apache/falcon/cluster/util/EmbeddedCluster.java b/test-util/src/main/java/org/apache/falcon/cluster/util/EmbeddedCluster.java
index 4b81006..296decb 100644
--- a/test-util/src/main/java/org/apache/falcon/cluster/util/EmbeddedCluster.java
+++ b/test-util/src/main/java/org/apache/falcon/cluster/util/EmbeddedCluster.java
@@ -18,10 +18,7 @@
package org.apache.falcon.cluster.util;
-import java.io.File;
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-
+import org.apache.falcon.entity.v0.cluster.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -29,15 +26,13 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.falcon.entity.v0.cluster.Cluster;
-import org.apache.falcon.entity.v0.cluster.Interface;
-import org.apache.falcon.entity.v0.cluster.Interfaces;
-import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.entity.v0.cluster.Location;
-import org.apache.falcon.entity.v0.cluster.Locations;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.log4j.Logger;
+import java.io.File;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+
public class EmbeddedCluster {
private static Logger LOG = Logger.getLogger(EmbeddedCluster.class);
@@ -105,7 +100,7 @@ public class EmbeddedCluster {
cluster.conf.set("mapreduce.jobtracker.staging.root.dir", "/user");
Path path = new Path("/tmp/hadoop-" + user, "mapred");
FileSystem.get(cluster.conf).mkdirs(path);
- FileSystem.get(cluster.conf).setPermission(path, new FsPermission((short)511));
+ FileSystem.get(cluster.conf).setPermission(path, new FsPermission((short) 511));
cluster.mrCluster = new MiniMRCluster(1,
hdfsUrl, 1);
Configuration mrConf = cluster.mrCluster.createJobConf();
@@ -137,14 +132,14 @@ public class EmbeddedCluster {
interfaces.getInterfaces().add(newInterface(Interfacetype.READONLY, fsUrl, "0.1"));
interfaces.getInterfaces().add(newInterface(Interfacetype.WRITE, fsUrl, "0.1"));
interfaces.getInterfaces().add(newInterface(Interfacetype.EXECUTE,
- conf.get("mapred.job.tracker"), "0.1"));
- interfaces
- .getInterfaces()
- .add(newInterface(
- Interfacetype.MESSAGING,
- "vm://localhost",
- "0.1"));
- clusterEntity.setInterfaces(interfaces);
+ conf.get("mapred.job.tracker"), "0.1"));
+ interfaces
+ .getInterfaces()
+ .add(newInterface(
+ Interfacetype.MESSAGING,
+ "vm://localhost",
+ "0.1"));
+ clusterEntity.setInterfaces(interfaces);
Location location = new Location();
location.setName("staging");
@@ -168,7 +163,9 @@ public class EmbeddedCluster {
}
public void shutdown() {
- if (mrCluster != null) mrCluster.shutdown();
+ if (mrCluster != null) {
+ mrCluster.shutdown();
+ }
dfsCluster.shutdown();
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
----------------------------------------------------------------------
diff --git a/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java b/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
index b65c0e3..baf6347 100644
--- a/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
+++ b/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
@@ -18,36 +18,38 @@
package org.apache.falcon.cluster.util;
-import java.io.File;
-
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.Interface;
import org.apache.falcon.entity.v0.cluster.Interfacetype;
import org.apache.log4j.Logger;
+import java.io.File;
-public class StandAloneCluster extends EmbeddedCluster{
+
+public class StandAloneCluster extends EmbeddedCluster {
private static final Logger LOG = Logger.getLogger(StandAloneCluster.class);
-
+
private StandAloneCluster() {
}
-
+
public static StandAloneCluster newCluster(String clusterFile) throws Exception {
LOG.debug("Initialising standalone cluster");
StandAloneCluster cluster = new StandAloneCluster();
cluster.clusterEntity = (Cluster) EntityType.CLUSTER.getUnmarshaller().unmarshal(new File(clusterFile));
-
- for(Interface inter:cluster.getCluster().getInterfaces().getInterfaces())
- if(inter.getType() == Interfacetype.WRITE)
+
+ for (Interface inter : cluster.getCluster().getInterfaces().getInterfaces()) {
+ if (inter.getType() == Interfacetype.WRITE) {
cluster.getConf().set("fs.default.name", inter.getEndpoint());
-
+ }
+ }
+
LOG.info("Cluster Namenode = " + cluster.getConf().get("fs.default.name"));
return cluster;
}
-
+
@Override
public void shutdown() {
-
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/java/org/apache/falcon/Debug.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/Debug.java b/webapp/src/main/java/org/apache/falcon/Debug.java
index dde4f92..73c99ed 100644
--- a/webapp/src/main/java/org/apache/falcon/Debug.java
+++ b/webapp/src/main/java/org/apache/falcon/Debug.java
@@ -21,11 +21,10 @@ package org.apache.falcon;
import org.apache.falcon.client.FalconClient;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.store.ConfigurationStore;
-import org.apache.falcon.entity.v0.Frequency;
-import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
-import org.apache.falcon.entity.v0.feed.Feed;
+import org.apache.falcon.entity.v0.Frequency;
+import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.service.Services;
import org.apache.falcon.util.DeploymentProperties;
@@ -58,7 +57,9 @@ public class Debug {
for (String line : deps) {
String[] fields = line.replace("(", "").replace(")", "").split(" ");
EntityType eType = EntityType.valueOf(fields[0].toUpperCase());
- if (ConfigurationStore.get().get(eType, fields[1]) != null) continue;
+ if (ConfigurationStore.get().get(eType, fields[1]) != null) {
+ continue;
+ }
String xml = client.getDefinition(eType.name().toLowerCase(), fields[1]);
System.out.println(xml);
store(eType, xml);
@@ -70,7 +71,7 @@ public class Debug {
entity = args[2];
Entity obj = EntityUtil.getEntity(type, entity);
- Process newEntity = (Process)obj.clone();
+ Process newEntity = (Process) obj.clone();
newEntity.setFrequency(Frequency.fromString("minutes(5)"));
System.out.println("##############OLD ENTITY " + EntityUtil.md5(obj));
System.out.println("##############NEW ENTITY " + EntityUtil.md5(newEntity));
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/java/org/apache/falcon/Main.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/Main.java b/webapp/src/main/java/org/apache/falcon/Main.java
index 8cbabc6..3726559 100644
--- a/webapp/src/main/java/org/apache/falcon/Main.java
+++ b/webapp/src/main/java/org/apache/falcon/Main.java
@@ -28,11 +28,11 @@ public class Main {
EmbeddedServer server = new EmbeddedServer(15000,
"webapp/target/falcon-webapp-0.2-SNAPSHOT");
server.start();
-
+
BrokerService broker = new BrokerService();
broker.setUseJmx(false);
- broker.setDataDirectory("target/");
- broker.addConnector("vm://localhost");
+ broker.setDataDirectory("target/");
+ broker.addConnector("vm://localhost");
broker.start();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/java/org/apache/falcon/resource/ConfigSyncService.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/ConfigSyncService.java b/webapp/src/main/java/org/apache/falcon/resource/ConfigSyncService.java
index 6b55911..248c2a0 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/ConfigSyncService.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/ConfigSyncService.java
@@ -31,8 +31,8 @@ public class ConfigSyncService extends AbstractEntityManager {
@POST
@Path("submit/{type}")
- @Consumes({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
- @Produces({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
+ @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
+ @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Monitored(event = "submit")
@Override
public APIResult submit(@Context HttpServletRequest request,
@@ -43,7 +43,7 @@ public class ConfigSyncService extends AbstractEntityManager {
@DELETE
@Path("delete/{type}/{entity}")
- @Produces({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
+ @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Monitored(event = "delete")
@Override
public APIResult delete(@Context HttpServletRequest request,
@@ -55,7 +55,7 @@ public class ConfigSyncService extends AbstractEntityManager {
@POST
@Path("update/{type}/{entity}")
- @Produces({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
+ @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Monitored(event = "update")
@Override
public APIResult update(@Context HttpServletRequest request,
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
index 094fe75..9f789a8 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
@@ -32,47 +32,47 @@ public class InstanceManager extends AbstractInstanceManager {
@GET
@Path("running/{type}/{entity}")
@Produces(MediaType.APPLICATION_JSON)
- @Monitored(event="running")
+ @Monitored(event = "running")
@Override
public InstancesResult getRunningInstances(@PathParam("type") String type,
- @PathParam("entity") String entity,
- @QueryParam("colo") String colo) {
+ @PathParam("entity") String entity,
+ @QueryParam("colo") String colo) {
return super.getRunningInstances(type, entity, colo);
}
@GET
@Path("status/{type}/{entity}")
@Produces(MediaType.APPLICATION_JSON)
- @Monitored(event="instance-status")
+ @Monitored(event = "instance-status")
@Override
public InstancesResult getStatus(@Dimension("type") @PathParam("type") String type,
- @Dimension("entity") @PathParam("entity") String entity,
- @Dimension("start-time") @QueryParam("start") String startStr,
- @Dimension("end-time") @QueryParam("end") String endStr,
- @Dimension("colo") @QueryParam("colo") String colo) {
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Dimension("colo") @QueryParam("colo") String colo) {
return super.getStatus(type, entity, startStr, endStr, colo);
}
- @GET
- @Path("logs/{type}/{entity}")
- @Produces(MediaType.APPLICATION_JSON)
- @Monitored(event = "instance-logs")
- @Override
- public InstancesResult getLogs(
- @Dimension("type") @PathParam("type") String type,
- @Dimension("entity") @PathParam("entity") String entity,
- @Dimension("start-time") @QueryParam("start") String startStr,
- @Dimension("end-time") @QueryParam("end") String endStr,
- @Dimension("colo") @QueryParam("colo") String colo,
- @Dimension("run-id") @QueryParam("runid") String runId) {
- return super.getLogs(type, entity, startStr, endStr, colo, runId);
- }
-
+ @GET
+ @Path("logs/{type}/{entity}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Monitored(event = "instance-logs")
+ @Override
+ public InstancesResult getLogs(
+ @Dimension("type") @PathParam("type") String type,
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Dimension("colo") @QueryParam("colo") String colo,
+ @Dimension("run-id") @QueryParam("runid") String runId) {
+ return super.getLogs(type, entity, startStr, endStr, colo, runId);
+ }
+
@POST
@Path("kill/{type}/{entity}")
@Produces(MediaType.APPLICATION_JSON)
- @Monitored(event="kill-instance")
+ @Monitored(event = "kill-instance")
@Override
public InstancesResult killInstance(@Context HttpServletRequest request,
@Dimension("type") @PathParam("type") String type,
@@ -86,7 +86,7 @@ public class InstanceManager extends AbstractInstanceManager {
@POST
@Path("suspend/{type}/{entity}")
@Produces(MediaType.APPLICATION_JSON)
- @Monitored(event="suspend-instance")
+ @Monitored(event = "suspend-instance")
@Override
public InstancesResult suspendInstance(@Context HttpServletRequest request,
@Dimension("type") @PathParam("type") String type,
@@ -100,7 +100,7 @@ public class InstanceManager extends AbstractInstanceManager {
@POST
@Path("resume/{type}/{entity}")
@Produces(MediaType.APPLICATION_JSON)
- @Monitored(event="resume-instance")
+ @Monitored(event = "resume-instance")
@Override
public InstancesResult resumeInstance(@Context HttpServletRequest request,
@Dimension("type") @PathParam("type") String type,
@@ -114,14 +114,14 @@ public class InstanceManager extends AbstractInstanceManager {
@POST
@Path("rerun/{type}/{entity}")
@Produces(MediaType.APPLICATION_JSON)
- @Monitored(event="re-run-instance")
+ @Monitored(event = "re-run-instance")
@Override
public InstancesResult reRunInstance(@Dimension("type") @PathParam("type") String type,
- @Dimension("entity") @PathParam("entity") String entity,
- @Dimension("start-time") @QueryParam("start") String startStr,
- @Dimension("end-time") @QueryParam("end") String endStr,
- @Context HttpServletRequest request,
- @Dimension("colo") @QueryParam("colo") String colo) {
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Context HttpServletRequest request,
+ @Dimension("colo") @QueryParam("colo") String colo) {
return super.reRunInstance(type, entity, startStr, endStr, request, colo);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
index 099fc5a..02e6614 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
@@ -32,12 +32,12 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager {
@GET
@Path("status/{type}/{entity}")
- @Produces({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
+ @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Monitored(event = "status")
@Override
public APIResult getStatus(@Dimension("entityType") @PathParam("type") String type,
- @Dimension("entityName") @PathParam("entity") String entity,
- @Dimension("colo") @QueryParam("colo") final String colo)
+ @Dimension("entityName") @PathParam("entity") String entity,
+ @Dimension("colo") @QueryParam("colo") final String colo)
throws FalconWebException {
return super.getStatus(type, entity, colo);
}
@@ -63,7 +63,7 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager {
@GET
@Path("definition/{type}/{entity}")
- @Produces({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
+ @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Monitored(event = "definition")
@Override
public String getEntityDefinition(@Dimension("type") @PathParam("type") String type,
@@ -73,7 +73,7 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager {
@POST
@Path("schedule/{type}/{entity}")
- @Produces({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
+ @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Monitored(event = "schedule")
@Override
public APIResult schedule(@Context HttpServletRequest request,
@@ -85,7 +85,7 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager {
@POST
@Path("suspend/{type}/{entity}")
- @Produces({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
+ @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Monitored(event = "suspend")
@Override
public APIResult suspend(@Context HttpServletRequest request,
@@ -97,7 +97,7 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager {
@POST
@Path("resume/{type}/{entity}")
- @Produces({ MediaType.TEXT_XML, MediaType.TEXT_PLAIN })
+ @Produces({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@Monitored(event = "resume")
@Override
public APIResult resume(@Context HttpServletRequest request,
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/webapp/src/main/resources/log4j.xml b/webapp/src/main/resources/log4j.xml
index da50fb2..189d096 100644
--- a/webapp/src/main/resources/log4j.xml
+++ b/webapp/src/main/resources/log4j.xml
@@ -20,72 +20,72 @@
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
- <appender name="console" class="org.apache.log4j.ConsoleAppender">
- <param name="Target" value="System.out"/>
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
- </layout>
- </appender>
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+ </layout>
+ </appender>
- <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
- <param name="File" value="/var/log/falcon/application.log"/>
- <param name="Append" value="true"/>
- <param name="Threshold" value="debug"/>
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
- </layout>
- </appender>
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="/var/log/falcon/application.log"/>
+ <param name="Append" value="true"/>
+ <param name="Threshold" value="debug"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+ </layout>
+ </appender>
+
+ <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="/var/log/falcon/audit.log"/>
+ <param name="Append" value="true"/>
+ <param name="Threshold" value="debug"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %x %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="TRANSACTIONLOG" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="/var/log/falcon/tranlog.log"/>
+ <param name="Append" value="true"/>
+ <param name="Threshold" value="debug"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %x %m%n"/>
+ </layout>
+ </appender>
- <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
- <param name="File" value="/var/log/falcon/audit.log"/>
- <param name="Append" value="true"/>
- <param name="Threshold" value="debug"/>
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d %x %m%n"/>
- </layout>
- </appender>
-
- <appender name="TRANSACTIONLOG" class="org.apache.log4j.DailyRollingFileAppender">
- <param name="File" value="/var/log/falcon/tranlog.log"/>
- <param name="Append" value="true"/>
- <param name="Threshold" value="debug"/>
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d %x %m%n"/>
- </layout>
- </appender>
-
<appender name="METRIC" class="org.apache.log4j.DailyRollingFileAppender">
- <param name="File" value="/var/log/falcon/metric.log"/>
- <param name="Append" value="true"/>
- <param name="Threshold" value="debug"/>
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d %m%n"/>
- </layout>
- </appender>
+ <param name="File" value="/var/log/falcon/metric.log"/>
+ <param name="Append" value="true"/>
+ <param name="Threshold" value="debug"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %m%n"/>
+ </layout>
+ </appender>
+
+ <logger name="org.apache.falcon">
+ <level value="debug"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="AUDIT">
+ <level value="info"/>
+ <appender-ref ref="AUDIT"/>
+ </logger>
- <logger name="org.apache.falcon">
- <level value="debug"/>
- <appender-ref ref="FILE" />
- </logger>
+ <logger name="TRANSACTIONLOG">
+ <level value="info"/>
+ <appender-ref ref="TRANSACTIONLOG"/>
+ </logger>
- <logger name="AUDIT">
- <level value="info"/>
- <appender-ref ref="AUDIT" />
- </logger>
-
- <logger name="TRANSACTIONLOG">
- <level value="info"/>
- <appender-ref ref="TRANSACTIONLOG" />
- </logger>
-
<logger name="METRIC">
- <level value="info"/>
- <appender-ref ref="METRIC" />
- </logger>
+ <level value="info"/>
+ <appender-ref ref="METRIC"/>
+ </logger>
- <root>
- <priority value ="info" />
- <appender-ref ref="console" />
- </root>
+ <root>
+ <priority value="info"/>
+ <appender-ref ref="console"/>
+ </root>
</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/webapp/WEB-INF/distributed/web.xml
----------------------------------------------------------------------
diff --git a/webapp/src/main/webapp/WEB-INF/distributed/web.xml b/webapp/src/main/webapp/WEB-INF/distributed/web.xml
index 68d6248..86fc8d2 100644
--- a/webapp/src/main/webapp/WEB-INF/distributed/web.xml
+++ b/webapp/src/main/webapp/WEB-INF/distributed/web.xml
@@ -53,7 +53,9 @@
</init-param>
<init-param>
<param-name>com.sun.jersey.config.property.classnames</param-name>
- <param-value>org.apache.falcon.resource.admin.AdminResource,org.apache.falcon.resource.provider.JAXBContextResolver,org.apache.falcon.resource.SchedulableEntityManager,org.apache.falcon.resource.InstanceManager</param-value>
+ <param-value>
+ org.apache.falcon.resource.admin.AdminResource,org.apache.falcon.resource.provider.JAXBContextResolver,org.apache.falcon.resource.SchedulableEntityManager,org.apache.falcon.resource.InstanceManager
+ </param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>
@@ -67,7 +69,9 @@
</init-param>
<init-param>
<param-name>com.sun.jersey.config.property.classnames</param-name>
- <param-value>org.apache.falcon.resource.provider.JAXBContextResolver,org.apache.falcon.resource.ConfigSyncService</param-value>
+ <param-value>
+ org.apache.falcon.resource.provider.JAXBContextResolver,org.apache.falcon.resource.ConfigSyncService
+ </param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/webapp/WEB-INF/embedded/web.xml
----------------------------------------------------------------------
diff --git a/webapp/src/main/webapp/WEB-INF/embedded/web.xml b/webapp/src/main/webapp/WEB-INF/embedded/web.xml
index e5e634b..bc74d91 100644
--- a/webapp/src/main/webapp/WEB-INF/embedded/web.xml
+++ b/webapp/src/main/webapp/WEB-INF/embedded/web.xml
@@ -25,37 +25,39 @@
<display-name>Apache Falcon Embedded Server</display-name>
<description>Apache Falcon Embedded Server</description>
- <filter>
- <filter-name>auth</filter-name>
- <filter-class>org.apache.falcon.security.BasicAuthFilter</filter-class>
- </filter>
-
- <filter-mapping>
- <filter-name>auth</filter-name>
- <servlet-name>FalconRESTApi</servlet-name>
- </filter-mapping>
-
- <listener>
- <listener-class>org.apache.falcon.listener.ContextStartupListener</listener-class>
- </listener>
-
- <servlet>
- <servlet-name>FalconRESTApi</servlet-name>
- <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
- <init-param>
- <param-name>com.sun.jersey.config.property.resourceConfigClass</param-name>
- <param-value>com.sun.jersey.api.core.PackagesResourceConfig</param-value>
- </init-param>
- <init-param>
- <param-name>com.sun.jersey.config.property.packages</param-name>
- <param-value>org.apache.falcon.resource.admin,org.apache.falcon.resource.provider,org.apache.falcon.resource.proxy</param-value>
- </init-param>
- <load-on-startup>1</load-on-startup>
- </servlet>
-
- <servlet-mapping>
- <servlet-name>FalconRESTApi</servlet-name>
- <url-pattern>/api/*</url-pattern>
- </servlet-mapping>
+ <filter>
+ <filter-name>auth</filter-name>
+ <filter-class>org.apache.falcon.security.BasicAuthFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>auth</filter-name>
+ <servlet-name>FalconRESTApi</servlet-name>
+ </filter-mapping>
+
+ <listener>
+ <listener-class>org.apache.falcon.listener.ContextStartupListener</listener-class>
+ </listener>
+
+ <servlet>
+ <servlet-name>FalconRESTApi</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.resourceConfigClass</param-name>
+ <param-value>com.sun.jersey.api.core.PackagesResourceConfig</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>
+ org.apache.falcon.resource.admin,org.apache.falcon.resource.provider,org.apache.falcon.resource.proxy
+ </param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>FalconRESTApi</servlet-name>
+ <url-pattern>/api/*</url-pattern>
+ </servlet-mapping>
</web-app>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/a4d79f0c/webapp/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/webapp/src/main/webapp/WEB-INF/web.xml b/webapp/src/main/webapp/WEB-INF/web.xml
index b0ffa39..3864789 100644
--- a/webapp/src/main/webapp/WEB-INF/web.xml
+++ b/webapp/src/main/webapp/WEB-INF/web.xml
@@ -25,37 +25,39 @@
<display-name>Apache Falcon Placeholder</display-name>
<description>Apache Falcon Placeholder</description>
- <filter>
- <filter-name>auth</filter-name>
- <filter-class>org.apache.falcon.security.BasicAuthFilter</filter-class>
- </filter>
-
- <filter-mapping>
- <filter-name>auth</filter-name>
- <servlet-name>FalconRESTApi</servlet-name>
- </filter-mapping>
-
- <listener>
- <listener-class>org.apache.falcon.listener.ContextStartupListener</listener-class>
- </listener>
-
- <servlet>
- <servlet-name>FalconRESTApi</servlet-name>
- <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
- <init-param>
- <param-name>com.sun.jersey.config.property.resourceConfigClass</param-name>
- <param-value>com.sun.jersey.api.core.PackagesResourceConfig</param-value>
- </init-param>
- <init-param>
- <param-name>com.sun.jersey.config.property.packages</param-name>
- <param-value>org.apache.falcon.resource.admin,org.apache.falcon.resource.proxy,org.apache.falcon.resource.provider</param-value>
- </init-param>
- <load-on-startup>1</load-on-startup>
- </servlet>
-
- <servlet-mapping>
- <servlet-name>FalconRESTApi</servlet-name>
- <url-pattern>/api/*</url-pattern>
- </servlet-mapping>
+ <filter>
+ <filter-name>auth</filter-name>
+ <filter-class>org.apache.falcon.security.BasicAuthFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>auth</filter-name>
+ <servlet-name>FalconRESTApi</servlet-name>
+ </filter-mapping>
+
+ <listener>
+ <listener-class>org.apache.falcon.listener.ContextStartupListener</listener-class>
+ </listener>
+
+ <servlet>
+ <servlet-name>FalconRESTApi</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.resourceConfigClass</param-name>
+ <param-value>com.sun.jersey.api.core.PackagesResourceConfig</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>
+ org.apache.falcon.resource.admin,org.apache.falcon.resource.proxy,org.apache.falcon.resource.provider
+ </param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+
+ <servlet-mapping>
+ <servlet-name>FalconRESTApi</servlet-name>
+ <url-pattern>/api/*</url-pattern>
+ </servlet-mapping>
</web-app>