You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by su...@apache.org on 2015/04/13 20:12:24 UTC
falcon git commit: FALCON-1146 feed retention policy deleted
everything all the way up to the root. Contributed by Peeyush Bishnoi
Repository: falcon
Updated Branches:
refs/heads/0.6.1 7ec516ed4 -> 8cbe93e9c
FALCON-1146 feed retention policy deleted everything all the way up to the root. Contributed by Peeyush Bishnoi
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8cbe93e9
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8cbe93e9
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8cbe93e9
Branch: refs/heads/0.6.1
Commit: 8cbe93e9cf6bbd3ed46ba5bec4d2bc45c696f377
Parents: 7ec516e
Author: Suhas Vasu <su...@inmobi.com>
Authored: Mon Apr 13 23:42:02 2015 +0530
Committer: Suhas Vasu <su...@inmobi.com>
Committed: Mon Apr 13 23:42:02 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../apache/falcon/entity/FileSystemStorage.java | 2 +-
.../falcon/retention/FeedEvictorTest.java | 61 +++++++++++++++-----
3 files changed, 52 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 77a0fef..02b8be2 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -137,6 +137,9 @@ Branch: 0.6.1 (Proposed Release Version: 0.6.1)
(Suhas vasu)
BUG FIXES
+ FALCON-1146 feed retention policy deleted everything all the way
+ up to the root (Peeyush Bishnoi via Suhas Vasu)
+
FALCON-1129 In a secure cluster, feed replication fails because of
Authentication issues (Venkat Ranganathan via Suhas Vasu)
http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index 8948ad4..9ff33e7 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -328,7 +328,7 @@ public class FileSystemStorage extends Configured implements Storage {
DateFormat dateFormat = new SimpleDateFormat(FeedHelper.FORMAT);
dateFormat.setTimeZone(timeZone);
- Path feedBasePath = FeedHelper.getFeedBasePath(feedPath);
+ Path feedBasePath = fs.makeQualified(FeedHelper.getFeedBasePath(feedPath));
for (Path path : toBeDeleted) {
deleteInstance(fs, path, feedBasePath);
Date date = FeedHelper.getDate(feedPath, new Path(path.toUri().getPath()), timeZone);
http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index a2feccf..e3daf8c 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -24,6 +24,7 @@ import org.apache.falcon.entity.Storage;
import org.apache.falcon.entity.v0.feed.LocationType;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
@@ -32,19 +33,10 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
+import java.io.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
+import java.util.*;
import java.util.concurrent.TimeUnit;
/**
@@ -370,7 +362,7 @@ public class FeedEvictorTest {
}
}
- @Test(enabled = false)
+ @Test
public void testEvictionWithEmptyDirs() throws Exception {
try {
Configuration conf = cluster.getConf();
@@ -398,7 +390,7 @@ public class FeedEvictorTest {
compare(map.get("feed1"), stream.getBuffer());
- String expectedInstancePaths = getExpectedInstancePaths(dataPath.replaceAll(storageUrl, ""));
+ String expectedInstancePaths = getExpectedInstancePaths(dataPath);
Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths);
String deletedPath = expectedInstancePaths.split(",")[0].split("=")[1];
@@ -421,6 +413,49 @@ public class FeedEvictorTest {
}
}
+ @Test
+ public void testFeedBasePathExists() throws Exception {
+ try {
+ Configuration conf = cluster.getConf();
+ FileSystem fs = FileSystem.get(conf);
+ fs.delete(new Path("/"), true);
+ stream.clear();
+
+ Pair<List<String>, List<String>> pair = generateInstances(fs, "feed3",
+ "yyyy/MM/dd/hh/mm", 1, TimeUnit.MINUTES, "/data", false);
+ final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
+ String dataPath = LocationType.DATA.name() + "="
+ + storageUrl + "/data/YYYY/feed3/mmHH/dd/MM/?{YEAR}/?{MONTH}/?{DAY}/?{HOUR}/?{MINUTE}";
+ String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
+ long beforeDelCount = fs.getContentSummary(new Path(("/data/YYYY/feed3/mmHH/dd/MM/"))).getDirectoryCount();
+
+ FeedEvictor.main(new String[]{
+ "-feedBasePath", dataPath,
+ "-retentionType", "instance",
+ "-retentionLimit", "minutes(-1)",
+ "-timeZone", "UTC",
+ "-frequency", "minutes",
+ "-logFile", logFile,
+ "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+ });
+
+ String expectedInstancePaths = getExpectedInstancePaths(dataPath);
+ Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths);
+
+ //Feed Base path must exist
+ Assert.assertTrue(fs.exists(new Path("/data/YYYY/feed3/mmHH/dd/MM/")));
+ FileStatus[] files = fs.listStatus(new Path(("/data/YYYY/feed3/mmHH/dd/MM/")));
+ //Number of directories/files inside feed base path should be 0
+ Assert.assertEquals(files.length, 0);
+ long afterDelCount = fs.getContentSummary(new Path(("/data/YYYY/feed3/mmHH/dd/MM/"))).getDirectoryCount();
+ //Number of directories deleted
+ Assert.assertEquals((beforeDelCount - afterDelCount), 11);
+
+ } catch (Exception e) {
+ Assert.fail("Unknown exception", e);
+ }
+ }
+
private Pair<List<String>, List<String>> createTestData(String locationType) throws Exception {
Configuration conf = cluster.getConf();