You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pe...@apache.org on 2016/03/23 16:13:46 UTC
falcon git commit: FALCON-1840 Archive older definition in case of
update
Repository: falcon
Updated Branches:
refs/heads/master c2f7a2a5e -> 820572d8a
FALCON-1840 Archive older definition in case of update
Author: Praveen Adlakha <ad...@gmail.com>
Reviewers: Peeyush Bishnoi<pe...@apache.org>, Balu<bv...@hortonworks.com>, Pallavi<pa...@inmobi.com>, Sandeep<sa...@gmail.com>
Closes #75 from PraveenAdlakha/1840 and squashes the following commits:
c9c19d8 [Praveen Adlakha] documentation added for archival in delete and update
6c570e3 [Praveen Adlakha] comments addressed
b475fc4 [Praveen Adlakha] IT added
7ba81af [Praveen Adlakha] FALCON-1840 Archive older definition in case of update
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/820572d8
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/820572d8
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/820572d8
Branch: refs/heads/master
Commit: 820572d8afb5b1244af7f8c36c810c4efd0b749d
Parents: c2f7a2a
Author: Praveen Adlakha <ad...@gmail.com>
Authored: Wed Mar 23 20:43:12 2016 +0530
Committer: peeyush b <pb...@hortonworks.com>
Committed: Wed Mar 23 20:43:12 2016 +0530
----------------------------------------------------------------------
.../falcon/entity/store/ConfigurationStore.java | 15 ++++-
.../entity/store/FeedLocationStoreTest.java | 58 ++++++++++++++++++--
.../src/site/twiki/falconcli/DeleteEntity.twiki | 2 +-
.../src/site/twiki/falconcli/UpdateEntity.twiki | 3 +-
4 files changed, 68 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java b/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java
index 9c7a932..bdcd1af 100644
--- a/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java
+++ b/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java
@@ -98,10 +98,18 @@ public final class ConfigurationStore implements FalconService {
public static ConfigurationStore get() {
return STORE;
}
-
private FileSystem fs;
private Path storePath;
+ public FileSystem getFs() {
+ return fs;
+ }
+
+ public Path getStorePath() {
+ return storePath;
+ }
+
+
private ConfigurationStore() {
for (EntityType type : EntityType.values()) {
dictionary.put(type, new ConcurrentHashMap<String, Entity>());
@@ -250,6 +258,11 @@ public final class ConfigurationStore implements FalconService {
public synchronized void update(EntityType type, Entity entity) throws FalconException {
if (updatesInProgress.get() == entity) {
+ try {
+ archive(type, entity.getName());
+ } catch (IOException e) {
+ throw new StoreAccessException(e);
+ }
updateInternal(type, entity);
} else {
throw new FalconException(entity.toShortString() + " is not initialized for update");
http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java b/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
index 033a55b..40c077e 100644
--- a/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java
@@ -33,6 +33,8 @@ import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.util.FalconRadixUtils;
import org.apache.falcon.util.FalconTestUtil;
import org.apache.falcon.util.StartupProperties;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
@@ -40,6 +42,7 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.io.File;
+import java.io.IOException;
import java.net.URI;
import java.util.Collection;
@@ -50,7 +53,6 @@ import java.util.Collection;
public class FeedLocationStoreTest extends AbstractTestBase {
private ConfigurationStore store;
-
@BeforeClass
public void initConfigStore() throws Exception {
String configPath = new URI(StartupProperties.get().getProperty("config.store.uri")).getPath();
@@ -65,7 +67,6 @@ public class FeedLocationStoreTest extends AbstractTestBase {
StartupProperties.get().setProperty("configstore.listeners", listeners);
store = ConfigurationStore.get();
store.init();
-
CurrentUser.authenticate(FalconTestUtil.TEST_USER_2);
}
@@ -102,10 +103,41 @@ public class FeedLocationStoreTest extends AbstractTestBase {
}
@Test
+ public void testOnUpdate() throws FalconException{
+ Feed f1 = createFeed("f1");
+ f1.getLocations().getLocations().add(createLocation(LocationType.DATA,
+ "/projects/cas/data/hourly/2014/09/09/09"));
+ store.publish(EntityType.FEED, f1);
+
+ Feed f2 = createFeed("f1");
+ f2.getLocations().getLocations().add(createLocation(LocationType.DATA,
+ "/projects/cas/data/monthly"));
+ store.initiateUpdate(f2);
+ store.update(EntityType.FEED, f2);
+ store.cleanupUpdateInit();
+ boolean isArchived = false;
+ try {
+ Path archivePath = new Path(store.getStorePath(), "archive" + Path.SEPARATOR + "FEED");
+ FileStatus [] files= store.getFs().listStatus(archivePath);
+ for(FileStatus f:files){
+ String name = f.getPath().getName();
+ if (name.startsWith(f2.getName())){
+ isArchived= true;
+ break;
+ }
+ }
+ }catch (IOException e){
+ e.printStackTrace();
+ }
+ Assert.assertTrue(isArchived);
+ }
+
+
+ @Test
public void testOnRemove() throws FalconException{
int initialSize = FeedLocationStore.get().store.getSize();
-
- Feed f1 = createFeed("f1ForRemove");
+ String feedName = "f1ForRemove";
+ Feed f1 = createFeed(feedName);
f1.getLocations().getLocations().add(createLocation(LocationType.DATA,
"/projects/cas/data/hourly/2014/09/09/09"));
f1.getLocations().getLocations().add(createLocation(LocationType.STATS,
@@ -113,9 +145,23 @@ public class FeedLocationStoreTest extends AbstractTestBase {
store.publish(EntityType.FEED, f1);
Assert.assertEquals(FeedLocationStore.get().store.getSize() - initialSize, 4);
- store.remove(EntityType.FEED, "f1ForRemove");
+ store.remove(EntityType.FEED, feedName);
+ boolean isArchived = false;
+ try {
+ Path archivePath = new Path(store.getStorePath(), "archive" + Path.SEPARATOR + "FEED");
+ FileStatus [] files= store.getFs().listStatus(archivePath);
+ for(FileStatus f:files){
+ String name = f.getPath().getName();
+ if (name.startsWith(feedName)){
+ isArchived= true;
+ break;
+ }
+ }
+ }catch (IOException e){
+ e.printStackTrace();
+ }
+ Assert.assertTrue(isArchived);
Assert.assertEquals(FeedLocationStore.get().store.getSize(), initialSize);
-
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/docs/src/site/twiki/falconcli/DeleteEntity.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/falconcli/DeleteEntity.twiki b/docs/src/site/twiki/falconcli/DeleteEntity.twiki
index f2b3080..cc07406 100644
--- a/docs/src/site/twiki/falconcli/DeleteEntity.twiki
+++ b/docs/src/site/twiki/falconcli/DeleteEntity.twiki
@@ -2,7 +2,7 @@
[[CommonCLI][Common CLI Options]]
-Delete removes the submitted entity definition for the specified entity and put it into the archive.
+Delete removes the submitted entity definition for the specified entity and put it into the archive.Archive path is defined in startup.properties in variable "config.store.uri".
Usage:
$FALCON_HOME/bin/falcon entity -type [cluster|datasource|feed|process] -name <<name>> -delete
http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/docs/src/site/twiki/falconcli/UpdateEntity.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/falconcli/UpdateEntity.twiki b/docs/src/site/twiki/falconcli/UpdateEntity.twiki
index ae60559..5d49a76 100644
--- a/docs/src/site/twiki/falconcli/UpdateEntity.twiki
+++ b/docs/src/site/twiki/falconcli/UpdateEntity.twiki
@@ -2,8 +2,7 @@
[[CommonCLI][Common CLI Options]]
-Update operation allows an already submitted/scheduled entity to be updated. Cluster and datasource updates are
-currently not allowed.
+Update operation allows an already submitted/scheduled entity to be updated and put it into the archive.Archive path is defined in startup.properties in variable "config.store.uri". Cluster and datasource updates are currently not allowed.
Usage:
$FALCON_HOME/bin/falcon entity -type [feed|process] -name <<name>> -update -file <<path_to_file>>