You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sr...@apache.org on 2013/10/17 17:45:27 UTC

[2/8] git commit: FALCON-94 Retention to handle hive table eviction. Contributed by Venkatesh Seetharam

FALCON-94 Retention to handle hive table eviction. Contributed by Venkatesh Seetharam


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/5a8fa133
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/5a8fa133
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/5a8fa133

Branch: refs/heads/FALCON-85
Commit: 5a8fa133a91d84343102f9609de46b00ad351831
Parents: 6a49e44
Author: srikanth.sundarrajan <sr...@inmobi.com>
Authored: Thu Oct 17 20:09:34 2013 +0530
Committer: srikanth.sundarrajan <sr...@inmobi.com>
Committed: Thu Oct 17 20:09:34 2013 +0530

----------------------------------------------------------------------
 build-tools/src/bin/oozie-patch-pkg.sh          |   2 +-
 .../falcon/catalog/AbstractCatalogService.java  |  40 ++-
 .../apache/falcon/catalog/CatalogPartition.java | 182 +++++++++++
 .../falcon/catalog/HiveCatalogService.java      |  74 ++++-
 .../falcon/catalog/WebHCatCatalogService.java   | 104 ------
 .../apache/falcon/entity/CatalogStorage.java    |  34 +-
 .../apache/falcon/entity/FileSystemStorage.java |  10 +-
 .../java/org/apache/falcon/entity/Storage.java  |  10 +
 .../falcon/entity/CatalogStorageTest.java       |  17 +
 .../falcon/converter/OozieFeedMapper.java       | 151 +++++----
 .../config/workflow/retention-workflow.xml      |  11 +
 .../falcon/converter/OozieFeedMapperTest.java   |  17 +-
 .../src/main/resources/webhcat-site.xml         |  29 --
 .../apache/falcon/retention/FeedEvictor.java    | 176 ++++++++--
 .../falcon/retention/FeedEvictorTest.java       | 196 ++++++-----
 .../falcon/cluster/util/StandAloneCluster.java  |   1 +
 .../falcon/catalog/HiveCatalogServiceIT.java    | 205 ++++++++++--
 .../catalog/TableStorageFeedEvictorIT.java      | 325 +++++++++++++++++++
 .../org/apache/falcon/util/HiveTestUtils.java   | 158 +++++++++
 19 files changed, 1364 insertions(+), 378 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/build-tools/src/bin/oozie-patch-pkg.sh
----------------------------------------------------------------------
diff --git a/build-tools/src/bin/oozie-patch-pkg.sh b/build-tools/src/bin/oozie-patch-pkg.sh
index 4bb2e9c..5750ef4 100755
--- a/build-tools/src/bin/oozie-patch-pkg.sh
+++ b/build-tools/src/bin/oozie-patch-pkg.sh
@@ -18,7 +18,7 @@
 
 set -e
 
-if [ -d `mvn help:effective-settings | grep localRepository | cut -d\> -f2 | cut -d\< -f1`/org/apache/oozie/oozie-core/4.0.0 ]
+if [ -d `mvn help:effective-settings | grep localRepository | cut -d\> -f2 | cut -d\< -f1`/org/apache/oozie/oozie-core/4.0.0-falcon ]
 then
     echo "Oozie already setup. skipping";
     exit 0;

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
index 8d1d1ed..a5d7d6b 100644
--- a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
+++ b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
@@ -20,6 +20,7 @@ package org.apache.falcon.catalog;
 
 import org.apache.falcon.FalconException;
 
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -50,16 +51,43 @@ public abstract class AbstractCatalogService {
         throws FalconException;
 
     /**
-     * Returns a list of table properties. Most important here are:
-     * 1. Table type: external table or a managed table
-     * 2. Location on HDFS
+     * Returns if the table is external or not.
      *
      * @param catalogUrl url for the catalog service
      * @param database database the table belongs to
      * @param tableName tableName to check if it exists
-     * @return Bag of property name and associated value
+     * @return true if external else false
      * @throws FalconException
      */
-    public abstract Map<String, String> listTableProperties(String catalogUrl, String database,
-                                                            String tableName) throws FalconException;
+    public abstract boolean isTableExternal(String catalogUrl, String database,
+                                            String tableName) throws FalconException;
+
+    /**
+     * List partitions by filter.
+     *
+     * @param catalogUrl url for the catalog service
+     * @param database database the table belongs to
+     * @param tableName tableName to check if it exists
+     * @param filter The filter string,
+     *    for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". Filtering can
+     *    be done only on string partition keys.
+     * @return list of partitions
+     * @throws FalconException
+     */
+    public abstract List<CatalogPartition> listPartitionsByFilter(String catalogUrl, String database,
+                                                                  String tableName, String filter)
+        throws FalconException;
+
+    /**
+     * Drops a given partition.
+     *
+     * @param catalogUrl url for the catalog service
+     * @param database database the table belongs to
+     * @param tableName tableName to check if it exists
+     * @param partitions list of partitions as Key=Value pairs
+     * @return if the partition was dropped
+     * @throws FalconException
+     */
+    public abstract boolean dropPartitions(String catalogUrl, String database, String tableName,
+                                           Map<String, String> partitions) throws FalconException;
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java b/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java
new file mode 100644
index 0000000..ec656a3
--- /dev/null
+++ b/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.catalog;
+
+import java.util.List;
+
+/**
+ * The CatalogPartition is a wrapper around org.apache.hcatalog.api.HCatPartition.
+ */
+public class CatalogPartition {
+
+    private String databaseName;
+    private String tableName;
+    private List<String> values;
+    private int createTime;
+    private int lastAccessTime;
+    private List<String> tableColumns;
+    private String inputFormat;
+    private String outputFormat;
+    private String location;
+    private String serdeInfo;
+
+    protected CatalogPartition() {
+    }
+
+    protected void setDatabaseName(String databaseName) {
+        this.databaseName = databaseName;
+    }
+
+    protected void setTableName(String tableName) {
+        this.tableName = tableName;
+    }
+
+    protected void setValues(List<String> values) {
+        this.values = values;
+    }
+
+    protected void setCreateTime(int createTime) {
+        this.createTime = createTime;
+    }
+
+    protected void setLastAccessTime(int lastAccessTime) {
+        this.lastAccessTime = lastAccessTime;
+    }
+
+    protected void setTableColumns(List<String> tableColumns) {
+        this.tableColumns = tableColumns;
+    }
+
+    protected void setInputFormat(String inputFormat) {
+        this.inputFormat = inputFormat;
+    }
+
+    protected void setOutputFormat(String outputFormat) {
+        this.outputFormat = outputFormat;
+    }
+
+    protected void setLocation(String location) {
+        this.location = location;
+    }
+
+    protected void setSerdeInfo(String serdeInfo) {
+        this.serdeInfo = serdeInfo;
+    }
+
+    /**
+     * Gets the database name.
+     *
+     * @return the database name
+     */
+    public String getDatabaseName() {
+        return this.databaseName;
+    }
+
+    /**
+     * Gets the table name.
+     *
+     * @return the table name
+     */
+    public String getTableName() {
+        return this.tableName;
+    }
+
+    /**
+     * Gets the columns of the table.
+     *
+     * @return the columns
+     */
+    public List<String> getColumns() {
+        return this.tableColumns;
+    }
+
+    /**
+     * Gets the input format.
+     *
+     * @return the input format
+     */
+    public String getInputFormat() {
+        return this.inputFormat;
+    }
+
+    /**
+     * Gets the output format.
+     *
+     * @return the output format
+     */
+    public String getOutputFormat() {
+        return this.outputFormat;
+    }
+
+    /**
+     * Gets the location.
+     *
+     * @return the location
+     */
+    public String getLocation() {
+        return this.location;
+    }
+
+    /**
+     * Gets the serde.
+     *
+     * @return the serde
+     */
+    public String getSerDe() {
+        return this.serdeInfo;
+    }
+
+    /**
+     * Gets the last access time.
+     *
+     * @return the last access time
+     */
+    public int getLastAccessTime() {
+        return this.lastAccessTime;
+    }
+
+    /**
+     * Gets the creates the time.
+     *
+     * @return the creates the time
+     */
+    public int getCreateTime() {
+        return this.createTime;
+    }
+
+    /**
+     * Gets the values.
+     *
+     * @return the values
+     */
+    public List<String> getValues() {
+        return this.values;
+    }
+
+    @Override
+    public String toString() {
+        return "HCatPartition ["
+            + (tableName != null ? "tableName=" + tableName + ", " : "tableName=null")
+            + (databaseName != null ? "dbName=" + databaseName + ", " : "dbName=null")
+            + (values != null ? "values=" + values + ", " : "values=null")
+            + "createTime=" + createTime + ", lastAccessTime="
+            + lastAccessTime + ", " + "]";
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
index f9e7bee..e3ddc3b 100644
--- a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
+++ b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
@@ -25,12 +25,15 @@ import org.apache.falcon.entity.v0.cluster.Interfacetype;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hcatalog.api.HCatClient;
 import org.apache.hcatalog.api.HCatDatabase;
+import org.apache.hcatalog.api.HCatPartition;
 import org.apache.hcatalog.api.HCatTable;
 import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
 import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.log4j.Logger;
 
-import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -111,24 +114,75 @@ public class HiveCatalogService extends AbstractCatalogService {
     }
 
     @Override
-    public Map<String, String> listTableProperties(String catalogUrl, String database,
-                                                   String tableName) throws FalconException {
+    public boolean isTableExternal(String catalogUrl, String database, String tableName)
+        throws FalconException {
         LOG.info("Returns a list of table properties for:" + tableName);
 
         try {
             HCatClient client = get(catalogUrl);
             HCatTable table = client.getTable(database, tableName);
+            return !table.getTabletype().equals("MANAGED_TABLE");
+        } catch (HCatException e) {
+            throw new FalconException(e);
+        }
+    }
+
+    @Override
+    public List<CatalogPartition> listPartitionsByFilter(String catalogUrl, String database,
+                                                      String tableName, String filter)
+        throws FalconException {
+        LOG.info("List partitions for : " + tableName + ", partition filter: " + filter);
+
+        try {
+            List<CatalogPartition> catalogPartitionList = new ArrayList<CatalogPartition>();
+
+            HCatClient client = get(catalogUrl);
+            List<HCatPartition> hCatPartitions = client.listPartitionsByFilter(database, tableName, filter);
+            for (HCatPartition hCatPartition : hCatPartitions) {
+                CatalogPartition partition = createCatalogPartition(hCatPartition);
+                catalogPartitionList.add(partition);
+            }
+
+            return catalogPartitionList;
+        } catch (HCatException e) {
+            throw new FalconException(e);
+        }
+    }
 
-            Map<String, String> tableProperties = new HashMap<String, String>();
-            tableProperties.put("database", table.getDbName());
-            tableProperties.put("tableName", table.getTableName());
-            tableProperties.put("tabletype", table.getTabletype());
-            tableProperties.put("location", table.getLocation());
-            // tableProperties.putAll(table.getTblProps());
+    private CatalogPartition createCatalogPartition(HCatPartition hCatPartition) {
+        final CatalogPartition catalogPartition = new CatalogPartition();
+        catalogPartition.setDatabaseName(hCatPartition.getDatabaseName());
+        catalogPartition.setTableName(hCatPartition.getTableName());
+        catalogPartition.setValues(hCatPartition.getValues());
+        catalogPartition.setInputFormat(hCatPartition.getInputFormat());
+        catalogPartition.setOutputFormat(hCatPartition.getOutputFormat());
+        catalogPartition.setLocation(hCatPartition.getLocation());
+        catalogPartition.setSerdeInfo(hCatPartition.getSerDe());
+        catalogPartition.setCreateTime(hCatPartition.getCreateTime());
+        catalogPartition.setLastAccessTime(hCatPartition.getLastAccessTime());
+
+        List<String> tableColumns = new ArrayList<String>();
+        for (HCatFieldSchema hCatFieldSchema : hCatPartition.getColumns()) {
+            tableColumns.add(hCatFieldSchema.getName());
+        }
+        catalogPartition.setTableColumns(tableColumns);
+
+        return catalogPartition;
+    }
 
-            return tableProperties;
+    @Override
+    public boolean dropPartitions(String catalogUrl, String database,
+                                  String tableName, Map<String, String> partitions)
+        throws FalconException {
+        LOG.info("Dropping partitions for : " + tableName + ", partitions: " + partitions);
+
+        try {
+            HCatClient client = get(catalogUrl);
+            client.dropPartitions(database, tableName, partitions, true);
         } catch (HCatException e) {
             throw new FalconException(e);
         }
+
+        return true;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/catalog/WebHCatCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/WebHCatCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/WebHCatCatalogService.java
deleted file mode 100644
index 49f97d1..0000000
--- a/common/src/main/java/org/apache/falcon/catalog/WebHCatCatalogService.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.catalog;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.falcon.FalconException;
-import org.apache.falcon.security.CurrentUser;
-import org.apache.log4j.Logger;
-
-import javax.ws.rs.core.MediaType;
-import java.util.Map;
-
-/**
- * An implementation of CatalogService that uses HTTP RESTful API to
- * talk to the Hive Meta Store (HCatalog) as the backing Catalog registry.
- */
-public class WebHCatCatalogService extends AbstractCatalogService {
-
-    private static final Logger LOG = Logger.getLogger(WebHCatCatalogService.class);
-
-    @Override
-    public boolean isAlive(String catalogBaseUrl) throws FalconException {
-        LOG.info("Checking if the service is alive for: " + catalogBaseUrl);
-
-        Client client = Client.create();
-        WebResource service = client.resource(catalogBaseUrl);
-        ClientResponse response = service.path("status")
-                .accept(MediaType.APPLICATION_JSON)
-                .get(ClientResponse.class);
-
-        if (LOG.isDebugEnabled() && response.getStatus() != 200) {
-            LOG.debug("Output from Server .... \n" + response.getEntity(String.class));
-        }
-
-        return response.getStatus() == 200;
-    }
-
-    @Override
-    public boolean tableExists(String catalogUrl, String database, String tableName)
-        throws FalconException {
-        LOG.info("Checking if the table exists: " + tableName);
-
-        Client client = Client.create();
-        WebResource service = client.resource(catalogUrl);
-
-        ClientResponse response = service.path("ddl/database").path(database)
-                .path("table").path(tableName)
-                .queryParam("user.name", CurrentUser.getUser())
-                .accept(MediaType.APPLICATION_JSON)
-                .get(ClientResponse.class);
-
-        if (LOG.isDebugEnabled() && response.getStatus() != 200) {
-            LOG.debug("Output from Server .... \n" + response.getEntity(String.class));
-        }
-
-        return response.getStatus() == 200;
-    }
-
-    @Override
-    public Map listTableProperties(String catalogUrl, String database,
-                                   String tableName) throws FalconException {
-        LOG.info("Returns a list of table properties for:" + tableName);
-
-        Client client = Client.create();
-        WebResource service = client.resource(catalogUrl);
-
-        ClientResponse response = service.path("ddl/database/").path(database)
-                .path("table").path(tableName)
-                .path("property")
-//                .queryParam("format", "extended")
-                .queryParam("user.name", CurrentUser.getUser())
-                .accept(MediaType.APPLICATION_JSON)
-                .get(ClientResponse.class);
-
-        if (LOG.isDebugEnabled() && response.getStatus() != 200) {
-            LOG.debug("Output from Server .... \n" + response.getEntity(String.class));
-        }
-
-        if (response.getStatus() != 200) {
-            throw new FalconException("There was an error in this request: "
-                    + response.getEntity(String.class));
-        }
-
-        return response.getEntity(Map.class);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
index 6518140..8e49e38 100644
--- a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
@@ -20,6 +20,7 @@ package org.apache.falcon.entity;
 
 import org.apache.falcon.FalconException;
 import org.apache.falcon.catalog.CatalogServiceFactory;
+import org.apache.falcon.entity.common.FeedDataPath;
 import org.apache.falcon.entity.v0.cluster.Cluster;
 import org.apache.falcon.entity.v0.cluster.Interfacetype;
 import org.apache.falcon.entity.v0.feed.CatalogTable;
@@ -30,6 +31,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.regex.Matcher;
 
 /**
  * A catalog registry implementation of a feed storage.
@@ -42,8 +44,6 @@ public class CatalogStorage implements Storage {
     public static final String OUTPUT_PATH_SEPARATOR = "/";
 
     public static final String CATALOG_URL = "${hcatNode}";
-    private static final String DOLLAR_EXPR_START = "_D__START_";
-    private static final String EXPR_CLOSE = "_CLOSE_";
 
     private final String catalogUrl;
     private String database;
@@ -82,8 +82,8 @@ public class CatalogStorage implements Storage {
      */
     private void parseFeedUri(String catalogTableUri) throws URISyntaxException {
 
-        final String processed = catalogTableUri.replaceAll("\\$\\{", DOLLAR_EXPR_START)
-                                                .replaceAll("}", EXPR_CLOSE);
+        final String processed = catalogTableUri.replaceAll(DOLLAR_EXPR_START_REGEX, DOLLAR_EXPR_START_NORMALIZED)
+                                                .replaceAll("}", EXPR_CLOSE_NORMALIZED);
         URI tableUri = new URI(processed);
 
         if (!"catalog".equals(tableUri.getScheme())) {
@@ -116,8 +116,8 @@ public class CatalogStorage implements Storage {
             throw new URISyntaxException(tableUri.toString(), "Partition details are missing");
         }
 
-        final String rawPartition = partRaw.replaceAll(DOLLAR_EXPR_START, "\\$\\{")
-                                           .replaceAll(EXPR_CLOSE, "\\}");
+        final String rawPartition = partRaw.replaceAll(DOLLAR_EXPR_START_NORMALIZED, DOLLAR_EXPR_START_REGEX)
+                                           .replaceAll(EXPR_CLOSE_NORMALIZED, EXPR_CLOSE_REGEX);
         partitions = new HashMap<String, String>();
         String[] parts = rawPartition.split(PARTITION_SEPARATOR);
         for (String part : parts) {
@@ -147,8 +147,8 @@ public class CatalogStorage implements Storage {
             throw new IllegalArgumentException("URI template cannot be null or empty");
         }
 
-        final String processed = uriTemplate.replaceAll("\\$\\{", DOLLAR_EXPR_START)
-                                            .replaceAll("}", EXPR_CLOSE);
+        final String processed = uriTemplate.replaceAll(DOLLAR_EXPR_START_REGEX, DOLLAR_EXPR_START_NORMALIZED)
+                                            .replaceAll("}", EXPR_CLOSE_NORMALIZED);
         URI uri = new URI(processed);
 
         this.catalogUrl = uri.getScheme() + "://" + uri.getAuthority();
@@ -178,7 +178,8 @@ public class CatalogStorage implements Storage {
             throw new URISyntaxException(uriTemplate.toString(), "Partition details are missing");
         }
 
-        String rawPartition = partRaw.replaceAll(DOLLAR_EXPR_START, "\\$\\{").replaceAll(EXPR_CLOSE, "\\}");
+        String rawPartition = partRaw.replaceAll(DOLLAR_EXPR_START_NORMALIZED, DOLLAR_EXPR_START_REGEX)
+                .replaceAll(EXPR_CLOSE_NORMALIZED, EXPR_CLOSE_REGEX);
         partitions = new HashMap<String, String>();
         String[] parts = rawPartition.split(PARTITION_SEPARATOR);
         for (String part : parts) {
@@ -228,6 +229,21 @@ public class CatalogStorage implements Storage {
         return partitions.containsKey(key);
     }
 
+    public String getDatedPartitionKey() {
+        String datedPartitionKey = null;
+
+        for (Map.Entry<String, String> entry : getPartitions().entrySet()) {
+
+            Matcher matcher = FeedDataPath.PATTERN.matcher(entry.getValue());
+            if (matcher.find()) {
+                datedPartitionKey = entry.getKey();
+                break;
+            }
+        }
+
+        return datedPartitionKey;
+    }
+
     @Override
     public TYPE getType() {
         return TYPE.TABLE;

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index 664813b..4e2dae9 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -39,8 +39,6 @@ public class FileSystemStorage implements Storage {
     public static final String LOCATION_TYPE_SEP = "=";
 
     public static final String FILE_SYSTEM_URL = "${nameNode}";
-    private static final String DOLLAR_EXPR_START = "_D__START_";
-    private static final String EXPR_CLOSE = "_CLOSE_";
 
     private final String storageUrl;
     private final List<Location> locations;
@@ -87,16 +85,16 @@ public class FileSystemStorage implements Storage {
         String[] feedLocs = uriTemplate.split(FEED_PATH_SEP);
         for (String rawPath : feedLocs) {
             String[] typeAndPath = rawPath.split(LOCATION_TYPE_SEP);
-            final String processed = typeAndPath[1].replaceAll("\\$\\{", DOLLAR_EXPR_START)
-                                                   .replaceAll("}", EXPR_CLOSE);
+            final String processed = typeAndPath[1].replaceAll(DOLLAR_EXPR_START_REGEX, DOLLAR_EXPR_START_NORMALIZED)
+                                                   .replaceAll("}", EXPR_CLOSE_NORMALIZED);
             URI uri = new URI(processed);
             if (rawStorageUrl == null) {
                 rawStorageUrl = uri.getScheme() + "://" + uri.getAuthority();
             }
 
             String path = uri.getPath();
-            final String finalPath = path.replaceAll(DOLLAR_EXPR_START, "\\$\\{")
-                                         .replaceAll(EXPR_CLOSE, "\\}");
+            final String finalPath = path.replaceAll(DOLLAR_EXPR_START_NORMALIZED, DOLLAR_EXPR_START_REGEX)
+                                         .replaceAll(EXPR_CLOSE_NORMALIZED, EXPR_CLOSE_REGEX);
 
             Location location = new Location();
             location.setPath(finalPath);

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/entity/Storage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/Storage.java b/common/src/main/java/org/apache/falcon/entity/Storage.java
index 18dad65..0634969 100644
--- a/common/src/main/java/org/apache/falcon/entity/Storage.java
+++ b/common/src/main/java/org/apache/falcon/entity/Storage.java
@@ -27,6 +27,16 @@ import org.apache.falcon.entity.v0.feed.LocationType;
  */
 public interface Storage {
 
+    String DOLLAR_EXPR_START_REGEX = "\\$\\{";
+    String QUESTION_EXPR_START_REGEX = "\\?\\{";
+    String EXPR_CLOSE_REGEX = "\\}";
+
+    /**
+     * URI Friendly expression.
+     */
+    String DOLLAR_EXPR_START_NORMALIZED = "_D__START_";
+    String EXPR_CLOSE_NORMALIZED = "_CLOSE_";
+
     /**
      * Enumeration for the various storage types.
      */

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
index 90eaebc..5c1878b 100644
--- a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
@@ -51,6 +51,23 @@ public class CatalogStorageTest {
         Assert.assertTrue(storage.hasPartition("region"));
         Assert.assertNull(storage.getPartitionValue("unknown"));
         Assert.assertFalse(storage.hasPartition("unknown"));
+        Assert.assertEquals(storage.getDatedPartitionKey(), "ds");
+    }
+
+    @Test
+    public void testParseFeedUriValid2() throws URISyntaxException {
+        String table = "catalog:clicksdb:clicks#ds=${YEAR}${MONTH}${DAY};region=us";
+        CatalogStorage storage = new CatalogStorage(CatalogStorage.CATALOG_URL, table);
+        Assert.assertEquals("${hcatNode}", storage.getCatalogUrl());
+        Assert.assertEquals("clicksdb", storage.getDatabase());
+        Assert.assertEquals("clicks", storage.getTable());
+        Assert.assertEquals(Storage.TYPE.TABLE, storage.getType());
+        Assert.assertEquals(2, storage.getPartitions().size());
+        Assert.assertEquals("us", storage.getPartitionValue("region"));
+        Assert.assertTrue(storage.hasPartition("region"));
+        Assert.assertNull(storage.getPartitionValue("unknown"));
+        Assert.assertFalse(storage.hasPartition("unknown"));
+        Assert.assertEquals(storage.getDatedPartitionKey(), "ds");
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java
----------------------------------------------------------------------
diff --git a/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java b/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java
index bb0958c..bf5367c 100644
--- a/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java
+++ b/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java
@@ -58,14 +58,14 @@ public class OozieFeedMapper extends AbstractOozieEntityMapper<Feed> {
 
     private static final int THIRTY_MINUTES = 30 * 60 * 1000;
 
-    private static final String RETENTION_WF_TEMPLATE = "/config/workflow/retention-workflow.xml";
     private static final String REPLICATION_COORD_TEMPLATE = "/config/coordinator/replication-coordinator.xml";
     private static final String REPLICATION_WF_TEMPLATE = "/config/workflow/replication-workflow.xml";
 
-    public static final String FEED_PATH_SEP = "#";
     private static final String TIMEOUT = "timeout";
     private static final String PARALLEL = "parallel";
 
+    private final RetentionOozieWorkflowMapper retentionMapper = new RetentionOozieWorkflowMapper();
+
     public OozieFeedMapper(Feed feed) {
         super(feed);
     }
@@ -92,70 +92,8 @@ public class OozieFeedMapper extends AbstractOozieEntityMapper<Feed> {
                     + " is not in the future");
             return null;
         }
-        COORDINATORAPP retentionApp = new COORDINATORAPP();
-        String coordName = EntityUtil.getWorkflowName(Tag.RETENTION, feed).toString();
-        retentionApp.setName(coordName);
-        retentionApp.setEnd(SchemaHelper.formatDateUTC(feedCluster.getValidity().getEnd()));
-        retentionApp.setStart(SchemaHelper.formatDateUTC(new Date()));
-        retentionApp.setTimezone(feed.getTimezone().getID());
-        TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
-        if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
-            retentionApp.setFrequency("${coord:hours(6)}");
-        } else {
-            retentionApp.setFrequency("${coord:days(1)}");
-        }
-
-        Path wfPath = getCoordPath(bundlePath, coordName);
-        retentionApp.setAction(getRetentionWorkflowAction(cluster, wfPath, coordName));
-        return retentionApp;
-    }
-
-    private ACTION getRetentionWorkflowAction(Cluster cluster, Path wfPath, String wfName) throws FalconException {
-        Feed feed = getEntity();
-        ACTION retentionAction = new ACTION();
-        WORKFLOW retentionWorkflow = new WORKFLOW();
-        createRetentionWorkflow(cluster, wfPath, wfName);
-        retentionWorkflow.setAppPath(getStoragePath(wfPath.toString()));
-
-        Map<String, String> props = createCoordDefaultConfiguration(cluster, wfPath, wfName);
-
-        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());
-        String feedBasePaths = getFeedDataPath(cluster, feed);
-
-        props.put("feedDataPath", feedBasePaths);
-        props.put("timeZone", feed.getTimezone().getID());
-        props.put("frequency", feed.getFrequency().getTimeUnit().name());
-        props.put("limit", feedCluster.getRetention().getLimit().toString());
-        props.put(ARG.operation.getPropName(), EntityOps.DELETE.name());
-        props.put(ARG.feedNames.getPropName(), feed.getName());
-        props.put(ARG.feedInstancePaths.getPropName(), "IGNORE");
-
-        retentionWorkflow.setConfiguration(getCoordConfig(props));
-        retentionAction.setWorkflow(retentionWorkflow);
-        return retentionAction;
-    }
-
-    protected String getFeedDataPath(Cluster cluster, Feed feed) throws FalconException {
-        final Storage storage = FeedHelper.createStorage(cluster, feed);
-        String feedPathMask = storage.getUriTemplate(LocationType.DATA);
-        String metaPathMask = storage.getUriTemplate(LocationType.META);
-        String statsPathMask = storage.getUriTemplate(LocationType.STATS);
-        String tmpPathMask = storage.getUriTemplate(LocationType.TMP);
-
-        StringBuilder feedBasePaths = new StringBuilder(feedPathMask);
-        if (metaPathMask != null) {
-            feedBasePaths.append(FEED_PATH_SEP).append(metaPathMask);
-        }
-
-        if (statsPathMask != null) {
-            feedBasePaths.append(FEED_PATH_SEP).append(statsPathMask);
-        }
-
-        if (tmpPathMask != null) {
-            feedBasePaths.append(FEED_PATH_SEP).append(tmpPathMask);
-        }
 
-        return feedBasePaths.toString().replaceAll("\\$\\{", "\\?\\{");
+        return retentionMapper.getRetentionCoordinator(cluster, bundlePath, feed, feedCluster);
     }
 
     private List<COORDINATORAPP> getReplicationCoordinators(Cluster targetCluster, Path bundlePath)
@@ -344,17 +282,6 @@ public class OozieFeedMapper extends AbstractOozieEntityMapper<Feed> {
         }
     }
 
-    private void createRetentionWorkflow(Cluster cluster, Path wfPath, String wfName) throws FalconException {
-        try {
-            WORKFLOWAPP retWfApp = getWorkflowTemplate(RETENTION_WF_TEMPLATE);
-            retWfApp.setName(wfName);
-            addLibExtensionsToWorkflow(cluster, retWfApp, EntityType.FEED, "retention");
-            marshal(cluster, retWfApp, wfPath);
-        } catch(IOException e) {
-            throw new FalconException("Unable to create retention workflow", e);
-        }
-    }
-
     @Override
     protected Map<String, String> getEntityProperties() {
         Feed feed = getEntity();
@@ -366,4 +293,76 @@ public class OozieFeedMapper extends AbstractOozieEntityMapper<Feed> {
         }
         return props;
     }
+
+    private final class RetentionOozieWorkflowMapper {
+
+        private static final String RETENTION_WF_TEMPLATE = "/config/workflow/retention-workflow.xml";
+
+        private COORDINATORAPP getRetentionCoordinator(Cluster cluster, Path bundlePath, Feed feed,
+                                                       org.apache.falcon.entity.v0.feed.Cluster feedCluster)
+            throws FalconException {
+
+            COORDINATORAPP retentionApp = new COORDINATORAPP();
+            String coordName = EntityUtil.getWorkflowName(Tag.RETENTION, feed).toString();
+            retentionApp.setName(coordName);
+            retentionApp.setEnd(SchemaHelper.formatDateUTC(feedCluster.getValidity().getEnd()));
+            retentionApp.setStart(SchemaHelper.formatDateUTC(new Date()));
+            retentionApp.setTimezone(feed.getTimezone().getID());
+            TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
+            if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
+                retentionApp.setFrequency("${coord:hours(6)}");
+            } else {
+                retentionApp.setFrequency("${coord:days(1)}");
+            }
+
+            Path wfPath = getCoordPath(bundlePath, coordName);
+            retentionApp.setAction(getRetentionWorkflowAction(cluster, wfPath, coordName));
+            return retentionApp;
+        }
+
+        private ACTION getRetentionWorkflowAction(Cluster cluster, Path wfPath, String wfName)
+            throws FalconException {
+            Feed feed = getEntity();
+            ACTION retentionAction = new ACTION();
+            WORKFLOW retentionWorkflow = new WORKFLOW();
+            createRetentionWorkflow(cluster, wfPath, wfName);
+            retentionWorkflow.setAppPath(getStoragePath(wfPath.toString()));
+
+            Map<String, String> props = createCoordDefaultConfiguration(cluster, wfPath, wfName);
+
+            props.put("timeZone", feed.getTimezone().getID());
+            props.put("frequency", feed.getFrequency().getTimeUnit().name());
+
+            final Storage storage = FeedHelper.createStorage(cluster, feed);
+            props.put("falconFeedStorageType", storage.getType().name());
+
+            String feedDataPath = storage.getUriTemplate();
+            props.put("feedDataPath",
+                    feedDataPath.replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));
+
+            org.apache.falcon.entity.v0.feed.Cluster feedCluster =
+                    FeedHelper.getCluster(feed, cluster.getName());
+            props.put("limit", feedCluster.getRetention().getLimit().toString());
+
+            props.put(ARG.operation.getPropName(), EntityOps.DELETE.name());
+            props.put(ARG.feedNames.getPropName(), feed.getName());
+            props.put(ARG.feedInstancePaths.getPropName(), "IGNORE");
+
+            retentionWorkflow.setConfiguration(getCoordConfig(props));
+            retentionAction.setWorkflow(retentionWorkflow);
+
+            return retentionAction;
+        }
+
+        private void createRetentionWorkflow(Cluster cluster, Path wfPath, String wfName) throws FalconException {
+            try {
+                WORKFLOWAPP retWfApp = getWorkflowTemplate(RETENTION_WF_TEMPLATE);
+                retWfApp.setName(wfName);
+                addLibExtensionsToWorkflow(cluster, retWfApp, EntityType.FEED, "retention");
+                marshal(cluster, retWfApp, wfPath);
+            } catch(IOException e) {
+                throw new FalconException("Unable to create retention workflow", e);
+            }
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/feed/src/main/resources/config/workflow/retention-workflow.xml
----------------------------------------------------------------------
diff --git a/feed/src/main/resources/config/workflow/retention-workflow.xml b/feed/src/main/resources/config/workflow/retention-workflow.xml
index 72e0133..8b444f5 100644
--- a/feed/src/main/resources/config/workflow/retention-workflow.xml
+++ b/feed/src/main/resources/config/workflow/retention-workflow.xml
@@ -30,10 +30,21 @@
                     <name>oozie.launcher.mapred.job.priority</name>
                     <value>${jobPriority}</value>
                 </property>
+                <!-- HCatalog jars -->
+                <property>
+                    <name>oozie.use.system.libpath</name>
+                    <value>true</value>
+                </property>
+                <property>
+                    <name>oozie.action.sharelib.for.java</name>
+                    <value>hcatalog</value>
+                </property>
             </configuration>
             <main-class>org.apache.falcon.retention.FeedEvictor</main-class>
             <arg>-feedBasePath</arg>
             <arg>${feedDataPath}</arg>
+            <arg>-falconFeedStorageType</arg>
+            <arg>${falconFeedStorageType}</arg>
             <arg>-retentionType</arg>
             <arg>instance</arg>
             <arg>-retentionLimit</arg>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
----------------------------------------------------------------------
diff --git a/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java b/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
index cec278e..330cc63 100644
--- a/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
+++ b/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
@@ -17,8 +17,6 @@
  */
 package org.apache.falcon.converter;
 
-import static org.testng.Assert.assertEquals;
-
 import java.util.Calendar;
 import java.util.Collection;
 import java.util.List;
@@ -31,6 +29,7 @@ import org.apache.falcon.FalconException;
 import org.apache.falcon.cluster.util.EmbeddedCluster;
 import org.apache.falcon.entity.ClusterHelper;
 import org.apache.falcon.entity.FeedHelper;
+import org.apache.falcon.entity.Storage;
 import org.apache.falcon.entity.store.ConfigurationStore;
 import org.apache.falcon.entity.v0.Entity;
 import org.apache.falcon.entity.v0.EntityType;
@@ -162,7 +161,7 @@ public class OozieFeedMapperTest {
 
         for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
             if (prop.getName().equals("mapred.job.priority")) {
-                assertEquals(prop.getValue(), "NORMAL");
+                Assert.assertEquals(prop.getValue(), "NORMAL");
                 break;
             }
         }
@@ -211,17 +210,25 @@ public class OozieFeedMapperTest {
         Assert.assertEquals(coord.getFrequency(), "${coord:hours(6)}");
 
         String feedDataPath = null;
+        String storageType = null;
         org.apache.falcon.oozie.coordinator.CONFIGURATION configuration =
                 coord.getAction().getWorkflow().getConfiguration();
         for (Property property : configuration.getProperty()) {
             if ("feedDataPath".equals(property.getName())) {
                 feedDataPath = property.getValue();
-                break;
+            } else if ("falconFeedStorageType".equals(property.getName())) {
+                storageType = property.getValue();
             }
         }
 
+        final Storage storage = FeedHelper.createStorage(cluster, feed);
         if (feedDataPath != null) {
-            Assert.assertEquals(feedDataPath, feedMapper.getFeedDataPath(srcCluster, feed));
+            Assert.assertEquals(feedDataPath, storage.getUriTemplate()
+                    .replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));
+        }
+
+        if (storageType != null) {
+            Assert.assertEquals(storageType, storage.getType().name());
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/hadoop-webapp/src/main/resources/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/webhcat-site.xml b/hadoop-webapp/src/main/resources/webhcat-site.xml
deleted file mode 100644
index 6d0772c..0000000
--- a/hadoop-webapp/src/main/resources/webhcat-site.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<configuration>
-
-    <property>
-        <name>templeton.hadoop.conf.dir</name>
-        <value>${env.HADOOP_CONF_DIR}</value>
-        <description>The path to the Hadoop configuration.</description>
-    </property>
-
-</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
----------------------------------------------------------------------
diff --git a/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java b/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
index a0c7bb7..75ef78c 100644
--- a/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
+++ b/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
@@ -46,9 +47,17 @@ import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.el.ExpressionEvaluatorImpl;
+import org.apache.falcon.FalconException;
 import org.apache.falcon.Pair;
+import org.apache.falcon.catalog.CatalogPartition;
+import org.apache.falcon.catalog.CatalogServiceFactory;
+import org.apache.falcon.entity.CatalogStorage;
+import org.apache.falcon.entity.FeedHelper;
+import org.apache.falcon.entity.FileSystemStorage;
+import org.apache.falcon.entity.Storage;
 import org.apache.falcon.entity.common.FeedDataPath;
 import org.apache.falcon.entity.common.FeedDataPath.VARS;
+import org.apache.falcon.entity.v0.feed.Location;
 import org.apache.falcon.expression.ExpressionHelper;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -72,7 +81,6 @@ public class FeedEvictor extends Configured implements Tool {
     private static final ExpressionHelper RESOLVER = ExpressionHelper.get();
 
     public static final AtomicReference<PrintStream> OUT = new AtomicReference<PrintStream>(System.out);
-//    static PrintStream stream = System.out;
 
     private static final String FORMAT = "yyyyMMddHHmm";
 
@@ -88,7 +96,6 @@ public class FeedEvictor extends Configured implements Tool {
         }
     }
 
-    private FileSystem fs;
     private final Map<VARS, String> map = new TreeMap<VARS, String>();
     private final StringBuffer instancePaths = new StringBuffer("instancePaths=");
     private final StringBuffer buffer = new StringBuffer();
@@ -97,47 +104,73 @@ public class FeedEvictor extends Configured implements Tool {
     public int run(String[] args) throws Exception {
 
         CommandLine cmd = getCommand(args);
-        String feedBasePath = cmd.getOptionValue("feedBasePath").replaceAll("\\?\\{", "\\$\\{");
+        String feedBasePath = cmd.getOptionValue("feedBasePath")
+                .replaceAll(Storage.QUESTION_EXPR_START_REGEX, Storage.DOLLAR_EXPR_START_REGEX);
         String retentionType = cmd.getOptionValue("retentionType");
         String retentionLimit = cmd.getOptionValue("retentionLimit");
         String timeZone = cmd.getOptionValue("timeZone");
         String frequency = cmd.getOptionValue("frequency"); //to write out smart path filters
         String logFile = cmd.getOptionValue("logFile");
+        String feedStorageType = cmd.getOptionValue("falconFeedStorageType");
 
-        String[] feedLocs = feedBasePath.split("#");
-        for (String path : feedLocs) {
-            evictor(path, retentionType, retentionLimit, timeZone, frequency);
-        }
+        LOG.info("Applying retention on " + feedBasePath + " type: " + retentionType
+                + ", Limit: " + retentionLimit + ", timezone: " + timeZone
+                + ", frequency: " + frequency + ", storage" + feedStorageType);
+
+        Storage storage = FeedHelper.createStorage(feedStorageType, feedBasePath);
+        evict(storage, retentionLimit, timeZone);
 
         logInstancePaths(new Path(logFile));
+
         int len = buffer.length();
         if (len > 0) {
             OUT.get().println("instances=" + buffer.substring(0, len - 1));
         } else {
             OUT.get().println("instances=NULL");
         }
+
         return 0;
     }
 
-    private void evictor(String feedBasePath, String retentionType,
-                         String retentionLimit, String timeZone, String frequency) throws IOException, ELException {
+    private void evict(Storage storage, String retentionLimit, String timeZone)
+        throws Exception {
+
+        if (storage.getType() == Storage.TYPE.FILESYSTEM) {
+            evictFS((FileSystemStorage) storage, retentionLimit, timeZone);
+        } else if (storage.getType() == Storage.TYPE.TABLE) {
+            evictTable((CatalogStorage) storage, retentionLimit, timeZone);
+        }
+    }
+
+    private void evictFS(FileSystemStorage storage, String retentionLimit, String timeZone)
+        throws Exception {
+
+        for (Location location : storage.getLocations()) {
+            fileSystemEvictor(storage.getUriTemplate(location.getType()), retentionLimit, timeZone);
+        }
+    }
+
+    private void fileSystemEvictor(String feedBasePath, String retentionLimit, String timeZone)
+        throws IOException, ELException {
+
         Path normalizedPath = new Path(feedBasePath);
-        fs = normalizedPath.getFileSystem(getConf());
+        FileSystem fs = normalizedPath.getFileSystem(getConf());
         feedBasePath = normalizedPath.toUri().getPath();
         LOG.info("Normalized path : " + feedBasePath);
+
         Pair<Date, Date> range = getDateRange(retentionLimit);
         String dateMask = getDateFormatInPath(feedBasePath);
-        List<Path> toBeDeleted = discoverInstanceToDelete(feedBasePath,
-                timeZone, dateMask, range.first);
 
-        LOG.info("Applying retention on " + feedBasePath + " type: "
-                + retentionType + ", Limit: " + retentionLimit + ", timezone: "
-                + timeZone + ", frequency: " + frequency);
+        List<Path> toBeDeleted = discoverInstanceToDelete(feedBasePath, timeZone, dateMask, range.first, fs);
+        if (toBeDeleted.isEmpty()) {
+            LOG.info("No instances to delete.");
+            return;
+        }
 
         DateFormat dateFormat = new SimpleDateFormat(FORMAT);
         dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
         for (Path path : toBeDeleted) {
-            if (deleteInstance(path)) {
+            if (deleteInstance(fs, path)) {
                 LOG.info("Deleted instance " + path);
                 Date date = getDate(path, feedBasePath, dateMask, timeZone);
                 buffer.append(dateFormat.format(date)).append(',');
@@ -165,10 +198,10 @@ public class FeedEvictor extends Configured implements Tool {
         return Pair.of(start, end);
     }
 
-    private List<Path> discoverInstanceToDelete(String inPath, String timeZone,
-                                                String dateMask, Date start) throws IOException {
+    private List<Path> discoverInstanceToDelete(String inPath, String timeZone, String dateMask,
+                                                Date start, FileSystem fs) throws IOException {
 
-        FileStatus[] files = findFilesForFeed(inPath);
+        FileStatus[] files = findFilesForFeed(fs, inPath);
         if (files == null || files.length == 0) {
             return Collections.emptyList();
         }
@@ -196,7 +229,7 @@ public class FeedEvictor extends Configured implements Tool {
                 .replaceAll(VARS.MINUTE.regex(), "mm");
     }
 
-    private FileStatus[] findFilesForFeed(String feedBasePath) throws IOException {
+    private FileStatus[] findFilesForFeed(FileSystem fs, String feedBasePath) throws IOException {
 
         Matcher matcher = FeedDataPath.PATTERN.matcher(feedBasePath);
         while (matcher.find()) {
@@ -272,13 +305,13 @@ public class FeedEvictor extends Configured implements Tool {
         return date.compareTo(start) >= 0;
     }
 
-    private boolean deleteInstance(Path path) throws IOException {
+    private boolean deleteInstance(FileSystem fs, Path path) throws IOException {
         return fs.delete(path, true);
     }
 
-    private void debug(FileSystem myfs, Path outPath) throws IOException {
+    private void debug(FileSystem fs, Path outPath) throws IOException {
         ByteArrayOutputStream writer = new ByteArrayOutputStream();
-        InputStream instance = myfs.open(outPath);
+        InputStream instance = fs.open(outPath);
         IOUtils.copyBytes(instance, writer, 4096, true);
         LOG.debug("Instance Paths copied to " + outPath);
         LOG.debug("Written " + writer);
@@ -286,29 +319,122 @@ public class FeedEvictor extends Configured implements Tool {
 
     private CommandLine getCommand(String[] args) throws org.apache.commons.cli.ParseException {
         Options options = new Options();
-        Option opt;
-        opt = new Option("feedBasePath", true,
+
+        Option opt = new Option("feedBasePath", true,
                 "base path for feed, ex /data/feed/${YEAR}-${MONTH}");
         opt.setRequired(true);
         options.addOption(opt);
+
+        opt = new Option("falconFeedStorageType", true, "feed storage type, FileSystem or Table");
+        opt.setRequired(true);
+        options.addOption(opt);
+
         opt = new Option("retentionType", true,
                 "type of retention policy like delete, archive etc");
         opt.setRequired(true);
         options.addOption(opt);
+
         opt = new Option("retentionLimit", true,
                 "time limit for retention, ex hours(5), months(2), days(90)");
         opt.setRequired(true);
         options.addOption(opt);
+
         opt = new Option("timeZone", true, "timezone for feed, ex UTC");
         opt.setRequired(true);
         options.addOption(opt);
+
         opt = new Option("frequency", true,
                 "frequency of feed,  ex hourly, daily, monthly, minute, weekly, yearly");
         opt.setRequired(true);
         options.addOption(opt);
+
         opt = new Option("logFile", true, "log file for capturing size of feed");
         opt.setRequired(true);
         options.addOption(opt);
+
         return new GnuParser().parse(options, args);
     }
+
+    private void evictTable(CatalogStorage storage, String retentionLimit, String timeZone)
+        throws Exception {
+
+        LOG.info("Applying retention on " + storage.getTable()
+                + ", Limit: " + retentionLimit + ", timezone: " + timeZone);
+
+        String dateMask = getDateFormatInPath(storage.getUriTemplate());
+
+        List<CatalogPartition> toBeDeleted = discoverPartitionsToDelete(
+                storage, retentionLimit, timeZone, dateMask);
+        if (toBeDeleted.isEmpty()) {
+            LOG.info("No partitions to delete.");
+            return;
+        }
+
+        final boolean isTableExternal = CatalogServiceFactory.getCatalogService().isTableExternal(
+                storage.getCatalogUrl(), storage.getDatabase(), storage.getTable());
+
+        dropPartitions(storage, toBeDeleted, isTableExternal);
+    }
+
+    private List<CatalogPartition> discoverPartitionsToDelete(CatalogStorage storage, String retentionLimit,
+                                                           String timeZone, String dateMask)
+        throws FalconException, ELException {
+
+        final String filter = createFilter(storage, retentionLimit, timeZone, dateMask);
+        return CatalogServiceFactory.getCatalogService().listPartitionsByFilter(
+                storage.getCatalogUrl(), storage.getDatabase(), storage.getTable(), filter);
+    }
+
+    private String createFilter(CatalogStorage storage, String retentionLimit,
+                                String timeZone, String dateMask) throws ELException {
+
+        Pair<Date, Date> range = getDateRange(retentionLimit);
+        DateFormat dateFormat = new SimpleDateFormat(FORMAT.substring(0, dateMask.length()));
+        dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
+        String beforeDate = dateFormat.format(range.first);
+
+        String datedPartitionKey = storage.getDatedPartitionKey();
+
+        StringBuilder filterBuffer = new StringBuilder();
+        filterBuffer.append(datedPartitionKey)
+                .append(" < ")
+                .append("\"")
+                .append(beforeDate)
+                .append("\"");
+
+        return filterBuffer.toString();
+    }
+
+    private void dropPartitions(CatalogStorage storage, List<CatalogPartition> partitionsToDelete,
+                                boolean isTableExternal) throws FalconException, IOException {
+
+        for (CatalogPartition partitionToDrop : partitionsToDelete) {
+            if (dropPartition(storage, partitionToDrop, isTableExternal)) {
+                LOG.info("Deleted partition: " + partitionToDrop.getValues());
+                buffer.append(partitionToDrop.getValues().get(0)).append(',');
+                instancePaths.append(partitionToDrop.getValues()).append(",");
+            }
+        }
+    }
+
+    private boolean dropPartition(CatalogStorage storage, CatalogPartition partitionToDrop,
+                                  boolean isTableExternal) throws FalconException, IOException {
+
+        String datedPartitionKey = storage.getDatedPartitionKey();
+
+        Map<String, String> partitions = new HashMap<String, String>();
+        partitions.put(datedPartitionKey, partitionToDrop.getValues().get(0));
+
+        boolean dropped = CatalogServiceFactory.getCatalogService().dropPartitions(
+                storage.getCatalogUrl(), storage.getDatabase(), storage.getTable(), partitions);
+
+        boolean deleted = true;
+        if (isTableExternal) { // nuke the dirs if an external table
+            final String location = partitionToDrop.getLocation();
+            final Path path = new Path(location);
+            deleted = path.getFileSystem(new Configuration()).delete(path, true);
+        }
+
+        return dropped && deleted;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index 5377c48..529418b 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -18,6 +18,19 @@
 
 package org.apache.falcon.retention;
 
+import org.apache.falcon.Pair;
+import org.apache.falcon.cluster.util.EmbeddedCluster;
+import org.apache.falcon.entity.Storage;
+import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -33,17 +46,6 @@ import java.util.Map;
 import java.util.TimeZone;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.falcon.Pair;
-import org.apache.falcon.cluster.util.EmbeddedCluster;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
 /**
  * Test for FeedEvictor.
  */
@@ -100,20 +102,28 @@ public class FeedEvictorTest {
             fs.delete(new Path("/"), true);
             stream.clear();
 
-            Pair<List<String>, List<String>> pair;
-            pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
-            String dataPath = "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
+            Pair<List<String>, List<String>> pair =
+                    createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
+            final String storageUrl = cluster.getConf().get("fs.default.name");
+            String dataPath = LocationType.DATA.name() + "="
+                    + storageUrl + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
             String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
 
             FeedEvictor.main(new String[]{
-                "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
-                "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency",
-                "daily", "-logFile", logFile, });
+                "-feedBasePath", dataPath,
+                "-retentionType", "instance",
+                "-retentionLimit", "days(10)",
+                "-timeZone", "UTC",
+                "-frequency", "daily",
+                "-logFile", logFile,
+                "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+            });
 
             assertFailures(fs, pair);
             compare(map.get("feed1"), stream.getBuffer());
 
-            Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
+            Assert.assertEquals(readLogFile(new Path(logFile)),
+                    getExpectedInstancePaths(dataPath.replaceAll(storageUrl, "")));
 
         } catch (Exception e) {
             Assert.fail("Unknown exception", e);
@@ -121,7 +131,7 @@ public class FeedEvictorTest {
     }
 
     private String getExpectedInstancePaths(String dataPath) {
-        StringBuffer newBuffer = new StringBuffer("instancePaths=");
+        StringBuilder newBuffer = new StringBuilder("instancePaths=");
         DateFormat format = new SimpleDateFormat("yyyyMMddHHmm");
         format.setTimeZone(TimeZone.getTimeZone("UTC"));
         String[] locs = dataPath.split("#");
@@ -132,7 +142,9 @@ public class FeedEvictorTest {
 
         for (int i = 0; i < locs.length; i++) {
             for (int j = 0, k = i * instances.length / locs.length; j < instances.length / locs.length; j++) {
-                String instancePath = locs[i].replaceAll("\\?\\{YEAR\\}", instances[j + k].substring(0, 4));
+                String[] paths = locs[i].split("=");
+                String path = paths[1];
+                String instancePath = path.replaceAll("\\?\\{YEAR\\}", instances[j + k].substring(0, 4));
                 instancePath = instancePath.replaceAll("\\?\\{MONTH\\}", instances[j + k].substring(4, 6));
                 instancePath = instancePath.replaceAll("\\?\\{DAY\\}", instances[j + k].substring(6, 8));
                 instancePath = instancePath.replaceAll("\\?\\{HOUR\\}", instances[j + k].substring(8, 10));
@@ -182,19 +194,27 @@ public class FeedEvictorTest {
             fs.delete(new Path("/"), true);
             stream.clear();
 
-            Pair<List<String>, List<String>> pair;
-            pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
-            String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
+            Pair<List<String>, List<String>> pair =
+                    createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
+            final String storageUrl = cluster.getConf().get("fs.default.name");
+            String dataPath = LocationType.DATA.name() + "="
+                    + storageUrl + "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
             String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
             FeedEvictor.main(new String[]{
-                "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
-                "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency",
-                "hourly", "-logFile", logFile, });
+                "-feedBasePath", dataPath,
+                "-retentionType", "instance",
+                "-retentionLimit", "hours(5)",
+                "-timeZone", "UTC",
+                "-frequency", "hourly",
+                "-logFile", logFile,
+                "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+            });
             assertFailures(fs, pair);
 
             compare(map.get("feed2"), stream.getBuffer());
 
-            Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
+            Assert.assertEquals(readLogFile(new Path(logFile)),
+                    getExpectedInstancePaths(dataPath.replaceAll(storageUrl, "")));
 
         } catch (Exception e) {
             Assert.fail("Unknown exception", e);
@@ -210,21 +230,31 @@ public class FeedEvictorTest {
             fs.delete(new Path("/"), true);
             stream.clear();
 
-            Pair<List<String>, List<String>> pair;
-            pair = createTestData("/data");
+            Pair<List<String>, List<String>> pair = createTestData("/data");
             FeedEvictor.main(new String[] {
-                "-feedBasePath",
-                cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
-                "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
-                "UTC", "-frequency", "hourly", "-logFile", hdfsUrl + "/falcon/staging/feed/2012-01-01-04-00", });
+                "-feedBasePath", LocationType.DATA.name() + "="
+                    + cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
+                "-retentionType", "instance",
+                "-retentionLimit", "months(5)",
+                "-timeZone", "UTC",
+                "-frequency", "hourly",
+                "-logFile", conf.get("fs.default.name") + "/falcon/staging/feed/2012-01-01-04-00",
+                "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+            });
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             stream.clear();
             String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
             String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
-            FeedEvictor.main(new String[] {"-feedBasePath",
-                cluster.getConf().get("fs.default.name") + dataPath, "-retentionType", "instance",
-                "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency", "hourly", "-logFile", logFile, });
+            FeedEvictor.main(new String[] {
+                "-feedBasePath", LocationType.DATA.name() + "=" + cluster.getConf().get("fs.default.name") + dataPath,
+                "-retentionType", "instance",
+                "-retentionLimit", "hours(5)",
+                "-timeZone", "UTC",
+                "-frequency", "hourly",
+                "-logFile", logFile,
+                "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+            });
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
@@ -243,28 +273,38 @@ public class FeedEvictorTest {
             fs.delete(new Path("/"), true);
             stream.clear();
 
-            Pair<List<String>, List<String>> pair, statsPair, metaPair, tmpPair;
-            pair = createTestData("/data");
-            statsPair = createTestData("/stats");
-            metaPair = createTestData("/meta");
-            tmpPair = createTestData("/tmp");
-            FeedEvictor.main(new String[] {"-feedBasePath",
-                getFeedBasePath("/data") + "#"
-                    + getFeedBasePath("/stats") + "#"
-                    + getFeedBasePath("/meta") + "#"
-                    + getFeedBasePath("/tmp"),
-                "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
-                "UTC", "-frequency", "hourly", "-logFile", hdfsUrl + "/falcon/staging/feed/2012-01-01-04-00", });
+            Pair<List<String>, List<String>> pair = createTestData("/data");
+            createTestData("/stats");
+            createTestData("/meta");
+            createTestData("/tmp");
+            final String storageUrl = cluster.getConf().get("fs.default.name");
+            FeedEvictor.main(new String[] {
+                "-feedBasePath", getFeedBasePath(LocationType.DATA, storageUrl)
+                + "#" + getFeedBasePath(LocationType.STATS, storageUrl)
+                    + "#" + getFeedBasePath(LocationType.META, storageUrl)
+                    + "#" + getFeedBasePath(LocationType.TMP, storageUrl),
+                "-retentionType", "instance",
+                "-retentionLimit", "months(5)",
+                "-timeZone", "UTC",
+                "-frequency", "hourly",
+                "-logFile", conf.get("fs.default.name") + "/falcon/staging/feed/2012-01-01-04-00",
+                "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+            });
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             stream.clear();
-            String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
+            String dataPath = LocationType.DATA.name() + "="
+                    + cluster.getConf().get("fs.default.name") + "/data/YYYY/feed4/dd/MM/02/more/hello";
             String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
-            FeedEvictor.main(new String[]{"-feedBasePath",
-                                          cluster.getConf().get("fs.default.name") + dataPath,
-                                          "-retentionType", "instance", "-retentionLimit",
-                                          "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
-                                          "-logFile", logFile, });
+            FeedEvictor.main(new String[]{
+                "-feedBasePath", dataPath,
+                "-retentionType", "instance",
+                "-retentionLimit", "hours(5)",
+                "-timeZone", "UTC",
+                "-frequency", "hourly",
+                "-logFile", logFile,
+                "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+            });
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
@@ -283,29 +323,32 @@ public class FeedEvictorTest {
             fs.delete(new Path("/"), true);
             stream.clear();
 
-            Pair<List<String>, List<String>> pair, statsPair, metaPair;
-            pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
-            statsPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/stats");
-            metaPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/meta");
-            String dataPath = cluster.getConf().get("fs.default.name")
-                    + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
-                    + "#"
-                    + cluster.getConf().get("fs.default.name")
-                    + "/stats/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
-                    + "#"
-                    + cluster.getConf().get("fs.default.name")
-                    + "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
+            Pair<List<String>, List<String>> pair =
+                    createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
+            createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/stats");
+            createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/meta");
+
+            final String storageUrl = cluster.getConf().get("fs.default.name");
+            String dataPath =
+                    "DATA=" + storageUrl + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
+                    + "#STATS=" + storageUrl + "/stats/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
+                    + "#META=" + storageUrl + "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
             String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
 
-            FeedEvictor.main(new String[] {"-feedBasePath", dataPath, "-retentionType", "instance",
-                "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency", "daily", "-logFile", logFile, });
+            FeedEvictor.main(new String[] {
+                "-feedBasePath", dataPath,
+                "-retentionType", "instance",
+                "-retentionLimit", "days(10)",
+                "-timeZone", "UTC",
+                "-frequency", "daily",
+                "-logFile", logFile,
+                "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+            });
 
             assertFailures(fs, pair);
 
             Assert.assertEquals(readLogFile(new Path(logFile)),
-                    getExpectedInstancePaths(dataPath.replaceAll(cluster
-                            .getConf().get("fs.default.name"), "")));
-
+                    getExpectedInstancePaths(dataPath.replaceAll(storageUrl, "")));
 
         } catch (Exception e) {
             Assert.fail("Unknown exception", e);
@@ -375,10 +418,10 @@ public class FeedEvictorTest {
                 outOfRange.add(path);
             } else {
                 inRange.add(path);
-                buffer.append((displayFormat.format(date) + "0000").
-                        substring(0, 12)).append(',');
+                buffer.append((displayFormat.format(date) + "0000").substring(0, 12)).append(',');
             }
         }
+
         map.put(feed, "instances=" + buffer.substring(0, buffer.length() - 1));
         return Pair.of(inRange, outOfRange);
     }
@@ -387,10 +430,9 @@ public class FeedEvictorTest {
         fs.create(new Path(path)).close();
     }
 
-    private String getFeedBasePath(String locationType) {
-        return cluster.getConf().get("fs.default.name")
-                + "/data/YYYY/feed3/dd/MM/"
-                + "?{MONTH}/more/?{HOUR}";
+    private String getFeedBasePath(LocationType locationType, String storageUrl) {
+        return locationType.name() + "=" + storageUrl
+                + "/" + locationType.name().toLowerCase() + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}";
     }
 
     private static class InMemoryWriter extends PrintStream {

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
----------------------------------------------------------------------
diff --git a/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java b/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
index 6864651..ab1d038 100644
--- a/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
+++ b/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
@@ -43,6 +43,7 @@ public final class StandAloneCluster extends EmbeddedCluster {
         for (Interface inter : cluster.getCluster().getInterfaces().getInterfaces()) {
             if (inter.getType() == Interfacetype.WRITE) {
                 cluster.getConf().set("fs.default.name", inter.getEndpoint());
+                break;
             }
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
index 94d14ff..9e42b94 100644
--- a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
+++ b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
@@ -23,10 +23,14 @@ import org.apache.hcatalog.api.HCatAddPartitionDesc;
 import org.apache.hcatalog.api.HCatClient;
 import org.apache.hcatalog.api.HCatCreateDBDesc;
 import org.apache.hcatalog.api.HCatCreateTableDesc;
+import org.apache.hcatalog.api.HCatPartition;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.util.ArrayList;
@@ -40,9 +44,11 @@ import java.util.Map;
  */
 public class HiveCatalogServiceIT {
 
-    private static final String DATABASE_NAME = "falcondb";
-    private static final String TABLE_NAME = "foobar";
     private static final String METASTORE_URL = "thrift://localhost:49083";
+    private static final String DATABASE_NAME = "falcon_db";
+    private static final String TABLE_NAME = "falcon_table";
+    private static final String EXTERNAL_TABLE_NAME = "falcon_external";
+    private static final String EXTERNAL_TABLE_LOCATION = "hdfs://localhost:41020/falcon/staging/falcon_external";
 
     private HiveCatalogService hiveCatalogService;
     private HCatClient client;
@@ -54,7 +60,7 @@ public class HiveCatalogServiceIT {
 
         createDatabase();
         createTable();
-        addPartition();
+        createExternalTable();
     }
 
     private void createDatabase() throws Exception {
@@ -69,8 +75,8 @@ public class HiveCatalogServiceIT {
         cols.add(new HCatFieldSchema("value", HCatFieldSchema.Type.STRING, "value comment"));
 
         List<HCatFieldSchema> partitionSchema = Arrays.asList(
-            new HCatFieldSchema("ds", HCatFieldSchema.Type.STRING, ""),
-            new HCatFieldSchema("region", HCatFieldSchema.Type.STRING, "")
+                new HCatFieldSchema("ds", HCatFieldSchema.Type.STRING, ""),
+                new HCatFieldSchema("region", HCatFieldSchema.Type.STRING, "")
         );
 
         HCatCreateTableDesc tableDesc = HCatCreateTableDesc
@@ -83,41 +89,76 @@ public class HiveCatalogServiceIT {
         client.createTable(tableDesc);
     }
 
-    private void addPartition() throws Exception {
+    public void createExternalTable() throws Exception {
+        ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
+        cols.add(new HCatFieldSchema("id", HCatFieldSchema.Type.INT, "id comment"));
+        cols.add(new HCatFieldSchema("value", HCatFieldSchema.Type.STRING, "value comment"));
+
+        List<HCatFieldSchema> partitionSchema = Arrays.asList(
+                new HCatFieldSchema("ds", HCatFieldSchema.Type.STRING, ""),
+                new HCatFieldSchema("region", HCatFieldSchema.Type.STRING, "")
+        );
+
+        HCatCreateTableDesc tableDesc = HCatCreateTableDesc
+                .create(DATABASE_NAME, EXTERNAL_TABLE_NAME, cols)
+                .fileFormat("rcfile")
+                .ifNotExists(true)
+                .comments("falcon integration test")
+                .partCols(new ArrayList<HCatFieldSchema>(partitionSchema))
+                .isTableExternal(true)
+                .location(EXTERNAL_TABLE_LOCATION)
+                .build();
+        client.createTable(tableDesc);
+    }
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        dropTable(EXTERNAL_TABLE_NAME);
+        dropTable(TABLE_NAME);
+        dropDatabase();
+    }
+
+    private void dropTable(String tableName) throws Exception {
+        client.dropTable(DATABASE_NAME, tableName, true);
+    }
+
+    private void dropDatabase() throws Exception {
+        client.dropDatabase(DATABASE_NAME, true, HCatClient.DropDBMode.CASCADE);
+    }
+
+    @BeforeMethod
+    private void addPartitions() throws Exception {
         Map<String, String> firstPtn = new HashMap<String, String>();
-        firstPtn.put("ds", "09/03/2013");
-        firstPtn.put("region", "usa");
+        firstPtn.put("ds", "20130903"); //yyyyMMDD
+        firstPtn.put("region", "us");
         HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(
                 DATABASE_NAME, TABLE_NAME, null, firstPtn).build();
         client.addPartition(addPtn);
 
         Map<String, String> secondPtn = new HashMap<String, String>();
-        secondPtn.put("ds", "09/03/2013");
-        secondPtn.put("region", "india");
+        secondPtn.put("ds", "20130903");
+        secondPtn.put("region", "in");
         HCatAddPartitionDesc addPtn2 = HCatAddPartitionDesc.create(
                 DATABASE_NAME, TABLE_NAME, null, secondPtn).build();
         client.addPartition(addPtn2);
 
         Map<String, String> thirdPtn = new HashMap<String, String>();
-        thirdPtn.put("ds", "09/02/2013");
-        thirdPtn.put("region", "india");
+        thirdPtn.put("ds", "20130902");
+        thirdPtn.put("region", "in");
         HCatAddPartitionDesc addPtn3 = HCatAddPartitionDesc.create(
                 DATABASE_NAME, TABLE_NAME, null, thirdPtn).build();
         client.addPartition(addPtn3);
     }
 
-    @AfterClass
-    public void tearDown() throws Exception {
-        dropTable();
-        dropDatabase();
-    }
+    @AfterMethod
+    private void dropPartitions() throws Exception {
+        Map<String, String> partitionSpec = new HashMap<String, String>();
+        partitionSpec.put("ds", "20130903");
+        client.dropPartitions(DATABASE_NAME, TABLE_NAME, partitionSpec, true);
 
-    private void dropTable() throws Exception {
-        client.dropTable(DATABASE_NAME, TABLE_NAME, true);
-    }
-
-    private void dropDatabase() throws Exception {
-        client.dropDatabase(DATABASE_NAME, true, HCatClient.DropDBMode.CASCADE);
+        partitionSpec = new HashMap<String, String>();
+        partitionSpec.put("ds", "20130902");
+        client.dropPartitions(DATABASE_NAME, TABLE_NAME, partitionSpec, true);
     }
 
     @Test
@@ -146,12 +187,116 @@ public class HiveCatalogServiceIT {
     }
 
     @Test
-    public void testListTableProperties() throws Exception {
-        Map<String, String> tableProperties =
-                hiveCatalogService.listTableProperties(METASTORE_URL, DATABASE_NAME, TABLE_NAME);
-        Assert.assertEquals(tableProperties.get("database"), DATABASE_NAME);
-        Assert.assertEquals(tableProperties.get("tableName"), TABLE_NAME);
-        Assert.assertEquals(tableProperties.get("tabletype"), "MANAGED_TABLE");
-        Assert.assertTrue(tableProperties.containsKey("location"));
+    public void testIsTableExternalFalse() throws Exception {
+        Assert.assertFalse(hiveCatalogService.isTableExternal(METASTORE_URL, DATABASE_NAME, TABLE_NAME));
+    }
+    @Test
+    public void testIsTableExternalTrue() throws Exception {
+        Assert.assertTrue(hiveCatalogService.isTableExternal(METASTORE_URL, DATABASE_NAME, EXTERNAL_TABLE_NAME));
+    }
+
+    @Test
+    public void testListPartitionsByFilterNull() throws Exception {
+
+        List<CatalogPartition> filteredPartitions = hiveCatalogService.listPartitionsByFilter(
+                METASTORE_URL, DATABASE_NAME, TABLE_NAME, null);
+        Assert.assertEquals(filteredPartitions.size(), 3);
+    }
+
+    @DataProvider (name = "lessThanFilter")
+    public Object[][] createLessThanFilter() {
+        return new Object[][] {
+            {"ds < \"20130905\"", 3},
+            {"ds < \"20130904\"", 3},
+            {"ds < \"20130903\"", 1},
+            {"ds < \"20130902\"", 0},
+        };
+    }
+
+    @Test (dataProvider = "lessThanFilter")
+    public void testListPartitionsByFilterLessThan(String lessThanFilter, int expectedPartitionCount)
+        throws Exception {
+
+        List<CatalogPartition> filteredPartitions = hiveCatalogService.listPartitionsByFilter(
+                METASTORE_URL, DATABASE_NAME, TABLE_NAME, lessThanFilter);
+        Assert.assertEquals(filteredPartitions.size(), expectedPartitionCount);
+    }
+
+    @DataProvider (name = "greaterThanFilter")
+    public Object[][] createGreaterThanFilter() {
+        return new Object[][] {
+            {"ds > \"20130831\"", 3},
+            {"ds > \"20130905\"", 0},
+            {"ds > \"20130904\"", 0},
+            {"ds > \"20130903\"", 0},
+            {"ds > \"20130902\"", 2},
+        };
+    }
+
+    @Test (dataProvider = "greaterThanFilter")
+    public void testListPartitionsByFilterGreaterThan(String greaterThanFilter, int expectedPartitionCount)
+        throws Exception {
+
+        List<CatalogPartition> filteredPartitions = hiveCatalogService.listPartitionsByFilter(
+                METASTORE_URL, DATABASE_NAME, TABLE_NAME, greaterThanFilter);
+        Assert.assertEquals(filteredPartitions.size(), expectedPartitionCount);
+    }
+
+    @Test
+    public void testGetPartitionsFullSpec() throws Exception {
+        Map<String, String> partitionSpec = new HashMap<String, String>();
+        partitionSpec.put("ds", "20130902");
+        partitionSpec.put("region", "in");
+
+        HCatPartition ptn = client.getPartition(DATABASE_NAME, TABLE_NAME, partitionSpec);
+        Assert.assertTrue(ptn != null);
+    }
+
+    @Test
+    public void testGetPartitionsPartialSpec() throws Exception {
+        Map<String, String> partialPartitionSpec = new HashMap<String, String>();
+        partialPartitionSpec.put("ds", "20130903");
+
+        List<HCatPartition> partitions = client.getPartitions(DATABASE_NAME, TABLE_NAME, partialPartitionSpec);
+        Assert.assertEquals(partitions.size(), 2);
+    }
+
+    @Test
+    public void testDropPartition() throws Exception {
+        Map<String, String> partialPartitionSpec = new HashMap<String, String>();
+        partialPartitionSpec.put("ds", "20130903");
+
+        Assert.assertTrue(hiveCatalogService.dropPartitions(
+                METASTORE_URL, DATABASE_NAME, TABLE_NAME, partialPartitionSpec));
+
+        List<HCatPartition> partitions = client.getPartitions(DATABASE_NAME, TABLE_NAME);
+        Assert.assertEquals(1, partitions.size(), "Unexpected number of partitions");
+        Assert.assertEquals(new String[]{"20130902", "in"},
+                partitions.get(0).getValues().toArray(), "Mismatched partition");
+
+        partialPartitionSpec = new HashMap<String, String>();
+        partialPartitionSpec.put("ds", "20130902");
+
+        Assert.assertTrue(hiveCatalogService.dropPartitions(
+                METASTORE_URL, DATABASE_NAME, TABLE_NAME, partialPartitionSpec));
+        partitions = client.getPartitions(DATABASE_NAME, TABLE_NAME);
+        Assert.assertEquals(partitions.size(), 0, "Unexpected number of partitions");
+    }
+
+    /*
+    // this is NOT possible to do in Hive
+    @Test
+    public void testDropPartitionBulk() throws Exception {
+        Map<String, String> partialPartitionSpec = new HashMap<String, String>();
+        partialPartitionSpec.put("ds", "20130903");
+        partialPartitionSpec.put("ds", "20130902");
+        partialPartitionSpec.put("ds", "20130901");
+
+        Assert.assertTrue(hiveCatalogService.dropPartitions(
+                METASTORE_URL, DATABASE_NAME, TABLE_NAME, partialPartitionSpec));
+
+        List<HCatPartition> partitions = client.getPartitions(DATABASE_NAME, TABLE_NAME);
+        Assert.assertEquals(0, partitions.size(), "Unexpected number of partitions");
     }
+    */
 }