You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sr...@apache.org on 2013/10/17 17:45:27 UTC
[2/8] git commit: FALCON-94 Retention to handle hive table eviction.
Contributed by Venkatesh Seetharam
FALCON-94 Retention to handle hive table eviction. Contributed by Venkatesh Seetharam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/5a8fa133
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/5a8fa133
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/5a8fa133
Branch: refs/heads/FALCON-85
Commit: 5a8fa133a91d84343102f9609de46b00ad351831
Parents: 6a49e44
Author: srikanth.sundarrajan <sr...@inmobi.com>
Authored: Thu Oct 17 20:09:34 2013 +0530
Committer: srikanth.sundarrajan <sr...@inmobi.com>
Committed: Thu Oct 17 20:09:34 2013 +0530
----------------------------------------------------------------------
build-tools/src/bin/oozie-patch-pkg.sh | 2 +-
.../falcon/catalog/AbstractCatalogService.java | 40 ++-
.../apache/falcon/catalog/CatalogPartition.java | 182 +++++++++++
.../falcon/catalog/HiveCatalogService.java | 74 ++++-
.../falcon/catalog/WebHCatCatalogService.java | 104 ------
.../apache/falcon/entity/CatalogStorage.java | 34 +-
.../apache/falcon/entity/FileSystemStorage.java | 10 +-
.../java/org/apache/falcon/entity/Storage.java | 10 +
.../falcon/entity/CatalogStorageTest.java | 17 +
.../falcon/converter/OozieFeedMapper.java | 151 +++++----
.../config/workflow/retention-workflow.xml | 11 +
.../falcon/converter/OozieFeedMapperTest.java | 17 +-
.../src/main/resources/webhcat-site.xml | 29 --
.../apache/falcon/retention/FeedEvictor.java | 176 ++++++++--
.../falcon/retention/FeedEvictorTest.java | 196 ++++++-----
.../falcon/cluster/util/StandAloneCluster.java | 1 +
.../falcon/catalog/HiveCatalogServiceIT.java | 205 ++++++++++--
.../catalog/TableStorageFeedEvictorIT.java | 325 +++++++++++++++++++
.../org/apache/falcon/util/HiveTestUtils.java | 158 +++++++++
19 files changed, 1364 insertions(+), 378 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/build-tools/src/bin/oozie-patch-pkg.sh
----------------------------------------------------------------------
diff --git a/build-tools/src/bin/oozie-patch-pkg.sh b/build-tools/src/bin/oozie-patch-pkg.sh
index 4bb2e9c..5750ef4 100755
--- a/build-tools/src/bin/oozie-patch-pkg.sh
+++ b/build-tools/src/bin/oozie-patch-pkg.sh
@@ -18,7 +18,7 @@
set -e
-if [ -d `mvn help:effective-settings | grep localRepository | cut -d\> -f2 | cut -d\< -f1`/org/apache/oozie/oozie-core/4.0.0 ]
+if [ -d `mvn help:effective-settings | grep localRepository | cut -d\> -f2 | cut -d\< -f1`/org/apache/oozie/oozie-core/4.0.0-falcon ]
then
echo "Oozie already setup. skipping";
exit 0;
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
index 8d1d1ed..a5d7d6b 100644
--- a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
+++ b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
@@ -20,6 +20,7 @@ package org.apache.falcon.catalog;
import org.apache.falcon.FalconException;
+import java.util.List;
import java.util.Map;
/**
@@ -50,16 +51,43 @@ public abstract class AbstractCatalogService {
throws FalconException;
/**
- * Returns a list of table properties. Most important here are:
- * 1. Table type: external table or a managed table
- * 2. Location on HDFS
+ * Returns if the table is external or not.
*
* @param catalogUrl url for the catalog service
* @param database database the table belongs to
* @param tableName tableName to check if it exists
- * @return Bag of property name and associated value
+ * @return true if external else false
* @throws FalconException
*/
- public abstract Map<String, String> listTableProperties(String catalogUrl, String database,
- String tableName) throws FalconException;
+ public abstract boolean isTableExternal(String catalogUrl, String database,
+ String tableName) throws FalconException;
+
+ /**
+ * List partitions by filter.
+ *
+ * @param catalogUrl url for the catalog service
+ * @param database database the table belongs to
+ * @param tableName tableName to check if it exists
+ * @param filter The filter string,
+ * for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". Filtering can
+ * be done only on string partition keys.
+ * @return list of partitions
+ * @throws FalconException
+ */
+ public abstract List<CatalogPartition> listPartitionsByFilter(String catalogUrl, String database,
+ String tableName, String filter)
+ throws FalconException;
+
+ /**
+ * Drops a given partition.
+ *
+ * @param catalogUrl url for the catalog service
+ * @param database database the table belongs to
+ * @param tableName tableName to check if it exists
+ * @param partitions list of partitions as Key=Value pairs
+ * @return if the partition was dropped
+ * @throws FalconException
+ */
+ public abstract boolean dropPartitions(String catalogUrl, String database, String tableName,
+ Map<String, String> partitions) throws FalconException;
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java b/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java
new file mode 100644
index 0000000..ec656a3
--- /dev/null
+++ b/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.catalog;
+
+import java.util.List;
+
+/**
+ * The CatalogPartition is a wrapper around org.apache.hcatalog.api.HCatPartition.
+ */
+public class CatalogPartition {
+
+ private String databaseName;
+ private String tableName;
+ private List<String> values;
+ private int createTime;
+ private int lastAccessTime;
+ private List<String> tableColumns;
+ private String inputFormat;
+ private String outputFormat;
+ private String location;
+ private String serdeInfo;
+
+ protected CatalogPartition() {
+ }
+
+ protected void setDatabaseName(String databaseName) {
+ this.databaseName = databaseName;
+ }
+
+ protected void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ protected void setValues(List<String> values) {
+ this.values = values;
+ }
+
+ protected void setCreateTime(int createTime) {
+ this.createTime = createTime;
+ }
+
+ protected void setLastAccessTime(int lastAccessTime) {
+ this.lastAccessTime = lastAccessTime;
+ }
+
+ protected void setTableColumns(List<String> tableColumns) {
+ this.tableColumns = tableColumns;
+ }
+
+ protected void setInputFormat(String inputFormat) {
+ this.inputFormat = inputFormat;
+ }
+
+ protected void setOutputFormat(String outputFormat) {
+ this.outputFormat = outputFormat;
+ }
+
+ protected void setLocation(String location) {
+ this.location = location;
+ }
+
+ protected void setSerdeInfo(String serdeInfo) {
+ this.serdeInfo = serdeInfo;
+ }
+
+ /**
+ * Gets the database name.
+ *
+ * @return the database name
+ */
+ public String getDatabaseName() {
+ return this.databaseName;
+ }
+
+ /**
+ * Gets the table name.
+ *
+ * @return the table name
+ */
+ public String getTableName() {
+ return this.tableName;
+ }
+
+ /**
+ * Gets the columns of the table.
+ *
+ * @return the columns
+ */
+ public List<String> getColumns() {
+ return this.tableColumns;
+ }
+
+ /**
+ * Gets the input format.
+ *
+ * @return the input format
+ */
+ public String getInputFormat() {
+ return this.inputFormat;
+ }
+
+ /**
+ * Gets the output format.
+ *
+ * @return the output format
+ */
+ public String getOutputFormat() {
+ return this.outputFormat;
+ }
+
+ /**
+ * Gets the location.
+ *
+ * @return the location
+ */
+ public String getLocation() {
+ return this.location;
+ }
+
+ /**
+ * Gets the serde.
+ *
+ * @return the serde
+ */
+ public String getSerDe() {
+ return this.serdeInfo;
+ }
+
+ /**
+ * Gets the last access time.
+ *
+ * @return the last access time
+ */
+ public int getLastAccessTime() {
+ return this.lastAccessTime;
+ }
+
+ /**
+ * Gets the creates the time.
+ *
+ * @return the creates the time
+ */
+ public int getCreateTime() {
+ return this.createTime;
+ }
+
+ /**
+ * Gets the values.
+ *
+ * @return the values
+ */
+ public List<String> getValues() {
+ return this.values;
+ }
+
+ @Override
+ public String toString() {
+ return "HCatPartition ["
+ + (tableName != null ? "tableName=" + tableName + ", " : "tableName=null")
+ + (databaseName != null ? "dbName=" + databaseName + ", " : "dbName=null")
+ + (values != null ? "values=" + values + ", " : "values=null")
+ + "createTime=" + createTime + ", lastAccessTime="
+ + lastAccessTime + ", " + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
index f9e7bee..e3ddc3b 100644
--- a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
+++ b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
@@ -25,12 +25,15 @@ import org.apache.falcon.entity.v0.cluster.Interfacetype;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hcatalog.api.HCatClient;
import org.apache.hcatalog.api.HCatDatabase;
+import org.apache.hcatalog.api.HCatPartition;
import org.apache.hcatalog.api.HCatTable;
import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.log4j.Logger;
-import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -111,24 +114,75 @@ public class HiveCatalogService extends AbstractCatalogService {
}
@Override
- public Map<String, String> listTableProperties(String catalogUrl, String database,
- String tableName) throws FalconException {
+ public boolean isTableExternal(String catalogUrl, String database, String tableName)
+ throws FalconException {
LOG.info("Returns a list of table properties for:" + tableName);
try {
HCatClient client = get(catalogUrl);
HCatTable table = client.getTable(database, tableName);
+ return !table.getTabletype().equals("MANAGED_TABLE");
+ } catch (HCatException e) {
+ throw new FalconException(e);
+ }
+ }
+
+ @Override
+ public List<CatalogPartition> listPartitionsByFilter(String catalogUrl, String database,
+ String tableName, String filter)
+ throws FalconException {
+ LOG.info("List partitions for : " + tableName + ", partition filter: " + filter);
+
+ try {
+ List<CatalogPartition> catalogPartitionList = new ArrayList<CatalogPartition>();
+
+ HCatClient client = get(catalogUrl);
+ List<HCatPartition> hCatPartitions = client.listPartitionsByFilter(database, tableName, filter);
+ for (HCatPartition hCatPartition : hCatPartitions) {
+ CatalogPartition partition = createCatalogPartition(hCatPartition);
+ catalogPartitionList.add(partition);
+ }
+
+ return catalogPartitionList;
+ } catch (HCatException e) {
+ throw new FalconException(e);
+ }
+ }
- Map<String, String> tableProperties = new HashMap<String, String>();
- tableProperties.put("database", table.getDbName());
- tableProperties.put("tableName", table.getTableName());
- tableProperties.put("tabletype", table.getTabletype());
- tableProperties.put("location", table.getLocation());
- // tableProperties.putAll(table.getTblProps());
+ private CatalogPartition createCatalogPartition(HCatPartition hCatPartition) {
+ final CatalogPartition catalogPartition = new CatalogPartition();
+ catalogPartition.setDatabaseName(hCatPartition.getDatabaseName());
+ catalogPartition.setTableName(hCatPartition.getTableName());
+ catalogPartition.setValues(hCatPartition.getValues());
+ catalogPartition.setInputFormat(hCatPartition.getInputFormat());
+ catalogPartition.setOutputFormat(hCatPartition.getOutputFormat());
+ catalogPartition.setLocation(hCatPartition.getLocation());
+ catalogPartition.setSerdeInfo(hCatPartition.getSerDe());
+ catalogPartition.setCreateTime(hCatPartition.getCreateTime());
+ catalogPartition.setLastAccessTime(hCatPartition.getLastAccessTime());
+
+ List<String> tableColumns = new ArrayList<String>();
+ for (HCatFieldSchema hCatFieldSchema : hCatPartition.getColumns()) {
+ tableColumns.add(hCatFieldSchema.getName());
+ }
+ catalogPartition.setTableColumns(tableColumns);
+
+ return catalogPartition;
+ }
- return tableProperties;
+ @Override
+ public boolean dropPartitions(String catalogUrl, String database,
+ String tableName, Map<String, String> partitions)
+ throws FalconException {
+ LOG.info("Dropping partitions for : " + tableName + ", partitions: " + partitions);
+
+ try {
+ HCatClient client = get(catalogUrl);
+ client.dropPartitions(database, tableName, partitions, true);
} catch (HCatException e) {
throw new FalconException(e);
}
+
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/catalog/WebHCatCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/WebHCatCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/WebHCatCatalogService.java
deleted file mode 100644
index 49f97d1..0000000
--- a/common/src/main/java/org/apache/falcon/catalog/WebHCatCatalogService.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.catalog;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.falcon.FalconException;
-import org.apache.falcon.security.CurrentUser;
-import org.apache.log4j.Logger;
-
-import javax.ws.rs.core.MediaType;
-import java.util.Map;
-
-/**
- * An implementation of CatalogService that uses HTTP RESTful API to
- * talk to the Hive Meta Store (HCatalog) as the backing Catalog registry.
- */
-public class WebHCatCatalogService extends AbstractCatalogService {
-
- private static final Logger LOG = Logger.getLogger(WebHCatCatalogService.class);
-
- @Override
- public boolean isAlive(String catalogBaseUrl) throws FalconException {
- LOG.info("Checking if the service is alive for: " + catalogBaseUrl);
-
- Client client = Client.create();
- WebResource service = client.resource(catalogBaseUrl);
- ClientResponse response = service.path("status")
- .accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
-
- if (LOG.isDebugEnabled() && response.getStatus() != 200) {
- LOG.debug("Output from Server .... \n" + response.getEntity(String.class));
- }
-
- return response.getStatus() == 200;
- }
-
- @Override
- public boolean tableExists(String catalogUrl, String database, String tableName)
- throws FalconException {
- LOG.info("Checking if the table exists: " + tableName);
-
- Client client = Client.create();
- WebResource service = client.resource(catalogUrl);
-
- ClientResponse response = service.path("ddl/database").path(database)
- .path("table").path(tableName)
- .queryParam("user.name", CurrentUser.getUser())
- .accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
-
- if (LOG.isDebugEnabled() && response.getStatus() != 200) {
- LOG.debug("Output from Server .... \n" + response.getEntity(String.class));
- }
-
- return response.getStatus() == 200;
- }
-
- @Override
- public Map listTableProperties(String catalogUrl, String database,
- String tableName) throws FalconException {
- LOG.info("Returns a list of table properties for:" + tableName);
-
- Client client = Client.create();
- WebResource service = client.resource(catalogUrl);
-
- ClientResponse response = service.path("ddl/database/").path(database)
- .path("table").path(tableName)
- .path("property")
-// .queryParam("format", "extended")
- .queryParam("user.name", CurrentUser.getUser())
- .accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
-
- if (LOG.isDebugEnabled() && response.getStatus() != 200) {
- LOG.debug("Output from Server .... \n" + response.getEntity(String.class));
- }
-
- if (response.getStatus() != 200) {
- throw new FalconException("There was an error in this request: "
- + response.getEntity(String.class));
- }
-
- return response.getEntity(Map.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
index 6518140..8e49e38 100644
--- a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
@@ -20,6 +20,7 @@ package org.apache.falcon.entity;
import org.apache.falcon.FalconException;
import org.apache.falcon.catalog.CatalogServiceFactory;
+import org.apache.falcon.entity.common.FeedDataPath;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.Interfacetype;
import org.apache.falcon.entity.v0.feed.CatalogTable;
@@ -30,6 +31,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
+import java.util.regex.Matcher;
/**
* A catalog registry implementation of a feed storage.
@@ -42,8 +44,6 @@ public class CatalogStorage implements Storage {
public static final String OUTPUT_PATH_SEPARATOR = "/";
public static final String CATALOG_URL = "${hcatNode}";
- private static final String DOLLAR_EXPR_START = "_D__START_";
- private static final String EXPR_CLOSE = "_CLOSE_";
private final String catalogUrl;
private String database;
@@ -82,8 +82,8 @@ public class CatalogStorage implements Storage {
*/
private void parseFeedUri(String catalogTableUri) throws URISyntaxException {
- final String processed = catalogTableUri.replaceAll("\\$\\{", DOLLAR_EXPR_START)
- .replaceAll("}", EXPR_CLOSE);
+ final String processed = catalogTableUri.replaceAll(DOLLAR_EXPR_START_REGEX, DOLLAR_EXPR_START_NORMALIZED)
+ .replaceAll("}", EXPR_CLOSE_NORMALIZED);
URI tableUri = new URI(processed);
if (!"catalog".equals(tableUri.getScheme())) {
@@ -116,8 +116,8 @@ public class CatalogStorage implements Storage {
throw new URISyntaxException(tableUri.toString(), "Partition details are missing");
}
- final String rawPartition = partRaw.replaceAll(DOLLAR_EXPR_START, "\\$\\{")
- .replaceAll(EXPR_CLOSE, "\\}");
+ final String rawPartition = partRaw.replaceAll(DOLLAR_EXPR_START_NORMALIZED, DOLLAR_EXPR_START_REGEX)
+ .replaceAll(EXPR_CLOSE_NORMALIZED, EXPR_CLOSE_REGEX);
partitions = new HashMap<String, String>();
String[] parts = rawPartition.split(PARTITION_SEPARATOR);
for (String part : parts) {
@@ -147,8 +147,8 @@ public class CatalogStorage implements Storage {
throw new IllegalArgumentException("URI template cannot be null or empty");
}
- final String processed = uriTemplate.replaceAll("\\$\\{", DOLLAR_EXPR_START)
- .replaceAll("}", EXPR_CLOSE);
+ final String processed = uriTemplate.replaceAll(DOLLAR_EXPR_START_REGEX, DOLLAR_EXPR_START_NORMALIZED)
+ .replaceAll("}", EXPR_CLOSE_NORMALIZED);
URI uri = new URI(processed);
this.catalogUrl = uri.getScheme() + "://" + uri.getAuthority();
@@ -178,7 +178,8 @@ public class CatalogStorage implements Storage {
throw new URISyntaxException(uriTemplate.toString(), "Partition details are missing");
}
- String rawPartition = partRaw.replaceAll(DOLLAR_EXPR_START, "\\$\\{").replaceAll(EXPR_CLOSE, "\\}");
+ String rawPartition = partRaw.replaceAll(DOLLAR_EXPR_START_NORMALIZED, DOLLAR_EXPR_START_REGEX)
+ .replaceAll(EXPR_CLOSE_NORMALIZED, EXPR_CLOSE_REGEX);
partitions = new HashMap<String, String>();
String[] parts = rawPartition.split(PARTITION_SEPARATOR);
for (String part : parts) {
@@ -228,6 +229,21 @@ public class CatalogStorage implements Storage {
return partitions.containsKey(key);
}
+ public String getDatedPartitionKey() {
+ String datedPartitionKey = null;
+
+ for (Map.Entry<String, String> entry : getPartitions().entrySet()) {
+
+ Matcher matcher = FeedDataPath.PATTERN.matcher(entry.getValue());
+ if (matcher.find()) {
+ datedPartitionKey = entry.getKey();
+ break;
+ }
+ }
+
+ return datedPartitionKey;
+ }
+
@Override
public TYPE getType() {
return TYPE.TABLE;
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index 664813b..4e2dae9 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -39,8 +39,6 @@ public class FileSystemStorage implements Storage {
public static final String LOCATION_TYPE_SEP = "=";
public static final String FILE_SYSTEM_URL = "${nameNode}";
- private static final String DOLLAR_EXPR_START = "_D__START_";
- private static final String EXPR_CLOSE = "_CLOSE_";
private final String storageUrl;
private final List<Location> locations;
@@ -87,16 +85,16 @@ public class FileSystemStorage implements Storage {
String[] feedLocs = uriTemplate.split(FEED_PATH_SEP);
for (String rawPath : feedLocs) {
String[] typeAndPath = rawPath.split(LOCATION_TYPE_SEP);
- final String processed = typeAndPath[1].replaceAll("\\$\\{", DOLLAR_EXPR_START)
- .replaceAll("}", EXPR_CLOSE);
+ final String processed = typeAndPath[1].replaceAll(DOLLAR_EXPR_START_REGEX, DOLLAR_EXPR_START_NORMALIZED)
+ .replaceAll("}", EXPR_CLOSE_NORMALIZED);
URI uri = new URI(processed);
if (rawStorageUrl == null) {
rawStorageUrl = uri.getScheme() + "://" + uri.getAuthority();
}
String path = uri.getPath();
- final String finalPath = path.replaceAll(DOLLAR_EXPR_START, "\\$\\{")
- .replaceAll(EXPR_CLOSE, "\\}");
+ final String finalPath = path.replaceAll(DOLLAR_EXPR_START_NORMALIZED, DOLLAR_EXPR_START_REGEX)
+ .replaceAll(EXPR_CLOSE_NORMALIZED, EXPR_CLOSE_REGEX);
Location location = new Location();
location.setPath(finalPath);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/main/java/org/apache/falcon/entity/Storage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/Storage.java b/common/src/main/java/org/apache/falcon/entity/Storage.java
index 18dad65..0634969 100644
--- a/common/src/main/java/org/apache/falcon/entity/Storage.java
+++ b/common/src/main/java/org/apache/falcon/entity/Storage.java
@@ -27,6 +27,16 @@ import org.apache.falcon.entity.v0.feed.LocationType;
*/
public interface Storage {
+ String DOLLAR_EXPR_START_REGEX = "\\$\\{";
+ String QUESTION_EXPR_START_REGEX = "\\?\\{";
+ String EXPR_CLOSE_REGEX = "\\}";
+
+ /**
+ * URI Friendly expression.
+ */
+ String DOLLAR_EXPR_START_NORMALIZED = "_D__START_";
+ String EXPR_CLOSE_NORMALIZED = "_CLOSE_";
+
/**
* Enumeration for the various storage types.
*/
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
index 90eaebc..5c1878b 100644
--- a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
@@ -51,6 +51,23 @@ public class CatalogStorageTest {
Assert.assertTrue(storage.hasPartition("region"));
Assert.assertNull(storage.getPartitionValue("unknown"));
Assert.assertFalse(storage.hasPartition("unknown"));
+ Assert.assertEquals(storage.getDatedPartitionKey(), "ds");
+ }
+
+ @Test
+ public void testParseFeedUriValid2() throws URISyntaxException {
+ String table = "catalog:clicksdb:clicks#ds=${YEAR}${MONTH}${DAY};region=us";
+ CatalogStorage storage = new CatalogStorage(CatalogStorage.CATALOG_URL, table);
+ Assert.assertEquals("${hcatNode}", storage.getCatalogUrl());
+ Assert.assertEquals("clicksdb", storage.getDatabase());
+ Assert.assertEquals("clicks", storage.getTable());
+ Assert.assertEquals(Storage.TYPE.TABLE, storage.getType());
+ Assert.assertEquals(2, storage.getPartitions().size());
+ Assert.assertEquals("us", storage.getPartitionValue("region"));
+ Assert.assertTrue(storage.hasPartition("region"));
+ Assert.assertNull(storage.getPartitionValue("unknown"));
+ Assert.assertFalse(storage.hasPartition("unknown"));
+ Assert.assertEquals(storage.getDatedPartitionKey(), "ds");
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java
----------------------------------------------------------------------
diff --git a/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java b/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java
index bb0958c..bf5367c 100644
--- a/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java
+++ b/feed/src/main/java/org/apache/falcon/converter/OozieFeedMapper.java
@@ -58,14 +58,14 @@ public class OozieFeedMapper extends AbstractOozieEntityMapper<Feed> {
private static final int THIRTY_MINUTES = 30 * 60 * 1000;
- private static final String RETENTION_WF_TEMPLATE = "/config/workflow/retention-workflow.xml";
private static final String REPLICATION_COORD_TEMPLATE = "/config/coordinator/replication-coordinator.xml";
private static final String REPLICATION_WF_TEMPLATE = "/config/workflow/replication-workflow.xml";
- public static final String FEED_PATH_SEP = "#";
private static final String TIMEOUT = "timeout";
private static final String PARALLEL = "parallel";
+ private final RetentionOozieWorkflowMapper retentionMapper = new RetentionOozieWorkflowMapper();
+
public OozieFeedMapper(Feed feed) {
super(feed);
}
@@ -92,70 +92,8 @@ public class OozieFeedMapper extends AbstractOozieEntityMapper<Feed> {
+ " is not in the future");
return null;
}
- COORDINATORAPP retentionApp = new COORDINATORAPP();
- String coordName = EntityUtil.getWorkflowName(Tag.RETENTION, feed).toString();
- retentionApp.setName(coordName);
- retentionApp.setEnd(SchemaHelper.formatDateUTC(feedCluster.getValidity().getEnd()));
- retentionApp.setStart(SchemaHelper.formatDateUTC(new Date()));
- retentionApp.setTimezone(feed.getTimezone().getID());
- TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
- if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
- retentionApp.setFrequency("${coord:hours(6)}");
- } else {
- retentionApp.setFrequency("${coord:days(1)}");
- }
-
- Path wfPath = getCoordPath(bundlePath, coordName);
- retentionApp.setAction(getRetentionWorkflowAction(cluster, wfPath, coordName));
- return retentionApp;
- }
-
- private ACTION getRetentionWorkflowAction(Cluster cluster, Path wfPath, String wfName) throws FalconException {
- Feed feed = getEntity();
- ACTION retentionAction = new ACTION();
- WORKFLOW retentionWorkflow = new WORKFLOW();
- createRetentionWorkflow(cluster, wfPath, wfName);
- retentionWorkflow.setAppPath(getStoragePath(wfPath.toString()));
-
- Map<String, String> props = createCoordDefaultConfiguration(cluster, wfPath, wfName);
-
- org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());
- String feedBasePaths = getFeedDataPath(cluster, feed);
-
- props.put("feedDataPath", feedBasePaths);
- props.put("timeZone", feed.getTimezone().getID());
- props.put("frequency", feed.getFrequency().getTimeUnit().name());
- props.put("limit", feedCluster.getRetention().getLimit().toString());
- props.put(ARG.operation.getPropName(), EntityOps.DELETE.name());
- props.put(ARG.feedNames.getPropName(), feed.getName());
- props.put(ARG.feedInstancePaths.getPropName(), "IGNORE");
-
- retentionWorkflow.setConfiguration(getCoordConfig(props));
- retentionAction.setWorkflow(retentionWorkflow);
- return retentionAction;
- }
-
- protected String getFeedDataPath(Cluster cluster, Feed feed) throws FalconException {
- final Storage storage = FeedHelper.createStorage(cluster, feed);
- String feedPathMask = storage.getUriTemplate(LocationType.DATA);
- String metaPathMask = storage.getUriTemplate(LocationType.META);
- String statsPathMask = storage.getUriTemplate(LocationType.STATS);
- String tmpPathMask = storage.getUriTemplate(LocationType.TMP);
-
- StringBuilder feedBasePaths = new StringBuilder(feedPathMask);
- if (metaPathMask != null) {
- feedBasePaths.append(FEED_PATH_SEP).append(metaPathMask);
- }
-
- if (statsPathMask != null) {
- feedBasePaths.append(FEED_PATH_SEP).append(statsPathMask);
- }
-
- if (tmpPathMask != null) {
- feedBasePaths.append(FEED_PATH_SEP).append(tmpPathMask);
- }
- return feedBasePaths.toString().replaceAll("\\$\\{", "\\?\\{");
+ return retentionMapper.getRetentionCoordinator(cluster, bundlePath, feed, feedCluster);
}
private List<COORDINATORAPP> getReplicationCoordinators(Cluster targetCluster, Path bundlePath)
@@ -344,17 +282,6 @@ public class OozieFeedMapper extends AbstractOozieEntityMapper<Feed> {
}
}
- private void createRetentionWorkflow(Cluster cluster, Path wfPath, String wfName) throws FalconException {
- try {
- WORKFLOWAPP retWfApp = getWorkflowTemplate(RETENTION_WF_TEMPLATE);
- retWfApp.setName(wfName);
- addLibExtensionsToWorkflow(cluster, retWfApp, EntityType.FEED, "retention");
- marshal(cluster, retWfApp, wfPath);
- } catch(IOException e) {
- throw new FalconException("Unable to create retention workflow", e);
- }
- }
-
@Override
protected Map<String, String> getEntityProperties() {
Feed feed = getEntity();
@@ -366,4 +293,76 @@ public class OozieFeedMapper extends AbstractOozieEntityMapper<Feed> {
}
return props;
}
+
+ private final class RetentionOozieWorkflowMapper {
+
+ private static final String RETENTION_WF_TEMPLATE = "/config/workflow/retention-workflow.xml";
+
+ private COORDINATORAPP getRetentionCoordinator(Cluster cluster, Path bundlePath, Feed feed,
+ org.apache.falcon.entity.v0.feed.Cluster feedCluster)
+ throws FalconException {
+
+ COORDINATORAPP retentionApp = new COORDINATORAPP();
+ String coordName = EntityUtil.getWorkflowName(Tag.RETENTION, feed).toString();
+ retentionApp.setName(coordName);
+ retentionApp.setEnd(SchemaHelper.formatDateUTC(feedCluster.getValidity().getEnd()));
+ retentionApp.setStart(SchemaHelper.formatDateUTC(new Date()));
+ retentionApp.setTimezone(feed.getTimezone().getID());
+ TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
+ if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
+ retentionApp.setFrequency("${coord:hours(6)}");
+ } else {
+ retentionApp.setFrequency("${coord:days(1)}");
+ }
+
+ Path wfPath = getCoordPath(bundlePath, coordName);
+ retentionApp.setAction(getRetentionWorkflowAction(cluster, wfPath, coordName));
+ return retentionApp;
+ }
+
+ private ACTION getRetentionWorkflowAction(Cluster cluster, Path wfPath, String wfName)
+ throws FalconException {
+ Feed feed = getEntity();
+ ACTION retentionAction = new ACTION();
+ WORKFLOW retentionWorkflow = new WORKFLOW();
+ createRetentionWorkflow(cluster, wfPath, wfName);
+ retentionWorkflow.setAppPath(getStoragePath(wfPath.toString()));
+
+ Map<String, String> props = createCoordDefaultConfiguration(cluster, wfPath, wfName);
+
+ props.put("timeZone", feed.getTimezone().getID());
+ props.put("frequency", feed.getFrequency().getTimeUnit().name());
+
+ final Storage storage = FeedHelper.createStorage(cluster, feed);
+ props.put("falconFeedStorageType", storage.getType().name());
+
+ String feedDataPath = storage.getUriTemplate();
+ props.put("feedDataPath",
+ feedDataPath.replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));
+
+ org.apache.falcon.entity.v0.feed.Cluster feedCluster =
+ FeedHelper.getCluster(feed, cluster.getName());
+ props.put("limit", feedCluster.getRetention().getLimit().toString());
+
+ props.put(ARG.operation.getPropName(), EntityOps.DELETE.name());
+ props.put(ARG.feedNames.getPropName(), feed.getName());
+ props.put(ARG.feedInstancePaths.getPropName(), "IGNORE");
+
+ retentionWorkflow.setConfiguration(getCoordConfig(props));
+ retentionAction.setWorkflow(retentionWorkflow);
+
+ return retentionAction;
+ }
+
+ private void createRetentionWorkflow(Cluster cluster, Path wfPath, String wfName) throws FalconException {
+ try {
+ WORKFLOWAPP retWfApp = getWorkflowTemplate(RETENTION_WF_TEMPLATE);
+ retWfApp.setName(wfName);
+ addLibExtensionsToWorkflow(cluster, retWfApp, EntityType.FEED, "retention");
+ marshal(cluster, retWfApp, wfPath);
+ } catch(IOException e) {
+ throw new FalconException("Unable to create retention workflow", e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/feed/src/main/resources/config/workflow/retention-workflow.xml
----------------------------------------------------------------------
diff --git a/feed/src/main/resources/config/workflow/retention-workflow.xml b/feed/src/main/resources/config/workflow/retention-workflow.xml
index 72e0133..8b444f5 100644
--- a/feed/src/main/resources/config/workflow/retention-workflow.xml
+++ b/feed/src/main/resources/config/workflow/retention-workflow.xml
@@ -30,10 +30,21 @@
<name>oozie.launcher.mapred.job.priority</name>
<value>${jobPriority}</value>
</property>
+ <!-- HCatalog jars -->
+ <property>
+ <name>oozie.use.system.libpath</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>oozie.action.sharelib.for.java</name>
+ <value>hcatalog</value>
+ </property>
</configuration>
<main-class>org.apache.falcon.retention.FeedEvictor</main-class>
<arg>-feedBasePath</arg>
<arg>${feedDataPath}</arg>
+ <arg>-falconFeedStorageType</arg>
+ <arg>${falconFeedStorageType}</arg>
<arg>-retentionType</arg>
<arg>instance</arg>
<arg>-retentionLimit</arg>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
----------------------------------------------------------------------
diff --git a/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java b/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
index cec278e..330cc63 100644
--- a/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
+++ b/feed/src/test/java/org/apache/falcon/converter/OozieFeedMapperTest.java
@@ -17,8 +17,6 @@
*/
package org.apache.falcon.converter;
-import static org.testng.Assert.assertEquals;
-
import java.util.Calendar;
import java.util.Collection;
import java.util.List;
@@ -31,6 +29,7 @@ import org.apache.falcon.FalconException;
import org.apache.falcon.cluster.util.EmbeddedCluster;
import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.entity.FeedHelper;
+import org.apache.falcon.entity.Storage;
import org.apache.falcon.entity.store.ConfigurationStore;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
@@ -162,7 +161,7 @@ public class OozieFeedMapperTest {
for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
if (prop.getName().equals("mapred.job.priority")) {
- assertEquals(prop.getValue(), "NORMAL");
+ Assert.assertEquals(prop.getValue(), "NORMAL");
break;
}
}
@@ -211,17 +210,25 @@ public class OozieFeedMapperTest {
Assert.assertEquals(coord.getFrequency(), "${coord:hours(6)}");
String feedDataPath = null;
+ String storageType = null;
org.apache.falcon.oozie.coordinator.CONFIGURATION configuration =
coord.getAction().getWorkflow().getConfiguration();
for (Property property : configuration.getProperty()) {
if ("feedDataPath".equals(property.getName())) {
feedDataPath = property.getValue();
- break;
+ } else if ("falconFeedStorageType".equals(property.getName())) {
+ storageType = property.getValue();
}
}
+ final Storage storage = FeedHelper.createStorage(cluster, feed);
if (feedDataPath != null) {
- Assert.assertEquals(feedDataPath, feedMapper.getFeedDataPath(srcCluster, feed));
+ Assert.assertEquals(feedDataPath, storage.getUriTemplate()
+ .replaceAll(Storage.DOLLAR_EXPR_START_REGEX, Storage.QUESTION_EXPR_START_REGEX));
+ }
+
+ if (storageType != null) {
+ Assert.assertEquals(storageType, storage.getType().name());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/hadoop-webapp/src/main/resources/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/webhcat-site.xml b/hadoop-webapp/src/main/resources/webhcat-site.xml
deleted file mode 100644
index 6d0772c..0000000
--- a/hadoop-webapp/src/main/resources/webhcat-site.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
-
- <property>
- <name>templeton.hadoop.conf.dir</name>
- <value>${env.HADOOP_CONF_DIR}</value>
- <description>The path to the Hadoop configuration.</description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
----------------------------------------------------------------------
diff --git a/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java b/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
index a0c7bb7..75ef78c 100644
--- a/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
+++ b/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
@@ -46,9 +47,17 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.el.ExpressionEvaluatorImpl;
+import org.apache.falcon.FalconException;
import org.apache.falcon.Pair;
+import org.apache.falcon.catalog.CatalogPartition;
+import org.apache.falcon.catalog.CatalogServiceFactory;
+import org.apache.falcon.entity.CatalogStorage;
+import org.apache.falcon.entity.FeedHelper;
+import org.apache.falcon.entity.FileSystemStorage;
+import org.apache.falcon.entity.Storage;
import org.apache.falcon.entity.common.FeedDataPath;
import org.apache.falcon.entity.common.FeedDataPath.VARS;
+import org.apache.falcon.entity.v0.feed.Location;
import org.apache.falcon.expression.ExpressionHelper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -72,7 +81,6 @@ public class FeedEvictor extends Configured implements Tool {
private static final ExpressionHelper RESOLVER = ExpressionHelper.get();
public static final AtomicReference<PrintStream> OUT = new AtomicReference<PrintStream>(System.out);
-// static PrintStream stream = System.out;
private static final String FORMAT = "yyyyMMddHHmm";
@@ -88,7 +96,6 @@ public class FeedEvictor extends Configured implements Tool {
}
}
- private FileSystem fs;
private final Map<VARS, String> map = new TreeMap<VARS, String>();
private final StringBuffer instancePaths = new StringBuffer("instancePaths=");
private final StringBuffer buffer = new StringBuffer();
@@ -97,47 +104,73 @@ public class FeedEvictor extends Configured implements Tool {
public int run(String[] args) throws Exception {
CommandLine cmd = getCommand(args);
- String feedBasePath = cmd.getOptionValue("feedBasePath").replaceAll("\\?\\{", "\\$\\{");
+ String feedBasePath = cmd.getOptionValue("feedBasePath")
+ .replaceAll(Storage.QUESTION_EXPR_START_REGEX, Storage.DOLLAR_EXPR_START_REGEX);
String retentionType = cmd.getOptionValue("retentionType");
String retentionLimit = cmd.getOptionValue("retentionLimit");
String timeZone = cmd.getOptionValue("timeZone");
String frequency = cmd.getOptionValue("frequency"); //to write out smart path filters
String logFile = cmd.getOptionValue("logFile");
+ String feedStorageType = cmd.getOptionValue("falconFeedStorageType");
- String[] feedLocs = feedBasePath.split("#");
- for (String path : feedLocs) {
- evictor(path, retentionType, retentionLimit, timeZone, frequency);
- }
+ LOG.info("Applying retention on " + feedBasePath + " type: " + retentionType
+ + ", Limit: " + retentionLimit + ", timezone: " + timeZone
+ + ", frequency: " + frequency + ", storage" + feedStorageType);
+
+ Storage storage = FeedHelper.createStorage(feedStorageType, feedBasePath);
+ evict(storage, retentionLimit, timeZone);
logInstancePaths(new Path(logFile));
+
int len = buffer.length();
if (len > 0) {
OUT.get().println("instances=" + buffer.substring(0, len - 1));
} else {
OUT.get().println("instances=NULL");
}
+
return 0;
}
- private void evictor(String feedBasePath, String retentionType,
- String retentionLimit, String timeZone, String frequency) throws IOException, ELException {
+ private void evict(Storage storage, String retentionLimit, String timeZone)
+ throws Exception {
+
+ if (storage.getType() == Storage.TYPE.FILESYSTEM) {
+ evictFS((FileSystemStorage) storage, retentionLimit, timeZone);
+ } else if (storage.getType() == Storage.TYPE.TABLE) {
+ evictTable((CatalogStorage) storage, retentionLimit, timeZone);
+ }
+ }
+
+ private void evictFS(FileSystemStorage storage, String retentionLimit, String timeZone)
+ throws Exception {
+
+ for (Location location : storage.getLocations()) {
+ fileSystemEvictor(storage.getUriTemplate(location.getType()), retentionLimit, timeZone);
+ }
+ }
+
+ private void fileSystemEvictor(String feedBasePath, String retentionLimit, String timeZone)
+ throws IOException, ELException {
+
Path normalizedPath = new Path(feedBasePath);
- fs = normalizedPath.getFileSystem(getConf());
+ FileSystem fs = normalizedPath.getFileSystem(getConf());
feedBasePath = normalizedPath.toUri().getPath();
LOG.info("Normalized path : " + feedBasePath);
+
Pair<Date, Date> range = getDateRange(retentionLimit);
String dateMask = getDateFormatInPath(feedBasePath);
- List<Path> toBeDeleted = discoverInstanceToDelete(feedBasePath,
- timeZone, dateMask, range.first);
- LOG.info("Applying retention on " + feedBasePath + " type: "
- + retentionType + ", Limit: " + retentionLimit + ", timezone: "
- + timeZone + ", frequency: " + frequency);
+ List<Path> toBeDeleted = discoverInstanceToDelete(feedBasePath, timeZone, dateMask, range.first, fs);
+ if (toBeDeleted.isEmpty()) {
+ LOG.info("No instances to delete.");
+ return;
+ }
DateFormat dateFormat = new SimpleDateFormat(FORMAT);
dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
for (Path path : toBeDeleted) {
- if (deleteInstance(path)) {
+ if (deleteInstance(fs, path)) {
LOG.info("Deleted instance " + path);
Date date = getDate(path, feedBasePath, dateMask, timeZone);
buffer.append(dateFormat.format(date)).append(',');
@@ -165,10 +198,10 @@ public class FeedEvictor extends Configured implements Tool {
return Pair.of(start, end);
}
- private List<Path> discoverInstanceToDelete(String inPath, String timeZone,
- String dateMask, Date start) throws IOException {
+ private List<Path> discoverInstanceToDelete(String inPath, String timeZone, String dateMask,
+ Date start, FileSystem fs) throws IOException {
- FileStatus[] files = findFilesForFeed(inPath);
+ FileStatus[] files = findFilesForFeed(fs, inPath);
if (files == null || files.length == 0) {
return Collections.emptyList();
}
@@ -196,7 +229,7 @@ public class FeedEvictor extends Configured implements Tool {
.replaceAll(VARS.MINUTE.regex(), "mm");
}
- private FileStatus[] findFilesForFeed(String feedBasePath) throws IOException {
+ private FileStatus[] findFilesForFeed(FileSystem fs, String feedBasePath) throws IOException {
Matcher matcher = FeedDataPath.PATTERN.matcher(feedBasePath);
while (matcher.find()) {
@@ -272,13 +305,13 @@ public class FeedEvictor extends Configured implements Tool {
return date.compareTo(start) >= 0;
}
- private boolean deleteInstance(Path path) throws IOException {
+ private boolean deleteInstance(FileSystem fs, Path path) throws IOException {
return fs.delete(path, true);
}
- private void debug(FileSystem myfs, Path outPath) throws IOException {
+ private void debug(FileSystem fs, Path outPath) throws IOException {
ByteArrayOutputStream writer = new ByteArrayOutputStream();
- InputStream instance = myfs.open(outPath);
+ InputStream instance = fs.open(outPath);
IOUtils.copyBytes(instance, writer, 4096, true);
LOG.debug("Instance Paths copied to " + outPath);
LOG.debug("Written " + writer);
@@ -286,29 +319,122 @@ public class FeedEvictor extends Configured implements Tool {
private CommandLine getCommand(String[] args) throws org.apache.commons.cli.ParseException {
Options options = new Options();
- Option opt;
- opt = new Option("feedBasePath", true,
+
+ Option opt = new Option("feedBasePath", true,
"base path for feed, ex /data/feed/${YEAR}-${MONTH}");
opt.setRequired(true);
options.addOption(opt);
+
+ opt = new Option("falconFeedStorageType", true, "feed storage type, FileSystem or Table");
+ opt.setRequired(true);
+ options.addOption(opt);
+
opt = new Option("retentionType", true,
"type of retention policy like delete, archive etc");
opt.setRequired(true);
options.addOption(opt);
+
opt = new Option("retentionLimit", true,
"time limit for retention, ex hours(5), months(2), days(90)");
opt.setRequired(true);
options.addOption(opt);
+
opt = new Option("timeZone", true, "timezone for feed, ex UTC");
opt.setRequired(true);
options.addOption(opt);
+
opt = new Option("frequency", true,
"frequency of feed, ex hourly, daily, monthly, minute, weekly, yearly");
opt.setRequired(true);
options.addOption(opt);
+
opt = new Option("logFile", true, "log file for capturing size of feed");
opt.setRequired(true);
options.addOption(opt);
+
return new GnuParser().parse(options, args);
}
+
+ private void evictTable(CatalogStorage storage, String retentionLimit, String timeZone)
+ throws Exception {
+
+ LOG.info("Applying retention on " + storage.getTable()
+ + ", Limit: " + retentionLimit + ", timezone: " + timeZone);
+
+ String dateMask = getDateFormatInPath(storage.getUriTemplate());
+
+ List<CatalogPartition> toBeDeleted = discoverPartitionsToDelete(
+ storage, retentionLimit, timeZone, dateMask);
+ if (toBeDeleted.isEmpty()) {
+ LOG.info("No partitions to delete.");
+ return;
+ }
+
+ final boolean isTableExternal = CatalogServiceFactory.getCatalogService().isTableExternal(
+ storage.getCatalogUrl(), storage.getDatabase(), storage.getTable());
+
+ dropPartitions(storage, toBeDeleted, isTableExternal);
+ }
+
+ private List<CatalogPartition> discoverPartitionsToDelete(CatalogStorage storage, String retentionLimit,
+ String timeZone, String dateMask)
+ throws FalconException, ELException {
+
+ final String filter = createFilter(storage, retentionLimit, timeZone, dateMask);
+ return CatalogServiceFactory.getCatalogService().listPartitionsByFilter(
+ storage.getCatalogUrl(), storage.getDatabase(), storage.getTable(), filter);
+ }
+
+ private String createFilter(CatalogStorage storage, String retentionLimit,
+ String timeZone, String dateMask) throws ELException {
+
+ Pair<Date, Date> range = getDateRange(retentionLimit);
+ DateFormat dateFormat = new SimpleDateFormat(FORMAT.substring(0, dateMask.length()));
+ dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
+ String beforeDate = dateFormat.format(range.first);
+
+ String datedPartitionKey = storage.getDatedPartitionKey();
+
+ StringBuilder filterBuffer = new StringBuilder();
+ filterBuffer.append(datedPartitionKey)
+ .append(" < ")
+ .append("\"")
+ .append(beforeDate)
+ .append("\"");
+
+ return filterBuffer.toString();
+ }
+
+ private void dropPartitions(CatalogStorage storage, List<CatalogPartition> partitionsToDelete,
+ boolean isTableExternal) throws FalconException, IOException {
+
+ for (CatalogPartition partitionToDrop : partitionsToDelete) {
+ if (dropPartition(storage, partitionToDrop, isTableExternal)) {
+ LOG.info("Deleted partition: " + partitionToDrop.getValues());
+ buffer.append(partitionToDrop.getValues().get(0)).append(',');
+ instancePaths.append(partitionToDrop.getValues()).append(",");
+ }
+ }
+ }
+
+ private boolean dropPartition(CatalogStorage storage, CatalogPartition partitionToDrop,
+ boolean isTableExternal) throws FalconException, IOException {
+
+ String datedPartitionKey = storage.getDatedPartitionKey();
+
+ Map<String, String> partitions = new HashMap<String, String>();
+ partitions.put(datedPartitionKey, partitionToDrop.getValues().get(0));
+
+ boolean dropped = CatalogServiceFactory.getCatalogService().dropPartitions(
+ storage.getCatalogUrl(), storage.getDatabase(), storage.getTable(), partitions);
+
+ boolean deleted = true;
+ if (isTableExternal) { // nuke the dirs if an external table
+ final String location = partitionToDrop.getLocation();
+ final Path path = new Path(location);
+ deleted = path.getFileSystem(new Configuration()).delete(path, true);
+ }
+
+ return dropped && deleted;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index 5377c48..529418b 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -18,6 +18,19 @@
package org.apache.falcon.retention;
+import org.apache.falcon.Pair;
+import org.apache.falcon.cluster.util.EmbeddedCluster;
+import org.apache.falcon.entity.Storage;
+import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -33,17 +46,6 @@ import java.util.Map;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
-import org.apache.falcon.Pair;
-import org.apache.falcon.cluster.util.EmbeddedCluster;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
/**
* Test for FeedEvictor.
*/
@@ -100,20 +102,28 @@ public class FeedEvictorTest {
fs.delete(new Path("/"), true);
stream.clear();
- Pair<List<String>, List<String>> pair;
- pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
- String dataPath = "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
+ Pair<List<String>, List<String>> pair =
+ createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
+ final String storageUrl = cluster.getConf().get("fs.default.name");
+ String dataPath = LocationType.DATA.name() + "="
+ + storageUrl + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
FeedEvictor.main(new String[]{
- "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
- "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency",
- "daily", "-logFile", logFile, });
+ "-feedBasePath", dataPath,
+ "-retentionType", "instance",
+ "-retentionLimit", "days(10)",
+ "-timeZone", "UTC",
+ "-frequency", "daily",
+ "-logFile", logFile,
+ "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+ });
assertFailures(fs, pair);
compare(map.get("feed1"), stream.getBuffer());
- Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
+ Assert.assertEquals(readLogFile(new Path(logFile)),
+ getExpectedInstancePaths(dataPath.replaceAll(storageUrl, "")));
} catch (Exception e) {
Assert.fail("Unknown exception", e);
@@ -121,7 +131,7 @@ public class FeedEvictorTest {
}
private String getExpectedInstancePaths(String dataPath) {
- StringBuffer newBuffer = new StringBuffer("instancePaths=");
+ StringBuilder newBuffer = new StringBuilder("instancePaths=");
DateFormat format = new SimpleDateFormat("yyyyMMddHHmm");
format.setTimeZone(TimeZone.getTimeZone("UTC"));
String[] locs = dataPath.split("#");
@@ -132,7 +142,9 @@ public class FeedEvictorTest {
for (int i = 0; i < locs.length; i++) {
for (int j = 0, k = i * instances.length / locs.length; j < instances.length / locs.length; j++) {
- String instancePath = locs[i].replaceAll("\\?\\{YEAR\\}", instances[j + k].substring(0, 4));
+ String[] paths = locs[i].split("=");
+ String path = paths[1];
+ String instancePath = path.replaceAll("\\?\\{YEAR\\}", instances[j + k].substring(0, 4));
instancePath = instancePath.replaceAll("\\?\\{MONTH\\}", instances[j + k].substring(4, 6));
instancePath = instancePath.replaceAll("\\?\\{DAY\\}", instances[j + k].substring(6, 8));
instancePath = instancePath.replaceAll("\\?\\{HOUR\\}", instances[j + k].substring(8, 10));
@@ -182,19 +194,27 @@ public class FeedEvictorTest {
fs.delete(new Path("/"), true);
stream.clear();
- Pair<List<String>, List<String>> pair;
- pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
- String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
+ Pair<List<String>, List<String>> pair =
+ createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
+ final String storageUrl = cluster.getConf().get("fs.default.name");
+ String dataPath = LocationType.DATA.name() + "="
+ + storageUrl + "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
FeedEvictor.main(new String[]{
- "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
- "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency",
- "hourly", "-logFile", logFile, });
+ "-feedBasePath", dataPath,
+ "-retentionType", "instance",
+ "-retentionLimit", "hours(5)",
+ "-timeZone", "UTC",
+ "-frequency", "hourly",
+ "-logFile", logFile,
+ "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+ });
assertFailures(fs, pair);
compare(map.get("feed2"), stream.getBuffer());
- Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
+ Assert.assertEquals(readLogFile(new Path(logFile)),
+ getExpectedInstancePaths(dataPath.replaceAll(storageUrl, "")));
} catch (Exception e) {
Assert.fail("Unknown exception", e);
@@ -210,21 +230,31 @@ public class FeedEvictorTest {
fs.delete(new Path("/"), true);
stream.clear();
- Pair<List<String>, List<String>> pair;
- pair = createTestData("/data");
+ Pair<List<String>, List<String>> pair = createTestData("/data");
FeedEvictor.main(new String[] {
- "-feedBasePath",
- cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
- "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
- "UTC", "-frequency", "hourly", "-logFile", hdfsUrl + "/falcon/staging/feed/2012-01-01-04-00", });
+ "-feedBasePath", LocationType.DATA.name() + "="
+ + cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
+ "-retentionType", "instance",
+ "-retentionLimit", "months(5)",
+ "-timeZone", "UTC",
+ "-frequency", "hourly",
+ "-logFile", conf.get("fs.default.name") + "/falcon/staging/feed/2012-01-01-04-00",
+ "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+ });
Assert.assertEquals("instances=NULL", stream.getBuffer());
stream.clear();
String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
- FeedEvictor.main(new String[] {"-feedBasePath",
- cluster.getConf().get("fs.default.name") + dataPath, "-retentionType", "instance",
- "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency", "hourly", "-logFile", logFile, });
+ FeedEvictor.main(new String[] {
+ "-feedBasePath", LocationType.DATA.name() + "=" + cluster.getConf().get("fs.default.name") + dataPath,
+ "-retentionType", "instance",
+ "-retentionLimit", "hours(5)",
+ "-timeZone", "UTC",
+ "-frequency", "hourly",
+ "-logFile", logFile,
+ "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+ });
Assert.assertEquals("instances=NULL", stream.getBuffer());
Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
@@ -243,28 +273,38 @@ public class FeedEvictorTest {
fs.delete(new Path("/"), true);
stream.clear();
- Pair<List<String>, List<String>> pair, statsPair, metaPair, tmpPair;
- pair = createTestData("/data");
- statsPair = createTestData("/stats");
- metaPair = createTestData("/meta");
- tmpPair = createTestData("/tmp");
- FeedEvictor.main(new String[] {"-feedBasePath",
- getFeedBasePath("/data") + "#"
- + getFeedBasePath("/stats") + "#"
- + getFeedBasePath("/meta") + "#"
- + getFeedBasePath("/tmp"),
- "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
- "UTC", "-frequency", "hourly", "-logFile", hdfsUrl + "/falcon/staging/feed/2012-01-01-04-00", });
+ Pair<List<String>, List<String>> pair = createTestData("/data");
+ createTestData("/stats");
+ createTestData("/meta");
+ createTestData("/tmp");
+ final String storageUrl = cluster.getConf().get("fs.default.name");
+ FeedEvictor.main(new String[] {
+ "-feedBasePath", getFeedBasePath(LocationType.DATA, storageUrl)
+ + "#" + getFeedBasePath(LocationType.STATS, storageUrl)
+ + "#" + getFeedBasePath(LocationType.META, storageUrl)
+ + "#" + getFeedBasePath(LocationType.TMP, storageUrl),
+ "-retentionType", "instance",
+ "-retentionLimit", "months(5)",
+ "-timeZone", "UTC",
+ "-frequency", "hourly",
+ "-logFile", conf.get("fs.default.name") + "/falcon/staging/feed/2012-01-01-04-00",
+ "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+ });
Assert.assertEquals("instances=NULL", stream.getBuffer());
stream.clear();
- String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
+ String dataPath = LocationType.DATA.name() + "="
+ + cluster.getConf().get("fs.default.name") + "/data/YYYY/feed4/dd/MM/02/more/hello";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
- FeedEvictor.main(new String[]{"-feedBasePath",
- cluster.getConf().get("fs.default.name") + dataPath,
- "-retentionType", "instance", "-retentionLimit",
- "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
- "-logFile", logFile, });
+ FeedEvictor.main(new String[]{
+ "-feedBasePath", dataPath,
+ "-retentionType", "instance",
+ "-retentionLimit", "hours(5)",
+ "-timeZone", "UTC",
+ "-frequency", "hourly",
+ "-logFile", logFile,
+ "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+ });
Assert.assertEquals("instances=NULL", stream.getBuffer());
Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
@@ -283,29 +323,32 @@ public class FeedEvictorTest {
fs.delete(new Path("/"), true);
stream.clear();
- Pair<List<String>, List<String>> pair, statsPair, metaPair;
- pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
- statsPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/stats");
- metaPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/meta");
- String dataPath = cluster.getConf().get("fs.default.name")
- + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
- + "#"
- + cluster.getConf().get("fs.default.name")
- + "/stats/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
- + "#"
- + cluster.getConf().get("fs.default.name")
- + "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
+ Pair<List<String>, List<String>> pair =
+ createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
+ createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/stats");
+ createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/meta");
+
+ final String storageUrl = cluster.getConf().get("fs.default.name");
+ String dataPath =
+ "DATA=" + storageUrl + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
+ + "#STATS=" + storageUrl + "/stats/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
+ + "#META=" + storageUrl + "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
- FeedEvictor.main(new String[] {"-feedBasePath", dataPath, "-retentionType", "instance",
- "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency", "daily", "-logFile", logFile, });
+ FeedEvictor.main(new String[] {
+ "-feedBasePath", dataPath,
+ "-retentionType", "instance",
+ "-retentionLimit", "days(10)",
+ "-timeZone", "UTC",
+ "-frequency", "daily",
+ "-logFile", logFile,
+ "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+ });
assertFailures(fs, pair);
Assert.assertEquals(readLogFile(new Path(logFile)),
- getExpectedInstancePaths(dataPath.replaceAll(cluster
- .getConf().get("fs.default.name"), "")));
-
+ getExpectedInstancePaths(dataPath.replaceAll(storageUrl, "")));
} catch (Exception e) {
Assert.fail("Unknown exception", e);
@@ -375,10 +418,10 @@ public class FeedEvictorTest {
outOfRange.add(path);
} else {
inRange.add(path);
- buffer.append((displayFormat.format(date) + "0000").
- substring(0, 12)).append(',');
+ buffer.append((displayFormat.format(date) + "0000").substring(0, 12)).append(',');
}
}
+
map.put(feed, "instances=" + buffer.substring(0, buffer.length() - 1));
return Pair.of(inRange, outOfRange);
}
@@ -387,10 +430,9 @@ public class FeedEvictorTest {
fs.create(new Path(path)).close();
}
- private String getFeedBasePath(String locationType) {
- return cluster.getConf().get("fs.default.name")
- + "/data/YYYY/feed3/dd/MM/"
- + "?{MONTH}/more/?{HOUR}";
+ private String getFeedBasePath(LocationType locationType, String storageUrl) {
+ return locationType.name() + "=" + storageUrl
+ + "/" + locationType.name().toLowerCase() + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}";
}
private static class InMemoryWriter extends PrintStream {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
----------------------------------------------------------------------
diff --git a/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java b/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
index 6864651..ab1d038 100644
--- a/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
+++ b/test-util/src/main/java/org/apache/falcon/cluster/util/StandAloneCluster.java
@@ -43,6 +43,7 @@ public final class StandAloneCluster extends EmbeddedCluster {
for (Interface inter : cluster.getCluster().getInterfaces().getInterfaces()) {
if (inter.getType() == Interfacetype.WRITE) {
cluster.getConf().set("fs.default.name", inter.getEndpoint());
+ break;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/5a8fa133/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
index 94d14ff..9e42b94 100644
--- a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
+++ b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java
@@ -23,10 +23,14 @@ import org.apache.hcatalog.api.HCatAddPartitionDesc;
import org.apache.hcatalog.api.HCatClient;
import org.apache.hcatalog.api.HCatCreateDBDesc;
import org.apache.hcatalog.api.HCatCreateTableDesc;
+import org.apache.hcatalog.api.HCatPartition;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.util.ArrayList;
@@ -40,9 +44,11 @@ import java.util.Map;
*/
public class HiveCatalogServiceIT {
- private static final String DATABASE_NAME = "falcondb";
- private static final String TABLE_NAME = "foobar";
private static final String METASTORE_URL = "thrift://localhost:49083";
+ private static final String DATABASE_NAME = "falcon_db";
+ private static final String TABLE_NAME = "falcon_table";
+ private static final String EXTERNAL_TABLE_NAME = "falcon_external";
+ private static final String EXTERNAL_TABLE_LOCATION = "hdfs://localhost:41020/falcon/staging/falcon_external";
private HiveCatalogService hiveCatalogService;
private HCatClient client;
@@ -54,7 +60,7 @@ public class HiveCatalogServiceIT {
createDatabase();
createTable();
- addPartition();
+ createExternalTable();
}
private void createDatabase() throws Exception {
@@ -69,8 +75,8 @@ public class HiveCatalogServiceIT {
cols.add(new HCatFieldSchema("value", HCatFieldSchema.Type.STRING, "value comment"));
List<HCatFieldSchema> partitionSchema = Arrays.asList(
- new HCatFieldSchema("ds", HCatFieldSchema.Type.STRING, ""),
- new HCatFieldSchema("region", HCatFieldSchema.Type.STRING, "")
+ new HCatFieldSchema("ds", HCatFieldSchema.Type.STRING, ""),
+ new HCatFieldSchema("region", HCatFieldSchema.Type.STRING, "")
);
HCatCreateTableDesc tableDesc = HCatCreateTableDesc
@@ -83,41 +89,76 @@ public class HiveCatalogServiceIT {
client.createTable(tableDesc);
}
- private void addPartition() throws Exception {
+ public void createExternalTable() throws Exception {
+ ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
+ cols.add(new HCatFieldSchema("id", HCatFieldSchema.Type.INT, "id comment"));
+ cols.add(new HCatFieldSchema("value", HCatFieldSchema.Type.STRING, "value comment"));
+
+ List<HCatFieldSchema> partitionSchema = Arrays.asList(
+ new HCatFieldSchema("ds", HCatFieldSchema.Type.STRING, ""),
+ new HCatFieldSchema("region", HCatFieldSchema.Type.STRING, "")
+ );
+
+ HCatCreateTableDesc tableDesc = HCatCreateTableDesc
+ .create(DATABASE_NAME, EXTERNAL_TABLE_NAME, cols)
+ .fileFormat("rcfile")
+ .ifNotExists(true)
+ .comments("falcon integration test")
+ .partCols(new ArrayList<HCatFieldSchema>(partitionSchema))
+ .isTableExternal(true)
+ .location(EXTERNAL_TABLE_LOCATION)
+ .build();
+ client.createTable(tableDesc);
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ dropTable(EXTERNAL_TABLE_NAME);
+ dropTable(TABLE_NAME);
+ dropDatabase();
+ }
+
+ private void dropTable(String tableName) throws Exception {
+ client.dropTable(DATABASE_NAME, tableName, true);
+ }
+
+ private void dropDatabase() throws Exception {
+ client.dropDatabase(DATABASE_NAME, true, HCatClient.DropDBMode.CASCADE);
+ }
+
+ @BeforeMethod
+ private void addPartitions() throws Exception {
Map<String, String> firstPtn = new HashMap<String, String>();
- firstPtn.put("ds", "09/03/2013");
- firstPtn.put("region", "usa");
+ firstPtn.put("ds", "20130903"); //yyyyMMDD
+ firstPtn.put("region", "us");
HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(
DATABASE_NAME, TABLE_NAME, null, firstPtn).build();
client.addPartition(addPtn);
Map<String, String> secondPtn = new HashMap<String, String>();
- secondPtn.put("ds", "09/03/2013");
- secondPtn.put("region", "india");
+ secondPtn.put("ds", "20130903");
+ secondPtn.put("region", "in");
HCatAddPartitionDesc addPtn2 = HCatAddPartitionDesc.create(
DATABASE_NAME, TABLE_NAME, null, secondPtn).build();
client.addPartition(addPtn2);
Map<String, String> thirdPtn = new HashMap<String, String>();
- thirdPtn.put("ds", "09/02/2013");
- thirdPtn.put("region", "india");
+ thirdPtn.put("ds", "20130902");
+ thirdPtn.put("region", "in");
HCatAddPartitionDesc addPtn3 = HCatAddPartitionDesc.create(
DATABASE_NAME, TABLE_NAME, null, thirdPtn).build();
client.addPartition(addPtn3);
}
- @AfterClass
- public void tearDown() throws Exception {
- dropTable();
- dropDatabase();
- }
+ @AfterMethod
+ private void dropPartitions() throws Exception {
+ Map<String, String> partitionSpec = new HashMap<String, String>();
+ partitionSpec.put("ds", "20130903");
+ client.dropPartitions(DATABASE_NAME, TABLE_NAME, partitionSpec, true);
- private void dropTable() throws Exception {
- client.dropTable(DATABASE_NAME, TABLE_NAME, true);
- }
-
- private void dropDatabase() throws Exception {
- client.dropDatabase(DATABASE_NAME, true, HCatClient.DropDBMode.CASCADE);
+ partitionSpec = new HashMap<String, String>();
+ partitionSpec.put("ds", "20130902");
+ client.dropPartitions(DATABASE_NAME, TABLE_NAME, partitionSpec, true);
}
@Test
@@ -146,12 +187,116 @@ public class HiveCatalogServiceIT {
}
@Test
- public void testListTableProperties() throws Exception {
- Map<String, String> tableProperties =
- hiveCatalogService.listTableProperties(METASTORE_URL, DATABASE_NAME, TABLE_NAME);
- Assert.assertEquals(tableProperties.get("database"), DATABASE_NAME);
- Assert.assertEquals(tableProperties.get("tableName"), TABLE_NAME);
- Assert.assertEquals(tableProperties.get("tabletype"), "MANAGED_TABLE");
- Assert.assertTrue(tableProperties.containsKey("location"));
+ public void testIsTableExternalFalse() throws Exception {
+ Assert.assertFalse(hiveCatalogService.isTableExternal(METASTORE_URL, DATABASE_NAME, TABLE_NAME));
+ }
+ @Test
+ public void testIsTableExternalTrue() throws Exception {
+ Assert.assertTrue(hiveCatalogService.isTableExternal(METASTORE_URL, DATABASE_NAME, EXTERNAL_TABLE_NAME));
+ }
+
+ @Test
+ public void testListPartitionsByFilterNull() throws Exception {
+
+ List<CatalogPartition> filteredPartitions = hiveCatalogService.listPartitionsByFilter(
+ METASTORE_URL, DATABASE_NAME, TABLE_NAME, null);
+ Assert.assertEquals(filteredPartitions.size(), 3);
+ }
+
+ @DataProvider (name = "lessThanFilter")
+ public Object[][] createLessThanFilter() {
+ return new Object[][] {
+ {"ds < \"20130905\"", 3},
+ {"ds < \"20130904\"", 3},
+ {"ds < \"20130903\"", 1},
+ {"ds < \"20130902\"", 0},
+ };
+ }
+
+ @Test (dataProvider = "lessThanFilter")
+ public void testListPartitionsByFilterLessThan(String lessThanFilter, int expectedPartitionCount)
+ throws Exception {
+
+ List<CatalogPartition> filteredPartitions = hiveCatalogService.listPartitionsByFilter(
+ METASTORE_URL, DATABASE_NAME, TABLE_NAME, lessThanFilter);
+ Assert.assertEquals(filteredPartitions.size(), expectedPartitionCount);
+ }
+
+ @DataProvider (name = "greaterThanFilter")
+ public Object[][] createGreaterThanFilter() {
+ return new Object[][] {
+ {"ds > \"20130831\"", 3},
+ {"ds > \"20130905\"", 0},
+ {"ds > \"20130904\"", 0},
+ {"ds > \"20130903\"", 0},
+ {"ds > \"20130902\"", 2},
+ };
+ }
+
+ @Test (dataProvider = "greaterThanFilter")
+ public void testListPartitionsByFilterGreaterThan(String greaterThanFilter, int expectedPartitionCount)
+ throws Exception {
+
+ List<CatalogPartition> filteredPartitions = hiveCatalogService.listPartitionsByFilter(
+ METASTORE_URL, DATABASE_NAME, TABLE_NAME, greaterThanFilter);
+ Assert.assertEquals(filteredPartitions.size(), expectedPartitionCount);
+ }
+
+ @Test
+ public void testGetPartitionsFullSpec() throws Exception {
+ Map<String, String> partitionSpec = new HashMap<String, String>();
+ partitionSpec.put("ds", "20130902");
+ partitionSpec.put("region", "in");
+
+ HCatPartition ptn = client.getPartition(DATABASE_NAME, TABLE_NAME, partitionSpec);
+ Assert.assertTrue(ptn != null);
+ }
+
+ @Test
+ public void testGetPartitionsPartialSpec() throws Exception {
+ Map<String, String> partialPartitionSpec = new HashMap<String, String>();
+ partialPartitionSpec.put("ds", "20130903");
+
+ List<HCatPartition> partitions = client.getPartitions(DATABASE_NAME, TABLE_NAME, partialPartitionSpec);
+ Assert.assertEquals(partitions.size(), 2);
+ }
+
+ @Test
+ public void testDropPartition() throws Exception {
+ Map<String, String> partialPartitionSpec = new HashMap<String, String>();
+ partialPartitionSpec.put("ds", "20130903");
+
+ Assert.assertTrue(hiveCatalogService.dropPartitions(
+ METASTORE_URL, DATABASE_NAME, TABLE_NAME, partialPartitionSpec));
+
+ List<HCatPartition> partitions = client.getPartitions(DATABASE_NAME, TABLE_NAME);
+ Assert.assertEquals(1, partitions.size(), "Unexpected number of partitions");
+ Assert.assertEquals(new String[]{"20130902", "in"},
+ partitions.get(0).getValues().toArray(), "Mismatched partition");
+
+ partialPartitionSpec = new HashMap<String, String>();
+ partialPartitionSpec.put("ds", "20130902");
+
+ Assert.assertTrue(hiveCatalogService.dropPartitions(
+ METASTORE_URL, DATABASE_NAME, TABLE_NAME, partialPartitionSpec));
+ partitions = client.getPartitions(DATABASE_NAME, TABLE_NAME);
+ Assert.assertEquals(partitions.size(), 0, "Unexpected number of partitions");
+ }
+
+ /*
+ // this is NOT possible to do in Hive
+ @Test
+ public void testDropPartitionBulk() throws Exception {
+ Map<String, String> partialPartitionSpec = new HashMap<String, String>();
+ partialPartitionSpec.put("ds", "20130903");
+ partialPartitionSpec.put("ds", "20130902");
+ partialPartitionSpec.put("ds", "20130901");
+
+ Assert.assertTrue(hiveCatalogService.dropPartitions(
+ METASTORE_URL, DATABASE_NAME, TABLE_NAME, partialPartitionSpec));
+
+ List<HCatPartition> partitions = client.getPartitions(DATABASE_NAME, TABLE_NAME);
+ Assert.assertEquals(0, partitions.size(), "Unexpected number of partitions");
}
+ */
}