You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ba...@apache.org on 2016/04/27 09:03:44 UTC
falcon git commit: FALCON-1909 Catalog instance triage action fails
with null pointer exception
Repository: falcon
Updated Branches:
refs/heads/master 2945fa80b -> 123391d46
FALCON-1909 Catalog instance triage action fails with null pointer exception
…ception.
Author: bvellanki <bv...@hortonworks.com>
Reviewers: " Praveen Adlakha <ad...@gmail.com>, yzheng-hortonworks <yz...@hortonworks.com>, Venkat Ranganathan <ve...@hortonworks.com>"
Closes #117 from bvellanki/FALCON-1909
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/123391d4
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/123391d4
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/123391d4
Branch: refs/heads/master
Commit: 123391d460b5c1ed30209d10618377da023fdb4b
Parents: 2945fa8
Author: bvellanki <bv...@hortonworks.com>
Authored: Wed Apr 27 00:03:34 2016 -0700
Committer: bvellanki <bv...@hortonworks.com>
Committed: Wed Apr 27 00:03:34 2016 -0700
----------------------------------------------------------------------
.../apache/falcon/entity/CatalogStorage.java | 21 +++++++++++++-------
.../falcon/entity/CatalogStorageTest.java | 10 ++++++----
.../apache/falcon/catalog/CatalogStorageIT.java | 12 ++++-------
3 files changed, 24 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
index c5860c9..31feee8 100644
--- a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
@@ -100,7 +100,7 @@ public class CatalogStorage extends Configured implements Storage {
if (catalogUrl == null || catalogUrl.length() == 0) {
throw new IllegalArgumentException("Catalog Registry URL cannot be null or empty");
}
-
+ verifyAndUpdateConfiguration(getConf());
this.catalogUrl = catalogUrl;
parseFeedUri(tableUri);
@@ -177,6 +177,10 @@ public class CatalogStorage extends Configured implements Storage {
* @throws URISyntaxException
*/
protected CatalogStorage(String uriTemplate) throws URISyntaxException {
+ this(uriTemplate, new Configuration());
+ }
+
+ protected CatalogStorage(String uriTemplate, Configuration conf) throws URISyntaxException {
if (uriTemplate == null || uriTemplate.length() == 0) {
throw new IllegalArgumentException("URI template cannot be null or empty");
}
@@ -186,15 +190,10 @@ public class CatalogStorage extends Configured implements Storage {
URI uri = new URI(processed);
this.catalogUrl = uri.getScheme() + "://" + uri.getAuthority();
-
+ verifyAndUpdateConfiguration(conf);
parseUriTemplate(uri);
}
- protected CatalogStorage(String uriTemplate, Configuration conf) throws URISyntaxException {
- this(uriTemplate);
- setConf(conf);
- }
-
private void parseUriTemplate(URI uriTemplate) throws URISyntaxException {
String path = uriTemplate.getPath();
String[] paths = path.split(OUTPUT_PATH_SEPARATOR);
@@ -589,4 +588,12 @@ public class CatalogStorage extends Configured implements Storage {
+ ", partitions=" + partitions
+ '}';
}
+
+ private void verifyAndUpdateConfiguration(Configuration conf) {
+ if (conf == null) {
+ setConf(new Configuration());
+ } else {
+ setConf(conf);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
index 5d06431..d20c722 100644
--- a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
@@ -36,6 +36,7 @@ public class CatalogStorageTest {
String table = "catalog:clicksdb:clicks#ds=${YEAR}-${MONTH}-${DAY};region=us";
CatalogStorage storage = new CatalogStorage(CatalogStorage.CATALOG_URL, table);
Assert.assertEquals(Storage.TYPE.TABLE, storage.getType());
+ Assert.assertNotNull(storage.getConf());
}
@Test
@@ -160,7 +161,7 @@ public class CatalogStorageTest {
String uriTemplate = "thrift://localhost:49083/clicksdb/clicks/ds=${YEAR}-${MONTH}-${DAY};region=us";
CatalogStorage table = new CatalogStorage(catalogUrl, tableUri);
-
+ Assert.assertNotNull(table.getConf());
Assert.assertEquals(uriTemplate, table.getUriTemplate());
Assert.assertEquals(uriTemplate, table.getUriTemplate(LocationType.DATA));
Assert.assertEquals(table.getUriTemplate(), table.getUriTemplate(LocationType.DATA));
@@ -172,7 +173,7 @@ public class CatalogStorageTest {
String uriTemplate = "${hcatNode}/clicksdb/clicks/ds=${YEAR}-${MONTH}-${DAY};region=us";
CatalogStorage table = new CatalogStorage(CatalogStorage.CATALOG_URL, tableUri);
-
+ Assert.assertNotNull(table.getConf());
Assert.assertEquals(uriTemplate, table.getUriTemplate());
Assert.assertEquals(uriTemplate, table.getUriTemplate(LocationType.DATA));
Assert.assertEquals(table.getUriTemplate(), table.getUriTemplate(LocationType.DATA));
@@ -183,9 +184,9 @@ public class CatalogStorageTest {
final String catalogUrl = "thrift://localhost:49083";
String tableUri = "catalog:clicksdb:clicks#ds=20130918;region=us";
String partitionFilter = "(ds='20130918';region='us')";
-
CatalogStorage table = new CatalogStorage(catalogUrl, tableUri);
Assert.assertEquals(table.toPartitionFilter(), partitionFilter);
+ Assert.assertNotNull(table.getConf());
}
@Test
@@ -193,9 +194,9 @@ public class CatalogStorageTest {
final String catalogUrl = "thrift://localhost:49083";
String tableUri = "catalog:clicksdb:clicks#ds=20130918;region=us";
String partitionPath = "ds=20130918/region=us";
-
CatalogStorage table = new CatalogStorage(catalogUrl, tableUri);
Assert.assertEquals(table.toPartitionAsPath(), partitionPath);
+ Assert.assertNotNull(table.getConf());
}
@Test
@@ -209,5 +210,6 @@ public class CatalogStorageTest {
Assert.assertEquals(1, storage.getPartitions().size());
Assert.assertEquals("2012-04-21-00", storage.getPartitionValue("ds"));
Assert.assertTrue(storage.hasPartition("ds"));
+ Assert.assertNotNull(storage.getConf());
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java b/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java
index bf3b2ec..262f8ac 100644
--- a/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java
+++ b/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java
@@ -24,16 +24,15 @@ import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.cluster.Interface;
import org.apache.falcon.entity.v0.cluster.Interfaces;
import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.entity.v0.feed.Feed;
-import org.apache.falcon.entity.v0.feed.Clusters;
-import org.apache.falcon.entity.v0.feed.Validity;
import org.apache.falcon.entity.v0.feed.CatalogTable;
import org.apache.falcon.entity.v0.feed.ClusterType;
+import org.apache.falcon.entity.v0.feed.Clusters;
+import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.falcon.entity.v0.feed.Validity;
import org.apache.falcon.resource.TestContext;
import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.util.HiveTestUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hive.hcatalog.api.HCatAddPartitionDesc;
import org.apache.hive.hcatalog.api.HCatClient;
import org.apache.hive.hcatalog.common.HCatException;
@@ -42,13 +41,12 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.HashMap;
import java.util.TimeZone;
@@ -116,8 +114,6 @@ public class CatalogStorageIT {
feed.setTable(table);
storage = new CatalogStorage(cluster, table);
- Configuration configuration = HiveCatalogService.createHiveConf(new Configuration(), storage.getCatalogUrl());
- storage.setConf(configuration);
}