You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2013/09/24 00:55:45 UTC
[3/6] git commit: Revert "FALCON-86 Hive table integration with
cluster entity. Contributed by Venkatesh Seetharam"
Revert "FALCON-86 Hive table integration with cluster entity. Contributed by Venkatesh Seetharam"
This reverts commit 8b09e1b272113e9b769a20f09b6c0c4e2ec7c18a.
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/d422dba4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/d422dba4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/d422dba4
Branch: refs/heads/FALCON-85
Commit: d422dba4816fb3003d72c8c1e3d835c19c99f50c
Parents: f1db98b
Author: srikanth.sundarrajan <sr...@inmobi.com>
Authored: Mon Sep 2 07:56:11 2013 +0530
Committer: srikanth.sundarrajan <sr...@inmobi.com>
Committed: Mon Sep 2 07:56:11 2013 +0530
----------------------------------------------------------------------
.../falcon/catalog/AbstractCatalogService.java | 37 --------------
.../falcon/catalog/CatalogServiceFactory.java | 49 ------------------
.../falcon/catalog/HiveCatalogService.java | 46 -----------------
.../entity/parser/ClusterEntityParser.java | 31 +----------
common/src/main/resources/startup.properties | 2 -
.../entity/parser/ClusterEntityParserTest.java | 54 ++++++++------------
.../resources/config/cluster/cluster-0.1.xml | 3 +-
.../config/cluster/cluster-no-registry.xml | 42 ---------------
src/conf/startup.properties | 2 -
.../resource/ClusterEntityValidationIT.java | 2 -
webapp/src/test/resources/cluster-template.xml | 6 +--
11 files changed, 28 insertions(+), 246 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
deleted file mode 100644
index 4086611..0000000
--- a/common/src/main/java/org/apache/falcon/catalog/AbstractCatalogService.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.catalog;
-
-import org.apache.falcon.FalconException;
-
-/**
- * Interface definition for a catalog registry service
- * such as Hive or HCatalog.
- */
-public abstract class AbstractCatalogService {
-
- /**
- * This method checks if the catalog service is alive.
- *
- * @param catalogBaseUrl url for the catalog service
- * @return if the service was reachable
- * @throws FalconException exception
- */
- public abstract boolean isAlive(String catalogBaseUrl) throws FalconException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/common/src/main/java/org/apache/falcon/catalog/CatalogServiceFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/CatalogServiceFactory.java b/common/src/main/java/org/apache/falcon/catalog/CatalogServiceFactory.java
deleted file mode 100644
index c8a0fa0..0000000
--- a/common/src/main/java/org/apache/falcon/catalog/CatalogServiceFactory.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.catalog;
-
-import org.apache.falcon.FalconException;
-import org.apache.falcon.util.ReflectionUtils;
-import org.apache.falcon.util.StartupProperties;
-
-/**
- * Factory for providing appropriate catalog service
- * implementation to the falcon service.
- */
-@SuppressWarnings("unchecked")
-public final class CatalogServiceFactory {
-
- public static final String CATALOG_SERVICE = "catalog.service.impl";
-
- private CatalogServiceFactory() {
- }
-
- public static boolean isEnabled() {
- return StartupProperties.get().containsKey(CATALOG_SERVICE);
- }
-
- public static AbstractCatalogService getCatalogService() throws FalconException {
- if (!isEnabled()) {
- throw new FalconException(
- "Catalog integration is not enabled in falcon. Implementation is missing: " + CATALOG_SERVICE);
- }
-
- return ReflectionUtils.getInstance(CATALOG_SERVICE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
deleted file mode 100644
index e6f7fe2..0000000
--- a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.falcon.catalog;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.falcon.FalconException;
-import org.apache.log4j.Logger;
-
-import javax.ws.rs.core.MediaType;
-
-/**
- * An implementation of CatalogService that uses Hive Meta Store (HCatalog)
- * as the backing Catalog registry.
- */
-public class HiveCatalogService extends AbstractCatalogService {
-
- private static final Logger LOG = Logger.getLogger(HiveCatalogService.class);
-
- @Override
- public boolean isAlive(String catalogBaseUrl) throws FalconException {
- LOG.info("Checking if the service is alive for: " + catalogBaseUrl);
-
- Client client = Client.create();
- WebResource service = client.resource(catalogBaseUrl);
- ClientResponse response = service.path("status").accept(MediaType.APPLICATION_JSON).head();
- return response.getStatus() == 200;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
index 631ec60..b4e4a95 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
@@ -23,13 +23,11 @@ import java.io.IOException;
import javax.jms.ConnectionFactory;
import org.apache.falcon.FalconException;
-import org.apache.falcon.catalog.CatalogServiceFactory;
import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.entity.store.StoreAccessException;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.entity.v0.cluster.Interface;
import org.apache.falcon.util.DeploymentUtil;
import org.apache.falcon.util.StartupProperties;
import org.apache.falcon.workflow.WorkflowEngineFactory;
@@ -58,9 +56,6 @@ public class ClusterEntityParser extends EntityParser<Cluster> {
validateScheme(cluster, Interfacetype.WRITE);
validateScheme(cluster, Interfacetype.WORKFLOW);
validateScheme(cluster, Interfacetype.MESSAGING);
- if (ClusterHelper.getInterface(cluster, Interfacetype.REGISTRY) != null) {
- validateScheme(cluster, Interfacetype.REGISTRY);
- }
// No interface validations in prism or other falcon servers.
// Only the falcon server for which the cluster belongs to should validate interfaces
@@ -74,7 +69,8 @@ public class ClusterEntityParser extends EntityParser<Cluster> {
validateExecuteInterface(cluster);
validateWorkflowInterface(cluster);
validateMessagingInterface(cluster);
- validateRegistryInterface(cluster);
+
+ // Interfacetype.REGISTRY is not validated as its not used
}
private void validateScheme(Cluster cluster, Interfacetype interfacetype)
@@ -157,27 +153,4 @@ public class ClusterEntityParser extends EntityParser<Cluster> {
+ " for: " + implementation, e);
}
}
-
- private void validateRegistryInterface(Cluster cluster) throws ValidationException {
- final Interface catalogInterface = ClusterHelper.getInterface(cluster, Interfacetype.REGISTRY);
- if (catalogInterface == null) {
- LOG.info("Catalog service is not enabled for cluster: " + cluster.getName());
- return;
- }
-
- if (!CatalogServiceFactory.isEnabled()) {
- throw new ValidationException("Catalog registry implementation is not defined: catalog.service.impl");
- }
-
- final String catalogUrl = catalogInterface.getEndpoint();
- LOG.info("Validating catalog registry interface: " + catalogUrl);
-
- try {
- if (!CatalogServiceFactory.getCatalogService().isAlive(catalogUrl)) {
- throw new ValidationException("Unable to reach Catalog server:" + catalogUrl);
- }
- } catch (FalconException e) {
- throw new ValidationException("Invalid Catalog server or port: " + catalogUrl, e);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/common/src/main/resources/startup.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties
index 80d46f7..1b4e470 100644
--- a/common/src/main/resources/startup.properties
+++ b/common/src/main/resources/startup.properties
@@ -27,8 +27,6 @@
*.SchedulableEntityManager.impl=org.apache.falcon.resource.SchedulableEntityManager
*.ConfigSyncService.impl=org.apache.falcon.resource.ConfigSyncService
*.ProcessInstanceManager.impl=org.apache.falcon.resource.InstanceManager
-*.catalog.service.impl=org.apache.falcon.catalog.HiveCatalogService
-
*.application.services=org.apache.falcon.entity.store.ConfigurationStore,\
org.apache.falcon.service.ProcessSubscriberService,\
org.apache.falcon.rerun.service.RetryService,\
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
index dffa691..20d14e8 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/ClusterEntityParserTest.java
@@ -18,6 +18,15 @@
package org.apache.falcon.entity.parser;
+import static org.testng.AssertJUnit.assertEquals;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+
import org.apache.falcon.FalconException;
import org.apache.falcon.cluster.util.EmbeddedCluster;
import org.apache.falcon.entity.AbstractTestBase;
@@ -26,16 +35,11 @@ import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.Interface;
import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.util.StartupProperties;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import javax.xml.bind.JAXBException;
-import java.io.IOException;
-import java.io.InputStream;
-
/**
* Test for validating cluster entity parsing.
*/
@@ -52,45 +56,31 @@ public class ClusterEntityParserTest extends AbstractTestBase {
ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(conf.get("fs.default.name"));
Assert.assertNotNull(cluster);
- Assert.assertEquals(cluster.getName(), "testCluster");
+ assertEquals(cluster.getName(), "testCluster");
Interface execute = ClusterHelper.getInterface(cluster, Interfacetype.EXECUTE);
- Assert.assertEquals(execute.getEndpoint(), "localhost:8021");
- Assert.assertEquals(execute.getVersion(), "0.20.2");
+ assertEquals(execute.getEndpoint(), "localhost:8021");
+ assertEquals(execute.getVersion(), "0.20.2");
Interface readonly = ClusterHelper.getInterface(cluster, Interfacetype.READONLY);
- Assert.assertEquals(readonly.getEndpoint(), "hftp://localhost:50010");
- Assert.assertEquals(readonly.getVersion(), "0.20.2");
+ assertEquals(readonly.getEndpoint(), "hftp://localhost:50010");
+ assertEquals(readonly.getVersion(), "0.20.2");
Interface write = ClusterHelper.getInterface(cluster, Interfacetype.WRITE);
//assertEquals(write.getEndpoint(), conf.get("fs.default.name"));
- Assert.assertEquals(write.getVersion(), "0.20.2");
+ assertEquals(write.getVersion(), "0.20.2");
Interface workflow = ClusterHelper.getInterface(cluster, Interfacetype.WORKFLOW);
- Assert.assertEquals(workflow.getEndpoint(), "http://localhost:11000/oozie/");
- Assert.assertEquals(workflow.getVersion(), "3.1");
-
- Interface catalog = ClusterHelper.getInterface(cluster, Interfacetype.REGISTRY);
- Assert.assertEquals(catalog.getEndpoint(), "http://localhost:48080/templeton/v1");
- Assert.assertEquals(catalog.getVersion(), "0.11.0");
-
- Assert.assertEquals(ClusterHelper.getLocation(cluster, "staging"), "/projects/falcon/staging");
- }
-
- @Test
- public void testParseClusterWithoutRegistry() throws IOException, FalconException, JAXBException {
-
- InputStream stream = this.getClass().getResourceAsStream("/config/cluster/cluster-no-registry.xml");
-
- Cluster cluster = parser.parse(stream);
+ assertEquals(workflow.getEndpoint(), "http://localhost:11000/oozie/");
+ assertEquals(workflow.getVersion(), "3.1");
- Interface catalog = ClusterHelper.getInterface(cluster, Interfacetype.REGISTRY);
- Assert.assertNull(catalog);
+ assertEquals(ClusterHelper.getLocation(cluster, "staging"), "/projects/falcon/staging");
- StartupProperties.get().setProperty("catalog.service.impl", "");
- catalog = ClusterHelper.getInterface(cluster, Interfacetype.REGISTRY);
- Assert.assertNull(catalog);
+ StringWriter stringWriter = new StringWriter();
+ Marshaller marshaller = EntityType.CLUSTER.getMarshaller();
+ marshaller.marshal(cluster, stringWriter);
+ System.out.println(stringWriter.toString());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/common/src/test/resources/config/cluster/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/cluster/cluster-0.1.xml b/common/src/test/resources/config/cluster/cluster-0.1.xml
index 658711d..fd6e06e 100644
--- a/common/src/test/resources/config/cluster/cluster-0.1.xml
+++ b/common/src/test/resources/config/cluster/cluster-0.1.xml
@@ -29,8 +29,7 @@
version="3.1"/>
<interface type="messaging" endpoint="tcp://localhost:61616?daemon=true"
version="5.1.6"/>
- <interface type="registry" endpoint="http://localhost:48080/templeton/v1"
- version="0.11.0"/>
+ <interface type="registry" endpoint="Hcat" version="1"/>
</interfaces>
<locations>
<location name="staging" path="/projects/falcon/staging"/>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/common/src/test/resources/config/cluster/cluster-no-registry.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/cluster/cluster-no-registry.xml b/common/src/test/resources/config/cluster/cluster-no-registry.xml
deleted file mode 100644
index 85dfe32..0000000
--- a/common/src/test/resources/config/cluster/cluster-no-registry.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0"?>
-<!--~
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<cluster colo="default" description="" name="testCluster" xmlns="uri:falcon:cluster:0.1">
- <tags>consumer=consumer@xyz.com, owner=producer@xyz.com, department=forecasting</tags>
- <interfaces>
- <interface type="readonly" endpoint="hftp://localhost:50010"
- version="0.20.2"/>
- <interface type="write" endpoint="hdfs://localhost:8020"
- version="0.20.2"/>
- <interface type="execute" endpoint="localhost:8021" version="0.20.2"/>
- <interface type="workflow" endpoint="http://localhost:11000/oozie/"
- version="3.1"/>
- <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true"
- version="5.1.6"/>
- </interfaces>
- <locations>
- <location name="staging" path="/projects/falcon/staging"/>
- <location name="temp" path="/tmp"/>
- <location name="working" path="/projects/falcon/working"/>
- </locations>
- <properties>
- <property name="field1" value="value1"/>
- <property name="field2" value="value2"/>
- </properties>
-</cluster>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index dca38b4..6b8617e 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -31,8 +31,6 @@
*.SchedulableEntityManager.impl=org.apache.falcon.resource.SchedulableEntityManager
*.ConfigSyncService.impl=org.apache.falcon.resource.ConfigSyncService
*.ProcessInstanceManager.impl=org.apache.falcon.resource.InstanceManager
-*.catalog.service.impl=org.apache.falcon.catalog.HiveCatalogService
-
*.application.services=org.apache.falcon.entity.store.ConfigurationStore,\
org.apache.falcon.service.ProcessSubscriberService,\
org.apache.falcon.rerun.service.RetryService,\
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java b/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java
index b96c994..8f2b6e4 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/ClusterEntityValidationIT.java
@@ -77,8 +77,6 @@ public class ClusterEntityValidationIT {
{Interfacetype.WORKFLOW, "http://workflow-interface:9999/oozie/"},
{Interfacetype.MESSAGING, "messaging-interface:9999"},
{Interfacetype.MESSAGING, "tcp://messaging-interface:9999"},
- {Interfacetype.REGISTRY, "catalog-interface:9999"},
- {Interfacetype.REGISTRY, "http://catalog-interface:9999"},
};
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/d422dba4/webapp/src/test/resources/cluster-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/cluster-template.xml b/webapp/src/test/resources/cluster-template.xml
index fbc46c5..11aaedf 100644
--- a/webapp/src/test/resources/cluster-template.xml
+++ b/webapp/src/test/resources/cluster-template.xml
@@ -17,7 +17,8 @@
limitations under the License.
-->
-<cluster colo="##colo##" description="" name="##cluster##" xmlns="uri:falcon:cluster:0.1">
+<cluster colo="##colo##" description="" name="##cluster##" xmlns="uri:falcon:cluster:0.1"
+ >
<interfaces>
<interface type="readonly" endpoint="hftp://localhost:41110"
version="0.20.2"/>
@@ -28,8 +29,7 @@
version="3.1"/>
<interface type="messaging" endpoint="tcp://localhost:61616?daemon=true"
version="5.1.6"/>
- <interface type="registry" endpoint="http://localhost:48080/templeton/v1/"
- version="0.11.0"/>
+ <interface type="registry" endpoint="Hcat" version="1"/>
</interfaces>
<locations>
<location name="staging" path="/projects/falcon/staging"/>