You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ka...@apache.org on 2012/11/20 11:11:48 UTC
svn commit: r1411608 - in /oozie/branches/hcat-intre: ./ core/
core/src/main/java/org/apache/oozie/
core/src/main/java/org/apache/oozie/hcat/
core/src/main/java/org/apache/oozie/service/
core/src/test/java/org/apache/oozie/hcat/ core/src/test/java/org/...
Author: kamrul
Date: Tue Nov 20 10:11:47 2012
New Revision: 1411608
URL: http://svn.apache.org/viewvc?rev=1411608&view=rev
Log:
OOZIE-1068 Metadata Accessor service for HCatalog(mohammad)
Added:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/hcat/
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/hcat/MetaDataClientWrapper.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorException.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorService.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestMetaDataAccessorService.java
Modified:
oozie/branches/hcat-intre/core/pom.xml
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/ErrorCode.java
oozie/branches/hcat-intre/pom.xml
Modified: oozie/branches/hcat-intre/core/pom.xml
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/pom.xml?rev=1411608&r1=1411607&r2=1411608&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/pom.xml (original)
+++ oozie/branches/hcat-intre/core/pom.xml Tue Nov 20 10:11:47 2012
@@ -277,6 +277,12 @@
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hcatalog</groupId>
+ <artifactId>webhcat-java-client</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
<!-- For drawing runtime DAG -->
<dependency>
<groupId>net.sf.jung</groupId>
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/ErrorCode.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/ErrorCode.java?rev=1411608&r1=1411607&r2=1411608&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/ErrorCode.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/ErrorCode.java Tue Nov 20 10:11:47 2012
@@ -232,6 +232,7 @@ public enum ErrorCode {
E1501(XLog.STD, "Partition Dependency Manager could not add cache entry"),
E1502(XLog.STD, "Partition cache lookup error"),
E1503(XLog.STD, "Error in Metadata URI [{0}]"),
+ E1504(XLog.STD, "Error in getting HCat Access [{0}]"),
ETEST(XLog.STD, "THIS SHOULD HAPPEN ONLY IN TESTING, invalid job id [{0}]"),;
Added: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/hcat/MetaDataClientWrapper.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/hcat/MetaDataClientWrapper.java?rev=1411608&view=auto
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/hcat/MetaDataClientWrapper.java (added)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/hcat/MetaDataClientWrapper.java Tue Nov 20 10:11:47 2012
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.hcat;
+
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hcatalog.api.HCatClient;
+import org.apache.hcatalog.api.HCatPartition;
+import org.apache.oozie.ErrorCode;
+import org.apache.oozie.service.MetaDataAccessorException;
+import org.apache.oozie.service.MetaDataAccessorService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.util.HCatURI;
+
+/**
+ * This class is a wrapper around the HCatalog client class
+ */
+public class MetaDataClientWrapper {
+
+ /**
+ * Query one partition.
+ *
+ * @param server : server end point
+ * @param db : database name
+ * @param table : table name
+ * @param partition : Map of partition key-val
+ * @param user :end user
+ * @return : Partition Object
+ * @throws MetaDataAccessorException
+ */
+ public HCatPartition getOnePartition(String server, String db, String table, Map<String, String> partition,
+ String user) throws MetaDataAccessorException {
+ HCatClient client = Services.get().get(MetaDataAccessorService.class).getHCatClient(server, user);
+ HCatPartition hPartition;
+ try {
+ hPartition = client.getPartition(db, table, partition);
+ }
+ catch (Exception e) {
+ throw new MetaDataAccessorException(ErrorCode.E1504, e);
+ }
+ return hPartition;
+ }
+
+ /**
+ * Query one partition.
+ *
+ * @param hcatURI : hcat URI
+ * @param user : end user
+ * @return Partition Object
+ * @throws MetaDataAccessorException
+ */
+ public HCatPartition getOnePartition(String hcatURI, String user) throws MetaDataAccessorException {
+ HCatURI uri = null;
+ try {
+ uri = new HCatURI(hcatURI);
+ }
+ catch (URISyntaxException e) {
+ throw new MetaDataAccessorException(ErrorCode.E1504, e);
+ }
+ return getOnePartition(uri.getServer(), uri.getDb(), uri.getTable(), uri.getPartitionMap(), user);
+ }
+
+ /**
+ * Query a set of partitions using specific filter.
+ *
+ * @param server : server end point
+ * @param db : database name
+ * @param table : table name
+ * @param filter : key=value strings like SQL where clause
+ * @param user :end user
+ * @return : List of Partition Object
+ * @throws MetaDataAccessorException
+ */
+ public List<HCatPartition> getPartitionsByFilter(String server, String db, String table, String filter, String user)
+ throws MetaDataAccessorException {
+ HCatClient client = Services.get().get(MetaDataAccessorService.class).getHCatClient(server, user);
+ List<HCatPartition> hPartitions;
+ try {
+ hPartitions = client.listPartitionsByFilter(db, table, filter);
+ }
+ catch (Exception e) {
+ throw new MetaDataAccessorException(ErrorCode.E1504, e);
+ }
+ return hPartitions;
+ }
+
+ /**
+ * Query a set of partitions using specific filter.
+ *
+ * @param hcatURI :HCat URI
+ * @param filter :key=value strings like SQL where clause
+ * @param user : end user
+ * @return List of Partition Object
+ * @throws MetaDataAccessorException
+ */
+ public List<HCatPartition> getPartitionsByFilter(String hcatURI, String filter, String user)
+ throws MetaDataAccessorException {
+ HCatURI uri;
+ try {
+ uri = new HCatURI(hcatURI);
+ }
+ catch (URISyntaxException e) {
+ throw new MetaDataAccessorException(ErrorCode.E1504, e);
+ }
+ return getPartitionsByFilter(uri.getServer(), uri.getDb(), uri.getTable(), filter, user);
+ }
+
+ /**
+ * Delete one partition.
+ *
+ * @param server : server end point
+ * @param db : database name
+ * @param table : table name
+ * @param partition : Map of partition key-val
+ * @param ifExists
+ * @param user :end user
+ * @return : Partition Object
+ * @throws MetaDataAccessorException
+ */
+ public void dropOnePartition(String server, String db, String table, Map<String, String> partition,
+ boolean ifExists, String user) throws MetaDataAccessorException {
+ HCatClient client = Services.get().get(MetaDataAccessorService.class).getHCatClient(server, user);
+ try {
+ client.dropPartition(db, table, partition, ifExists);
+ }
+ catch (Exception e) {
+ throw new MetaDataAccessorException(ErrorCode.E1504, e);
+ }
+ return;
+ }
+
+ /**
+ * Delete one partition.
+ *
+ * @param hcatURI :HCat URI
+ * @param ifExists
+ * @param user : end user id
+ * @throws MetaDataAccessorException
+ */
+ public void dropOnePartition(String hcatURI, boolean ifExists, String user) throws MetaDataAccessorException {
+ HCatURI uri;
+ try {
+ uri = new HCatURI(hcatURI);
+ }
+ catch (Exception e) {
+ throw new MetaDataAccessorException(ErrorCode.E1504, e);
+ }
+ dropOnePartition(uri.getServer(), uri.getDb(), uri.getTable(), uri.getPartitionMap(), ifExists, user);
+ return;
+ }
+
+}
Added: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorException.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorException.java?rev=1411608&view=auto
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorException.java (added)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorException.java Tue Nov 20 10:11:47 2012
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.service;
+
+import org.apache.oozie.ErrorCode;
+import org.apache.oozie.XException;
+
+public class MetaDataAccessorException extends XException {
+
+ /**
+ * Create an MetaDataAccessor Exception exception from a XException.
+ *
+ * @param cause the XException cause.
+ */
+ public MetaDataAccessorException(XException cause) {
+ super(cause);
+ }
+
+ /**
+ * Create a MetaDataAccessor exception.
+ *
+ * @param errorCode error code.
+ * @param params parameters for the error code message template.
+ */
+ public MetaDataAccessorException(ErrorCode errorCode, Object... params) {
+ super(errorCode, params);
+ }
+}
Added: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorService.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorService.java?rev=1411608&view=auto
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorService.java (added)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/MetaDataAccessorService.java Tue Nov 20 10:11:47 2012
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.service;
+
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hcatalog.api.HCatClient;
+import org.apache.oozie.ErrorCode;
+import org.apache.oozie.util.XLog;
+
+/**
+ * This service provides a way of getting HCatClient instance
+ */
+public class MetaDataAccessorService implements Service {
+
+ public static final String HIVE_METASTORE_URIS = "hive.metastore.uris";
+ private static XLog log;
+
+ @Override
+ public void init(Services services) throws ServiceException {
+ init(services.getConf());
+ }
+
+ private void init(Configuration conf) {
+ log = XLog.getLog(getClass());
+ }
+
+ /**
+ * Get an HCatClient object using doAs for end user
+ *
+ * @param server : server end point
+ * @param user : end user id
+ * @return : HCatClient
+ * @throws Exception
+ */
+ public HCatClient getHCatClient(String server, String user) throws MetaDataAccessorException {
+ final HiveConf conf = new HiveConf();
+ updateConf(conf, server);
+ HCatClient client = null;
+ try {
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
+ log.info("Create HCatClient for user [{0}] login_user [{1}] and server [{1}] ", user,
+ UserGroupInformation.getLoginUser(), server);
+ client = ugi.doAs(new PrivilegedExceptionAction<HCatClient>() {
+ public HCatClient run() throws Exception {
+ return HCatClient.create(conf);
+ }
+ });
+ }
+ catch (Exception e) {
+ throw new MetaDataAccessorException(ErrorCode.E1504, e);
+ }
+ return client;
+ }
+
+ private void updateConf(Configuration conf, String server) {
+ conf.set(HIVE_METASTORE_URIS, server);
+ // TODO: add more conf?
+ }
+
+ @Override
+ public void destroy() {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public Class<? extends Service> getInterface() {
+ return MetaDataAccessorService.class;
+ }
+
+}
Added: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java?rev=1411608&view=auto
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java (added)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java Tue Nov 20 10:11:47 2012
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.hcat;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hcatalog.api.HCatAddPartitionDesc;
+import org.apache.hcatalog.api.HCatClient;
+import org.apache.hcatalog.api.HCatCreateDBDesc;
+import org.apache.hcatalog.api.HCatCreateTableDesc;
+import org.apache.hcatalog.api.HCatPartition;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.data.schema.HCatFieldSchema.Type;
+import org.apache.oozie.service.MetaDataAccessorService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.test.XDataTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+public class TestMetaDataClientWrapper extends XDataTestCase {
+ private Services services;
+ private String server = "local";
+ // private String server = "thrift://localhost:11002"; // to specify the
+ // non-local endpoint.
+ private String isLocal = "true"; // false for non-local instance
+
+ @Before
+ protected void setUp() throws Exception {
+ super.setUp();
+ setSystemProperty("hive.metastore.local", isLocal);
+ services = new Services();
+ addServiceToRun(services.getConf(), MetaDataAccessorService.class.getName());
+ services.init();
+ }
+
+ @After
+ protected void tearDown() throws Exception {
+ services.destroy();
+ super.tearDown();
+ }
+
+ /**
+ * Test if query of one specific partition works. Also check the -ve case,
+ * if there is not such partition
+ *
+ * @throws Exception
+ */
+ public void testGetOnePartition() throws Exception {
+ MetaDataClientWrapper clWr = new MetaDataClientWrapper();
+
+ String db = "default";
+ String table = "tablename";
+
+ populateTable(server, db, table);
+ // +ve test
+ Map<String, String> existPartitions = new HashMap<String, String>();
+ existPartitions.put("dt", "04/12/2012"); // The same value added in
+ // addRecords methods
+ existPartitions.put("country", "brazil");
+
+ HCatPartition part = clWr.getOnePartition(server, db, table, existPartitions, getTestUser());
+ assertNotNull(part);
+ assertEquals(part.getDatabaseName(), db);
+ assertEquals(part.getTableName(), table);
+ assertTrue(part.getValues().size() == 2);
+
+ // -ve test
+ Map<String, String> nonexistPartitions = new HashMap<String, String>();
+ nonexistPartitions.put("dt", "04/01/2012"); // The same value added in
+ // addRecords methods
+ nonexistPartitions.put("country", "brazil2");
+ try {
+ clWr.getOnePartition(server, db, table, nonexistPartitions, getTestUser());
+ fail("Should throw exception earlier.");
+ }
+ catch (Exception ex) {
+ // Expected
+ }
+ }
+
+ /**
+ * Test the query a list of partition based on specific filter.
+ *
+ * @throws Exception
+ */
+ public void testGetPartitionsByFilter() throws Exception {
+ MetaDataClientWrapper clWr = new MetaDataClientWrapper();
+ String db = "default";
+ String table = "tablename";
+
+ populateTable(server, db, table);
+
+ String filter = "dt = '04/12/2012' AND country = 'brazil'";
+
+ List<HCatPartition> partList = clWr.getPartitionsByFilter(server, db, table, filter, getTestUser());
+ assertNotNull(partList);
+ assertTrue(partList.size() == 1);
+
+ filter = "country = 'brazil'";
+ partList = clWr.getPartitionsByFilter(server, db, table, filter, getTestUser());
+ assertNotNull(partList);
+ assertTrue(partList.size() == 2);
+
+ filter = "country = 'mexico'";
+ partList = clWr.getPartitionsByFilter(server, db, table, filter, getTestUser());
+ assertNotNull(partList);
+ assertTrue(partList.size() == 0);
+ }
+
+ /**
+ * Remove an existing filter.
+ *
+ * @throws Exception
+ */
+ public void testDropOnePartition() throws Exception {
+ MetaDataClientWrapper clWr = new MetaDataClientWrapper();
+ String db = "default";
+ String table = "tablename";
+
+ populateTable(server, db, table);
+ // +ve test
+ Map<String, String> existPartitions = new HashMap<String, String>();
+ existPartitions.put("dt", "04/12/2012"); // The same value added in
+ // addRecords methods
+ existPartitions.put("country", "brazil");
+
+ HCatPartition part = clWr.getOnePartition(server, db, table, existPartitions, getTestUser());
+ assertNotNull(part);
+ assertEquals(part.getDatabaseName(), db);
+ assertEquals(part.getTableName(), table);
+ assertTrue(part.getValues().size() == 2);
+
+ clWr.dropOnePartition(server, db, table, existPartitions, true, getTestUser());
+ // -ve test
+ try {
+ HCatPartition part2 = clWr.getOnePartition(server, db, table, existPartitions, getTestUser());
+ fail("Should throw exception earlier.");
+ }
+ catch (Exception ex) {
+ // Expected
+ }
+ }
+
+ private void populateTable(String server, String db, String table) throws Exception {
+ createTable(server, db, table);
+ addRecords(server, db, table);
+
+ }
+
+ private void createTable(String server, String db, String tableName) throws Exception {
+ HCatClient client = services.get(MetaDataAccessorService.class).getHCatClient(server, getTestUser());
+ assertNotNull(client);
+ // Creating a table
+ HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(db).ifNotExists(true).build();
+ client.createDatabase(dbDesc);
+ ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
+ cols.add(new HCatFieldSchema("userid", Type.INT, "id columns"));
+ cols.add(new HCatFieldSchema("viewtime", Type.BIGINT, "view time columns"));
+ cols.add(new HCatFieldSchema("pageurl", Type.STRING, ""));
+ cols.add(new HCatFieldSchema("ip", Type.STRING, "IP Address of the User"));
+ ArrayList<HCatFieldSchema> ptnCols = new ArrayList<HCatFieldSchema>();
+ ptnCols.add(new HCatFieldSchema("dt", Type.STRING, "date column"));
+ ptnCols.add(new HCatFieldSchema("country", Type.STRING, "country column"));
+ HCatCreateTableDesc tableDesc = HCatCreateTableDesc.create(db, tableName, cols).fileFormat("sequencefile")
+ .partCols(ptnCols).build();
+ client.dropTable(db, tableName, true);
+ client.createTable(tableDesc);
+ List<String> tables = client.listTableNamesByPattern(db, "*");
+ assertTrue(tables.size() > 0);
+ assertTrue(tables.contains(tableName));
+ List<String> dbNames = client.listDatabaseNamesByPattern(db);
+ assertTrue(dbNames.size() == 1);
+ assertTrue(dbNames.contains(db));
+ }
+
+ private void addRecords(String server, String dbName, String tableName) throws Exception {
+ HCatClient client = services.get(MetaDataAccessorService.class).getHCatClient(server, getTestUser());
+ Map<String, String> firstPtn = new HashMap<String, String>();
+ firstPtn.put("dt", "04/30/2012");
+ firstPtn.put("country", "usa");
+ HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(dbName, tableName, null, firstPtn).build();
+ client.addPartition(addPtn);
+
+ Map<String, String> secondPtn = new HashMap<String, String>();
+ secondPtn.put("dt", "04/12/2012");
+ secondPtn.put("country", "brazil");
+ HCatAddPartitionDesc addPtn2 = HCatAddPartitionDesc.create(dbName, tableName, null, secondPtn).build();
+ client.addPartition(addPtn2);
+
+ Map<String, String> thirdPtn = new HashMap<String, String>();
+ thirdPtn.put("dt", "04/13/2012");
+ thirdPtn.put("country", "brazil");
+ HCatAddPartitionDesc addPtn3 = HCatAddPartitionDesc.create(dbName, tableName, null, thirdPtn).build();
+ client.addPartition(addPtn3);
+ }
+}
Added: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestMetaDataAccessorService.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestMetaDataAccessorService.java?rev=1411608&view=auto
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestMetaDataAccessorService.java (added)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestMetaDataAccessorService.java Tue Nov 20 10:11:47 2012
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hcatalog.api.HCatClient;
+import org.apache.hcatalog.api.HCatCreateTableDesc;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.data.schema.HCatFieldSchema.Type;
+import org.apache.oozie.test.XDataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestMetaDataAccessorService extends XDataTestCase {
+ @Before
+ protected void setUp() throws Exception {
+ super.setUp();
+ setSystemProperty("hive.metastore.local", "true");
+ Services services = new Services();
+ addServiceToRun(services.getConf(), MetaDataAccessorService.class.getName());
+ services.init();
+ }
+
+ @After
+ protected void tearDown() throws Exception {
+ Services.get().destroy();
+ super.tearDown();
+ }
+
+ /**
+ * Test basic service startup
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testBasicService() throws Exception {
+ Services services = Services.get();
+ MetaDataAccessorService hcs = services.get(MetaDataAccessorService.class);
+ assertNotNull(hcs);
+ }
+
+ /**
+ * Test successful HCatClient connection and invocations
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testHCatClientSuccess() throws Exception {
+ Services services = Services.get();
+ MetaDataAccessorService hcs = services.get(MetaDataAccessorService.class);
+ // String server = "thrift://localhost:11002";
+ String server = "local";
+ String db = "default";
+ String tableName = "mytable";
+ HCatClient client = null;
+ try {
+ client = hcs.getHCatClient(server, getTestUser());
+ assertNotNull(client);
+ // Creating a table
+ ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
+ cols.add(new HCatFieldSchema("id", Type.INT, "id columns"));
+ cols.add(new HCatFieldSchema("value", Type.STRING, "id columns"));
+ client.dropTable(db, tableName, true);
+ HCatCreateTableDesc tableDesc = HCatCreateTableDesc.create(null, tableName, cols).fileFormat("rcfile")
+ .build();
+ client.createTable(tableDesc);
+ List<String> tables = client.listTableNamesByPattern(db, "*");
+ assertTrue(tables.size() > 0);
+ assertTrue(tables.contains(tableName));
+ List<String> dbNames = client.listDatabaseNamesByPattern(db);
+ assertTrue(dbNames.size() == 1);
+ assertTrue(dbNames.get(0).equals(db));
+ // Dropping the table
+ client.dropTable(db, tableName, true);
+ }
+ finally {
+ if (client != null) {
+ client.close();
+ }
+ }
+ }
+
+}
Modified: oozie/branches/hcat-intre/pom.xml
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/pom.xml?rev=1411608&r1=1411607&r2=1411608&view=diff
==============================================================================
--- oozie/branches/hcat-intre/pom.xml (original)
+++ oozie/branches/hcat-intre/pom.xml Tue Nov 20 10:11:47 2012
@@ -80,6 +80,7 @@
<!-- Sharelib component versions -->
<hive.version>0.9.0</hive.version>
<pig.version>0.9.0</pig.version>
+ <webhcat.version>0.5.0-SNAPSHOT</webhcat.version>
<sqoop.version>1.5.0-incubating-SNAPSHOT</sqoop.version>
<streaming.version>${hadoop.version}</streaming.version>
<distcp.version>${hadooplib.version}</distcp.version>
@@ -627,6 +628,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.hcatalog</groupId>
+ <artifactId>webhcat-java-client</artifactId>
+ <version>${webhcat.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.1</version>