You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/11/07 21:41:45 UTC
svn commit: r1637444 [2/20] - in /hive/branches/spark: ./
cli/src/test/org/apache/hadoop/hive/cli/ common/
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/hadoop/hive/conf/
common/src/test/org/apache/hadoop/hive/conf/ com...
Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java (original)
+++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java Fri Nov 7 20:41:34 2014
@@ -480,7 +480,9 @@ class PigHCatUtil {
Map<String, Object> result = new HashMap<String, Object>();
for (Entry<?, ?> entry : map.entrySet()) {
// since map key for Pig has to be Strings
- result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0)));
+ if (entry.getKey()!=null) {
+ result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0)));
+ }
}
return result;
}
Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original)
+++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Fri Nov 7 20:41:34 2014
@@ -101,12 +101,6 @@ public class TestHCatLoader {
private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS =
new HashMap<String, Set<String>>() {{
- put(IOConstants.AVRO, new HashSet<String>() {{
- add("testReadDataBasic");
- add("testReadPartitionedBasic");
- add("testProjectionsBasic");
- add("testSchemaLoadPrimitiveTypes");
- }});
put(IOConstants.PARQUETFILE, new HashSet<String>() {{
add("testReadDataBasic");
add("testReadPartitionedBasic");
Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java (original)
+++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java Fri Nov 7 20:41:34 2014
@@ -18,8 +18,6 @@
*/
package org.apache.hive.hcatalog.pig;
-import com.google.common.collect.ImmutableSet;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -52,7 +50,6 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -76,14 +73,13 @@ public class TestHCatLoaderComplexSchema
private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS =
new HashMap<String, Set<String>>() {{
put(IOConstants.AVRO, new HashSet<String>() {{
- add("testSyntheticComplexSchema");
- add("testTupleInBagInTupleInBag");
- add("testMapWithComplexData");
+ add("testMapNullKey");
}});
put(IOConstants.PARQUETFILE, new HashSet<String>() {{
add("testSyntheticComplexSchema");
add("testTupleInBagInTupleInBag");
add("testMapWithComplexData");
+ add("testMapNullKey");
}});
}};
@@ -223,6 +219,10 @@ public class TestHCatLoaderComplexSchema
private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, boolean provideSchemaToStorer)
throws IOException, CommandNeedRetryException, ExecException, FrontendException {
+ verifyWriteRead(tablename, pigSchema, tableSchema, data, data, provideSchemaToStorer);
+ }
+ private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, List<Tuple> result, boolean provideSchemaToStorer)
+ throws IOException, CommandNeedRetryException, ExecException, FrontendException {
MockLoader.setData(tablename + "Input", data);
try {
createTable(tablename, tableSchema);
@@ -244,7 +244,7 @@ public class TestHCatLoaderComplexSchema
Iterator<Tuple> it = server.openIterator("X");
int i = 0;
while (it.hasNext()) {
- Tuple input = data.get(i++);
+ Tuple input = result.get(i++);
Tuple output = it.next();
compareTuples(input, output);
LOG.info("tuple : {} ", output);
@@ -354,4 +354,40 @@ public class TestHCatLoaderComplexSchema
verifyWriteRead("testMapWithComplexData", pigSchema, tableSchema, data, true);
verifyWriteRead("testMapWithComplexData2", pigSchema, tableSchema, data, false);
}
+
+ /**
+ * artificially complex nested schema to test nested schema conversion
+ * @throws Exception
+ */
+ @Test
+ public void testMapNullKey() throws Exception {
+ assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
+ String pigSchema = "m:map[]";
+
+ String tableSchema = "m map<string, string>";
+
+ List<Tuple> data = new ArrayList<Tuple>();
+ Tuple t = t(
+ new HashMap<String, String>() {
+ {
+ put("ac test1", "test 1");
+ put("ac test2", "test 2");
+ put(null, "test 3");
+ };
+ });
+ data.add(t);
+
+ List<Tuple> result = new ArrayList<Tuple>();
+ t = t(
+ new HashMap<String, String>() {
+ {
+ put("ac test1", "test 1");
+ put("ac test2", "test 2");
+ };
+ });
+ result.add(t);
+
+ verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, true);
+ verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, false);
+ }
}
Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java (original)
+++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java Fri Nov 7 20:41:34 2014
@@ -64,7 +64,7 @@ public class TestHCatLoaderStorer extend
TestHCatLoader.executeStatementOnDriver("create external table " + tblName +
" (my_small_int smallint, my_tiny_int tinyint)" +
" row format delimited fields terminated by '\t' stored as textfile location '" +
- dataDir + "'", driver);
+ dataDir.toURI().getPath() + "'", driver);
TestHCatLoader.dropTable(tblName2, driver);
TestHCatLoader.createTable(tblName2, "my_small_int smallint, my_tiny_int tinyint", null, driver,
"textfile");
Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java (original)
+++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java Fri Nov 7 20:41:34 2014
@@ -76,29 +76,16 @@ public class TestHCatStorer extends HCat
private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS =
new HashMap<String, Set<String>>() {{
put(IOConstants.AVRO, new HashSet<String>() {{
- add("testBagNStruct");
- add("testDateCharTypes");
- add("testDynamicPartitioningMultiPartColsInDataNoSpec");
- add("testDynamicPartitioningMultiPartColsInDataPartialSpec");
- add("testMultiPartColsInData");
- add("testPartColsInData");
- add("testStoreFuncAllSimpleTypes");
- add("testStoreFuncSimple");
- add("testStoreInPartiitonedTbl");
- add("testStoreMultiTables");
- add("testStoreWithNoCtorArgs");
- add("testStoreWithNoSchema");
- add("testWriteChar");
- add("testWriteDate");
- add("testWriteDate2");
- add("testWriteDate3");
- add("testWriteDecimal");
- add("testWriteDecimalX");
- add("testWriteDecimalXY");
- add("testWriteSmallint");
- add("testWriteTimestamp");
- add("testWriteTinyint");
- add("testWriteVarchar");
+ add("testDateCharTypes"); // incorrect precision
+ // expected:<0 xxxxx yyy 5.2[]> but was:<0 xxxxx yyy 5.2[0]>
+ add("testWriteDecimalXY"); // incorrect precision
+ // expected:<1.2[]> but was:<1.2[0]>
+ add("testWriteSmallint"); // doesn't have a notion of small, and saves the full value as an int, so no overflow
+ // expected:<null> but was:<32768>
+ add("testWriteTimestamp"); // does not support timestamp
+ // TypeInfoToSchema.createAvroPrimitive : UnsupportedOperationException
+ add("testWriteTinyint"); // doesn't have a notion of tiny, and saves the full value as an int, so no overflow
+ // expected:<null> but was:<300>
}});
put(IOConstants.PARQUETFILE, new HashSet<String>() {{
add("testBagNStruct");
Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java (original)
+++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java Fri Nov 7 20:41:34 2014
@@ -70,11 +70,6 @@ public class TestHCatStorerMulti {
private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS =
new HashMap<String, Set<String>>() {{
- put(IOConstants.AVRO, new HashSet<String>() {{
- add("testStoreBasicTable");
- add("testStorePartitionedTable");
- add("testStoreTableMulti");
- }});
put(IOConstants.PARQUETFILE, new HashSet<String>() {{
add("testStoreBasicTable");
add("testStorePartitionedTable");
Modified: hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java (original)
+++ hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java Fri Nov 7 20:41:34 2014
@@ -19,10 +19,13 @@
package org.apache.hive.hcatalog.messaging;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hive.hcatalog.messaging.json.JSONMessageFactory;
@@ -131,6 +134,16 @@ public abstract class MessageFactory {
public abstract AddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions);
/**
+ * Factory method for AddPartitionMessage.
+ * @param table The Table to which the partitions are added.
+ * @param partitionSpec The set of Partitions being added.
+ * @return AddPartitionMessage instance.
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract AddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec);
+
+ /**
* Factory method for DropPartitionMessage.
* @param table The Table from which the partition is dropped.
* @param partition The Partition being dropped.
Modified: hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java (original)
+++ hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java Fri Nov 7 20:41:34 2014
@@ -19,9 +19,12 @@
package org.apache.hive.hcatalog.messaging.json;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hive.hcatalog.messaging.AddPartitionMessage;
import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage;
import org.apache.hive.hcatalog.messaging.CreateTableMessage;
@@ -87,6 +90,14 @@ public class JSONMessageFactory extends
}
@Override
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public AddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) {
+ return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), getPartitionKeyValues(table, partitionSpec), System.currentTimeMillis()/1000);
+ }
+
+ @Override
public DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) {
return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(),
partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)),
@@ -107,4 +118,16 @@ public class JSONMessageFactory extends
partitionList.add(getPartitionKeyValues(table, partition));
return partitionList;
}
+
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) {
+ List<Map<String, String>> partitionList = new ArrayList<Map<String, String>>();
+ PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator();
+ while (iterator.hasNext()) {
+ Partition partition = iterator.next();
+ partitionList.add(getPartitionKeyValues(table, partition));
+ }
+ return partitionList;
+ }
}
Modified: hive/branches/spark/hcatalog/webhcat/java-client/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/pom.xml?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/java-client/pom.xml (original)
+++ hive/branches/spark/hcatalog/webhcat/java-client/pom.xml Fri Nov 7 20:41:34 2014
@@ -47,6 +47,13 @@
</dependency>
<!-- test intra-project -->
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hive-hcatalog-core</artifactId>
<version>${project.version}</version>
Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java (original)
+++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java Fri Nov 7 20:41:34 2014
@@ -22,6 +22,8 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hive.hcatalog.common.HCatException;
@@ -213,6 +215,26 @@ public abstract class HCatClient {
public abstract List<HCatPartition> deserializePartitions(List<String> hcatPartitionStringReps) throws HCatException;
/**
+ * Serializer for HCatPartitionSpec.
+ * @param partitionSpec HCatPartitionSpec to be serialized.
+ * @return A list of Strings, representing the HCatPartitionSpec as a whole.
+ * @throws HCatException On failure to serialize.
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract List<String> serializePartitionSpec(HCatPartitionSpec partitionSpec) throws HCatException;
+
+ /**
+ * Deserializer for HCatPartitionSpec.
+ * @param hcatPartitionSpecStrings List of strings, representing the HCatPartitionSpec as a whole.
+ * @return HCatPartitionSpec, reconstructed from the list of strings.
+ * @throws HCatException On failure to deserialize.
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException;
+
+ /**
* Creates the table like an existing table.
*
* @param dbName The name of the database.
@@ -280,6 +302,21 @@ public abstract class HCatClient {
throws HCatException;
/**
+ * Gets partitions in terms of generic HCatPartitionSpec instances.
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, int maxPartitions) throws HCatException;
+
+ /**
+ * Gets partitions in terms of generic HCatPartitionSpec instances.
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, Map<String, String> partitionSelector, int maxPartitions)
+ throws HCatException;
+
+ /**
* Gets the partition.
*
* @param dbName The database name.
@@ -312,6 +349,17 @@ public abstract class HCatClient {
throws HCatException;
/**
+ * Adds partitions using HCatPartitionSpec.
+ * @param partitionSpec The HCatPartitionSpec representing the set of partitions added.
+ * @return The number of partitions added.
+ * @throws HCatException On failure to add partitions.
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract int addPartitionSpec(HCatPartitionSpec partitionSpec)
+ throws HCatException;
+
+ /**
* Drops partition(s) that match the specified (and possibly partial) partition specification.
* A partial partition-specification is one where not all partition-keys have associated values. For example,
* for a table ('myDb.myTable') with 2 partition keys (dt string, region string),
@@ -344,6 +392,14 @@ public abstract class HCatClient {
String filter) throws HCatException;
/**
+ * List partitions by filter, but as HCatPartitionSpecs.
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract HCatPartitionSpec listPartitionSpecsByFilter(String dbName, String tblName,
+ String filter, int maxPartitions) throws HCatException;
+
+ /**
* Mark partition for event.
*
* @param dbName The database name.
Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java (original)
+++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java Fri Nov 7 20:41:34 2014
@@ -25,6 +25,8 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -353,6 +355,31 @@ public class HCatClientHMSImpl extends H
return listPartitionsByFilter(dbName, tblName, getFilterString(partitionSpec));
}
+ @Override
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, int maxPartitions) throws HCatException {
+ try {
+ return new HCatPartitionSpec(getTable(dbName, tableName),
+ hmsClient.listPartitionSpecs(dbName, tableName, maxPartitions));
+ }
+ catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while retrieving partition.", e);
+ } catch (MetaException e) {
+ throw new HCatException(
+ "MetaException while retrieving partition.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while retrieving partition.", e);
+ }
+ }
+
+ @Override
+ public HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, Map<String, String> partitionSelector, int maxPartitions) throws HCatException {
+ return listPartitionSpecsByFilter(dbName, tableName, getFilterString(partitionSelector), maxPartitions);
+ }
+
private static String getFilterString(Map<String, String> partitionSpec) {
final String AND = " AND ";
@@ -413,7 +440,7 @@ public class HCatClientHMSImpl extends H
Table tbl = null;
try {
tbl = hmsClient.getTable(partInfo.getDatabaseName(),
- partInfo.getTableName());
+ partInfo.getTableName());
// TODO: Should be moved out.
if (tbl.getPartitionKeysSize() == 0) {
throw new HCatException("The table " + partInfo.getTableName()
@@ -511,6 +538,28 @@ public class HCatClientHMSImpl extends H
}
@Override
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public HCatPartitionSpec listPartitionSpecsByFilter(String dbName, String tblName, String filter, int maxPartitions)
+ throws HCatException {
+ try {
+ return new HCatPartitionSpec(getTable(dbName, tblName),
+ hmsClient.listPartitionSpecsByFilter(dbName, tblName, filter, maxPartitions));
+ }
+ catch(MetaException e) {
+ throw new HCatException("MetaException while fetching partitions.", e);
+ }
+ catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while fetching partitions.", e);
+ }
+ catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while fetching partitions.", e);
+ }
+ }
+
+ @Override
public void markPartitionForEvent(String dbName, String tblName,
Map<String, String> partKVs, PartitionEventType eventType)
throws HCatException {
@@ -572,7 +621,7 @@ public class HCatClientHMSImpl extends H
String token = null;
try {
token = hmsClient.getDelegationToken(owner,
- renewerKerberosPrincipalName);
+ renewerKerberosPrincipalName);
} catch (MetaException e) {
throw new HCatException(
"MetaException while getting delegation token.", e);
@@ -750,6 +799,30 @@ public class HCatClientHMSImpl extends H
}
@Override
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public int addPartitionSpec(HCatPartitionSpec partitionSpec) throws HCatException {
+
+ try {
+ return hmsClient.add_partitions_pspec(partitionSpec.toPartitionSpecProxy());
+ } catch (InvalidObjectException e) {
+ throw new HCatException(
+ "InvalidObjectException while adding partition.", e);
+ } catch (AlreadyExistsException e) {
+ throw new HCatException(
+ "AlreadyExistsException while adding partition.", e);
+ } catch (MetaException e) {
+ throw new HCatException("MetaException while adding partition.", e);
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException("The table "
+ + "could not be found.", e);
+ } catch (TException e) {
+ throw new ConnectionFailureException(
+ "TException while adding partition.", e);
+ }
+ }
+
+ @Override
public String getMessageBusTopicName(String dbName, String tableName) throws HCatException {
try {
return hmsClient.getTable(dbName, tableName).getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME);
@@ -824,4 +897,16 @@ public class HCatClientHMSImpl extends H
}
return partitions;
}
+
+ @Override
+ public List<String> serializePartitionSpec(HCatPartitionSpec partitionSpec) throws HCatException {
+ return MetadataSerializer.get().serializePartitionSpec(partitionSpec);
+ }
+
+ @Override
+ public HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException {
+ HCatPartitionSpec hcatPartitionSpec = MetadataSerializer.get().deserializePartitionSpec(hcatPartitionSpecStrings);
+ hcatPartitionSpec.hcatTable(getTable(hcatPartitionSpec.getDbName(), hcatPartitionSpec.getTableName()));
+ return hcatPartitionSpec;
+ }
}
Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java (original)
+++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java Fri Nov 7 20:41:34 2014
@@ -1,7 +1,11 @@
package org.apache.hive.hcatalog.api;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hive.hcatalog.common.HCatException;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
@@ -10,6 +14,9 @@ import org.apache.thrift.protocol.TJSONP
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* MetadataSerializer implementation, that serializes HCat API elements into JSON.
*/
@@ -68,4 +75,38 @@ class MetadataJSONSerializer extends Met
throw new HCatException("Could not de-serialize HCatPartition.", exception);
}
}
+
+ @Override
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public List<String> serializePartitionSpec(HCatPartitionSpec hcatPartitionSpec) throws HCatException {
+ try {
+ List<String> stringReps = new ArrayList<String>();
+ TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ for (PartitionSpec partitionSpec : hcatPartitionSpec.partitionSpecProxy.toPartitionSpec()) {
+ stringReps.add(serializer.toString(partitionSpec, "UTF-8"));
+ }
+ return stringReps;
+ }
+ catch (TException serializationException) {
+ throw new HCatException("Failed to serialize!", serializationException);
+ }
+ }
+
+ @Override
+ public HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException {
+ try {
+ List<PartitionSpec> partitionSpecList = new ArrayList<PartitionSpec>();
+ TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory());
+ for (String stringRep : hcatPartitionSpecStrings) {
+ PartitionSpec partSpec = new PartitionSpec();
+ deserializer.deserialize(partSpec, stringRep, "UTF-8");
+ partitionSpecList.add(partSpec);
+ }
+ return new HCatPartitionSpec(null, PartitionSpecProxy.Factory.get(partitionSpecList));
+ }
+ catch (TException deserializationException) {
+ throw new HCatException("Failed to deserialize!", deserializationException);
+ }
+ }
}
Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java (original)
+++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java Fri Nov 7 20:41:34 2014
@@ -1,7 +1,11 @@
package org.apache.hive.hcatalog.api;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hive.hcatalog.common.HCatException;
+import java.util.List;
+
/**
* Interface to serialize HCat API elements.
*/
@@ -51,4 +55,24 @@ abstract class MetadataSerializer {
*/
public abstract HCatPartition deserializePartition(String hcatPartitionStringRep) throws HCatException;
+ /**
+ * Serializer for HCatPartitionSpec.
+ * @param hcatPartitionSpec HCatPartitionSpec instance to be serialized.
+ * @return Serialized string-representations.
+ * @throws HCatException On failure to serialize.
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract List<String> serializePartitionSpec(HCatPartitionSpec hcatPartitionSpec) throws HCatException;
+
+ /**
+ * Deserializer for HCatPartitionSpec string-representations.
+ * @param hcatPartitionSpecStrings List of strings to be converted into an HCatPartitionSpec.
+ * @return Deserialized HCatPartitionSpec instance.
+ * @throws HCatException On failure to deserialize. (e.g. incompatible serialization format, etc.)
+ */
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public abstract HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException;
+
}
Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java (original)
+++ hive/branches/spark/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java Fri Nov 7 20:41:34 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
+import org.apache.hadoop.hive.ql.WindowsPathUtil;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
@@ -107,13 +108,17 @@ public class TestHCatClient {
@BeforeClass
public static void startMetaStoreServer() throws Exception {
+ hcatConf = new HiveConf(TestHCatClient.class);
+ if (Shell.WINDOWS) {
+ WindowsPathUtil.convertPathsFromWindowsToHdfs(hcatConf);
+ }
+
Thread t = new Thread(new RunMS(msPort));
t.start();
Thread.sleep(10000);
securityManager = System.getSecurityManager();
System.setSecurityManager(new NoExitSecurityManager());
- hcatConf = new HiveConf(TestHCatClient.class);
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:"
+ msPort);
hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
@@ -970,7 +975,7 @@ public class TestHCatClient {
sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build());
// The source table now has 2 partitions, one in TEXTFILE, the other in ORC.
- // Test that adding these partitions to the target-table *without* replicating the table-change.
+ // Test adding these partitions to the target-table *without* replicating the table-change.
List<HCatPartition> sourcePartitions = sourceMetaStore.getPartitions(dbName, tableName);
assertEquals("Unexpected number of source partitions.", 2, sourcePartitions.size());
@@ -1002,4 +1007,139 @@ public class TestHCatClient {
assertTrue("Unexpected exception! " + unexpected.getMessage(), false);
}
}
+
+ /**
+ * Test that partition-definitions can be replicated between HCat-instances,
+ * independently of table-metadata replication, using PartitionSpec interfaces.
+ * (This is essentially the same test as testPartitionRegistrationWithCustomSchema(),
+ * transliterated to use the PartitionSpec APIs.)
+ * 2 identical tables are created on 2 different HCat instances ("source" and "target").
+ * On the source instance,
+ * 1. One partition is added with the old format ("TEXTFILE").
+ * 2. The table is updated with an additional column and the data-format changed to ORC.
+ * 3. Another partition is added with the new format.
+ * 4. The partitions' metadata is copied to the target HCat instance, without updating the target table definition.
+ * 5. The partitions' metadata is tested to be an exact replica of that on the source.
+ * @throws Exception
+ */
+ @Test
+ public void testPartitionSpecRegistrationWithCustomSchema() throws Exception {
+ try {
+ startReplicationTargetMetaStoreIfRequired();
+
+ HCatClient sourceMetaStore = HCatClient.create(new Configuration(hcatConf));
+ final String dbName = "myDb";
+ final String tableName = "myTable";
+
+ sourceMetaStore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE);
+
+ sourceMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build());
+ List<HCatFieldSchema> columnSchema = new ArrayList<HCatFieldSchema>(
+ Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""),
+ new HCatFieldSchema("bar", Type.STRING, "")));
+
+ List<HCatFieldSchema> partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""),
+ new HCatFieldSchema("grid", Type.STRING, ""));
+
+ HCatTable sourceTable = new HCatTable(dbName, tableName).cols(columnSchema)
+ .partCols(partitionSchema)
+ .comment("Source table.");
+
+ sourceMetaStore.createTable(HCatCreateTableDesc.create(sourceTable).build());
+
+ // Verify that the sourceTable was created successfully.
+ sourceTable = sourceMetaStore.getTable(dbName, tableName);
+ assertNotNull("Table couldn't be queried for. ", sourceTable);
+
+ // Partitions added now should inherit table-schema, properties, etc.
+ Map<String, String> partitionSpec_1 = new HashMap<String, String>();
+ partitionSpec_1.put("grid", "AB");
+ partitionSpec_1.put("dt", "2011_12_31");
+ HCatPartition sourcePartition_1 = new HCatPartition(sourceTable, partitionSpec_1, "");
+
+ sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_1).build());
+ assertEquals("Unexpected number of partitions. ",
+ sourceMetaStore.getPartitions(dbName, tableName).size(), 1);
+ // Verify that partition_1 was added correctly, and properties were inherited from the HCatTable.
+ HCatPartition addedPartition_1 = sourceMetaStore.getPartition(dbName, tableName, partitionSpec_1);
+ assertEquals("Column schema doesn't match.", addedPartition_1.getColumns(), sourceTable.getCols());
+ assertEquals("InputFormat doesn't match.", addedPartition_1.getInputFormat(), sourceTable.getInputFileFormat());
+ assertEquals("OutputFormat doesn't match.", addedPartition_1.getOutputFormat(), sourceTable.getOutputFileFormat());
+ assertEquals("SerDe doesn't match.", addedPartition_1.getSerDe(), sourceTable.getSerdeLib());
+ assertEquals("SerDe params don't match.", addedPartition_1.getSerdeParams(), sourceTable.getSerdeParams());
+
+ // Replicate table definition.
+
+ HCatClient targetMetaStore = HCatClient.create(new Configuration(replicationTargetHCatConf));
+ targetMetaStore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE);
+
+ targetMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build());
+ // Make a copy of the source-table, as would be done across class-loaders.
+ HCatTable targetTable = targetMetaStore.deserializeTable(sourceMetaStore.serializeTable(sourceTable));
+ targetMetaStore.createTable(HCatCreateTableDesc.create(targetTable).build());
+ targetTable = targetMetaStore.getTable(dbName, tableName);
+
+ assertEquals("Created table doesn't match the source.",
+ targetTable.diff(sourceTable), HCatTable.NO_DIFF);
+
+ // Modify Table schema at the source.
+ List<HCatFieldSchema> newColumnSchema = new ArrayList<HCatFieldSchema>(columnSchema);
+ newColumnSchema.add(new HCatFieldSchema("goo_new", Type.DOUBLE, ""));
+ Map<String, String> tableParams = new HashMap<String, String>(1);
+ tableParams.put("orc.compress", "ZLIB");
+ sourceTable.cols(newColumnSchema) // Add a column.
+ .fileFormat("orcfile") // Change SerDe, File I/O formats.
+ .tblProps(tableParams)
+ .serdeParam(serdeConstants.FIELD_DELIM, Character.toString('\001'));
+ sourceMetaStore.updateTableSchema(dbName, tableName, sourceTable);
+ sourceTable = sourceMetaStore.getTable(dbName, tableName);
+
+ // Add another partition to the source.
+ Map<String, String> partitionSpec_2 = new HashMap<String, String>();
+ partitionSpec_2.put("grid", "AB");
+ partitionSpec_2.put("dt", "2012_01_01");
+ HCatPartition sourcePartition_2 = new HCatPartition(sourceTable, partitionSpec_2, "");
+ sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build());
+
+ // The source table now has 2 partitions, one in TEXTFILE, the other in ORC.
+ // Test adding these partitions to the target-table *without* replicating the table-change.
+
+ HCatPartitionSpec sourcePartitionSpec = sourceMetaStore.getPartitionSpecs(dbName, tableName, -1);
+ assertEquals("Unexpected number of source partitions.", 2, sourcePartitionSpec.size());
+
+ // Serialize the hcatPartitionSpec.
+ List<String> partitionSpecString = sourceMetaStore.serializePartitionSpec(sourcePartitionSpec);
+
+ // Deserialize the HCatPartitionSpec using the target HCatClient instance.
+ HCatPartitionSpec targetPartitionSpec = targetMetaStore.deserializePartitionSpec(partitionSpecString);
+ assertEquals("Could not add the expected number of partitions.",
+ sourcePartitionSpec.size(), targetMetaStore.addPartitionSpec(targetPartitionSpec));
+
+ // Retrieve partitions.
+ targetPartitionSpec = targetMetaStore.getPartitionSpecs(dbName, tableName, -1);
+ assertEquals("Could not retrieve the expected number of partitions.",
+ sourcePartitionSpec.size(), targetPartitionSpec.size());
+
+ // Assert that the source and target partitions are equivalent.
+ HCatPartitionSpec.HCatPartitionIterator sourceIterator = sourcePartitionSpec.getPartitionIterator();
+ HCatPartitionSpec.HCatPartitionIterator targetIterator = targetPartitionSpec.getPartitionIterator();
+
+ while (targetIterator.hasNext()) {
+ assertTrue("Fewer target partitions than source.", sourceIterator.hasNext());
+ HCatPartition sourcePartition = sourceIterator.next();
+ HCatPartition targetPartition = targetIterator.next();
+ assertEquals("Column schema doesn't match.", sourcePartition.getColumns(), targetPartition.getColumns());
+ assertEquals("InputFormat doesn't match.", sourcePartition.getInputFormat(), targetPartition.getInputFormat());
+ assertEquals("OutputFormat doesn't match.", sourcePartition.getOutputFormat(), targetPartition.getOutputFormat());
+ assertEquals("SerDe doesn't match.", sourcePartition.getSerDe(), targetPartition.getSerDe());
+ assertEquals("SerDe params don't match.", sourcePartition.getSerdeParams(), targetPartition.getSerdeParams());
+
+ }
+ }
+ catch (Exception unexpected) {
+ LOG.error( "Unexpected exception! ", unexpected);
+ assertTrue("Unexpected exception! " + unexpected.getMessage(), false);
+ }
+ }
+
}
Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java (original)
+++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java Fri Nov 7 20:41:34 2014
@@ -229,15 +229,17 @@ public class ExecServiceImpl implements
watchdog.checkException();
}
catch (Exception ex) {
- LOG.error("Command: " + cmd + " failed:", ex);
+ LOG.error("Command: " + cmd + " failed. res=" + res, ex);
}
if(watchdog.killedProcess()) {
String msg = " was terminated due to timeout(" + timeout + "ms). See " + AppConfig
.EXEC_TIMEOUT_NAME + " property";
- LOG.warn("Command: " + cmd + msg);
+ LOG.warn("Command: " + cmd + msg + " res=" + res);
res.stderr += " Command " + msg;
}
-
+ if(res.exitcode != 0) {
+ LOG.info("Command: " + cmd + " failed. res=" + res);
+ }
return res;
}
Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java (original)
+++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java Fri Nov 7 20:41:34 2014
@@ -40,7 +40,7 @@ import org.apache.thrift.TException;
/**
* Helper class to run jobs using Kerberos security. Always safe to
- * use these methods, it's a noop if security is not enabled.
+ * use these methods, it's a no-op if security is not enabled.
*/
public class SecureProxySupport {
private Path tokenPath;
@@ -140,6 +140,7 @@ public class SecureProxySupport {
ugi.doAs(new PrivilegedExceptionAction<Object>() {
public Object run() throws IOException {
FileSystem fs = FileSystem.get(conf);
+ //todo: according to JavaDoc this seems like private API: addDelegationToken should be used
twrapper.token = fs.getDelegationToken(ugi.getShortUserName());
return null;
}
Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java (original)
+++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java Fri Nov 7 20:41:34 2014
@@ -83,6 +83,20 @@ public class SqoopDelegator extends Laun
args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
args.add("-D" + TempletonControllerJob.MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER);
}
+ if(i == 0 && TempletonUtils.isset(libdir) && TempletonUtils.isset(appConf.sqoopArchive())) {
+ //http://sqoop.apache.org/docs/1.4.5/SqoopUserGuide.html#_using_generic_and_specific_arguments
+ String libJars = null;
+ for(String s : args) {
+ if(s.startsWith(JobSubmissionConstants.Sqoop.LIB_JARS)) {
+ libJars = s.substring(s.indexOf("=") + 1);
+ break;
+ }
+ }
+ //the jars in libJars will be localized to CWD of the launcher task; then -libjars will
+ //cause them to be localized for the Sqoop MR job tasks
+ args.add(TempletonUtils.quoteForWindows("-libjars"));
+ args.add(TempletonUtils.quoteForWindows(libJars));
+ }
}
} else if (TempletonUtils.isset(optionsFile)) {
args.add("--options-file");
@@ -114,11 +128,13 @@ public class SqoopDelegator extends Laun
/**Sqoop accesses databases via JDBC. This means it needs to have appropriate JDBC
drivers available. Normally, the user would install Sqoop and place these jars
into SQOOP_HOME/lib. When WebHCat is configured to auto-ship the Sqoop tar file, we
- need to make sure that relevant JDBC jars are available on target node.
+ need to make sure that relevant JDBC jars are available on target node but we cannot modify
+ lib/ of exploded tar because Dist Cache intentionally prevents this.
The user is expected to place any JDBC jars into an HDFS directory and specify this
- dir in "libdir" parameter. All the files in this dir will be copied to lib/ of the
- exploded Sqoop tar ball on target node.
+ dir in "libdir" parameter. WebHCat then ensures that these jars are localized for the launcher task
+ and made available to Sqoop.
{@link org.apache.hive.hcatalog.templeton.tool.LaunchMapper#handleSqoop(org.apache.hadoop.conf.Configuration, java.util.Map)}
+ {@link #makeArgs(String, String, String, String, String, boolean, String)}
*/
LOG.debug("libdir=" + libdir);
List<Path> jarList = TempletonUtils.hadoopFsListChildren(libdir, appConf, runAs);
Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Fri Nov 7 20:41:34 2014
@@ -18,7 +18,6 @@
*/
package org.apache.hive.hcatalog.templeton.tool;
-import com.google.common.io.Files;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -101,11 +100,18 @@ public class LaunchMapper extends Mapper
if(TempletonUtils.isset(conf.get(Sqoop.LIB_JARS))) {
//LIB_JARS should only be set if Sqoop is auto-shipped
LOG.debug(Sqoop.LIB_JARS + "=" + conf.get(Sqoop.LIB_JARS));
- //copy these (which have now been localized) jars to sqoop/lib
- String destDir = conf.get(AppConfig.SQOOP_HOME_PATH) + File.separator + "lib";
String[] files = conf.getStrings(Sqoop.LIB_JARS);
+ StringBuilder jdbcJars = new StringBuilder();
for(String f : files) {
- Files.copy(new File(f), new File(destDir + File.separator + f));
+ jdbcJars.append(f).append(File.pathSeparator);
+ }
+ jdbcJars.setLength(jdbcJars.length() - 1);
+ //this makes the jars available to Sqoop client
+ if(TempletonUtils.isset(System.getenv("HADOOP_CLASSPATH"))) {
+ env.put("HADOOP_CLASSPATH", System.getenv("HADOOP_CLASSPATH") + File.pathSeparator + jdbcJars.toString());
+ }
+ else {
+ env.put("HADOOP_CLASSPATH", jdbcJars.toString());
}
}
}
Modified: hive/branches/spark/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java (original)
+++ hive/branches/spark/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java Fri Nov 7 20:41:34 2014
@@ -171,8 +171,8 @@ public class TestJdbcWithMiniKdc {
MiniHiveKdc.HIVE_TEST_USER_2);
} catch (SQLException e) {
// Expected error
- assertTrue(e.getMessage().contains("Failed to validate proxy privilege"));
- assertTrue(e.getCause().getCause().getMessage().contains("Failed to validate proxy privilege"));
+ assertTrue(e.getMessage().contains("Error retrieving delegation token for user"));
+ assertTrue(e.getCause().getCause().getMessage().contains("is not allowed to impersonate"));
} finally {
hs2Conn.close();
}
Modified: hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestPasswordWithCredentialProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestPasswordWithCredentialProvider.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestPasswordWithCredentialProvider.java (original)
+++ hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestPasswordWithCredentialProvider.java Fri Nov 7 20:41:34 2014
@@ -81,7 +81,7 @@ public class TestPasswordWithCredentialP
conf.set("hadoop.security.credential.clear-text-fallback", "true");
// Set up CredentialProvider
- conf.set("hadoop.security.credential.provider.path", "jceks://file/" + tmpDir + "/test.jks");
+ conf.set("hadoop.security.credential.provider.path", "jceks://file/" + tmpDir.toURI().getPath() + "/test.jks");
// CredentialProvider/CredentialProviderFactory may not exist, depending on the version of
// hadoop-2 being used to build Hive. Use reflection to do the following lines
Modified: hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java (original)
+++ hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java Fri Nov 7 20:41:34 2014
@@ -21,14 +21,19 @@ package org.apache.hive.jdbc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.junit.After;
import org.junit.Before;
@@ -37,6 +42,26 @@ import org.junit.Test;
public class TestSchedulerQueue {
+ // hadoop group mapping that maps user to same group
+ public static class HiveTestSimpleGroupMapping implements GroupMappingServiceProvider {
+ public static String primaryTag = "";
+ @Override
+ public List<String> getGroups(String user) throws IOException {
+ List<String> results = new ArrayList<String>();
+ results.add(user + primaryTag);
+ results.add(user + "-group");
+ return results;
+ }
+
+ @Override
+ public void cacheGroupsRefresh() throws IOException {
+ }
+
+ @Override
+ public void cacheGroupsAdd(List<String> groups) throws IOException {
+ }
+ }
+
private MiniHS2 miniHS2 = null;
private static HiveConf conf = new HiveConf();
private Connection hs2Conn = null;
@@ -44,6 +69,8 @@ public class TestSchedulerQueue {
@BeforeClass
public static void beforeTest() throws Exception {
Class.forName(MiniHS2.getJdbcDriverName());
+ conf.set("hadoop.security.group.mapping",
+ HiveTestSimpleGroupMapping.class.getName());
}
@Before
@@ -56,6 +83,7 @@ public class TestSchedulerQueue {
miniHS2.setConfProperty(YarnConfiguration.RM_SCHEDULER,
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
miniHS2.start(new HashMap<String, String>());
+ HiveTestSimpleGroupMapping.primaryTag = "";
}
@After
@@ -79,6 +107,7 @@ public class TestSchedulerQueue {
@Test
public void testFairSchedulerQueueMapping() throws Exception {
hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1", "bar");
+ verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false");
verifyProperty("mapreduce.framework.name", "yarn");
verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname,
"true");
@@ -88,6 +117,31 @@ public class TestSchedulerQueue {
}
/**
+ * Verify:
+ * Test is running with MR2 and queue mapping are set correctly for primary group rule.
+ * @throws Exception
+ */
+ @Test
+ public void testFairSchedulerPrimaryQueueMapping() throws Exception {
+ miniHS2.setConfProperty(FairSchedulerConfiguration.ALLOCATION_FILE, "fair-scheduler-test.xml");
+ HiveTestSimpleGroupMapping.primaryTag = "-test";
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user2", "bar");
+ verifyProperty("mapreduce.job.queuename", "root.user2" + HiveTestSimpleGroupMapping.primaryTag);
+ }
+
+ /**
+ * Verify:
+ * Test is running with MR2 and queue mapping are set correctly for primary group rule.
+ * @throws Exception
+ */
+ @Test
+ public void testFairSchedulerSecondaryQueueMapping() throws Exception {
+ miniHS2.setConfProperty(FairSchedulerConfiguration.ALLOCATION_FILE, "fair-scheduler-test.xml");
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user3", "bar");
+ verifyProperty("mapreduce.job.queuename", "root.user3-group");
+ }
+
+ /**
* Verify that the queue refresh doesn't happen when configured to be off.
*
* @throws Exception
Modified: hive/branches/spark/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java (original)
+++ hive/branches/spark/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java Fri Nov 7 20:41:34 2014
@@ -106,7 +106,7 @@ public abstract class AbstractHiveServic
}
// get service host
- protected String getHost() {
+ public String getHost() {
return hostname;
}
@@ -127,12 +127,12 @@ public abstract class AbstractHiveServic
}
// Get binary service port #
- protected int getBinaryPort() {
+ public int getBinaryPort() {
return binaryPort;
}
// Get http service port #
- protected int getHttpPort() {
+ public int getHttpPort() {
return httpPort;
}
Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java (original)
+++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java Fri Nov 7 20:41:34 2014
@@ -156,7 +156,7 @@ public abstract class TestAuthorizationA
FunctionInvoker invoker = new FunctionInvoker() {
@Override
public void invoke() throws Exception {
- msc.create_role(new Role());
+ msc.create_role(new Role("role1", 0, "owner"));
}
};
testFunction(invoker);
Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java (original)
+++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java Fri Nov 7 20:41:34 2014
@@ -34,8 +34,10 @@ import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener;
import org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.WindowsPathUtil;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -55,7 +57,11 @@ public class StorageBasedMetastoreTestBa
}
protected HiveConf createHiveConf() throws Exception {
- return new HiveConf(this.getClass());
+ HiveConf conf = new HiveConf(this.getClass());
+ if (Shell.WINDOWS) {
+ WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
+ }
+ return conf;
}
@Before
@@ -71,9 +77,8 @@ public class StorageBasedMetastoreTestBa
System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname,
InjectableDummyAuthenticator.class.getName());
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
-
clientHiveConf = createHiveConf();
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), clientHiveConf);
// Turn off client-side authorization
clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false);
Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java (original)
+++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java Fri Nov 7 20:41:34 2014
@@ -18,10 +18,15 @@
package org.apache.hadoop.hive.ql.security;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
import org.junit.Assert;
import org.junit.Test;
@@ -30,6 +35,41 @@ import org.junit.Test;
*/
public class TestStorageBasedMetastoreAuthorizationDrops extends StorageBasedMetastoreTestBase {
+ protected static MiniDFSShim dfs = null;
+
+ @Override
+ protected HiveConf createHiveConf() throws Exception {
+ // Hadoop FS ACLs do not work with LocalFileSystem, so set up MiniDFS.
+ HiveConf conf = super.createHiveConf();
+
+ String currentUserName = ShimLoader.getHadoopShims().getUGIForConf(conf).getShortUserName();
+ conf.set("hadoop.proxyuser." + currentUserName + ".groups", "*");
+ conf.set("hadoop.proxyuser." + currentUserName + ".hosts", "*");
+ dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
+ FileSystem fs = dfs.getFileSystem();
+
+ Path warehouseDir = new Path(new Path(fs.getUri()), "/warehouse");
+ fs.mkdirs(warehouseDir);
+ conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehouseDir.toString());
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true);
+
+ // Set up scratch directory
+ Path scratchDir = new Path(new Path(fs.getUri()), "/scratchdir");
+ conf.setVar(HiveConf.ConfVars.SCRATCHDIR, scratchDir.toString());
+
+ return conf;
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+
+ if (dfs != null) {
+ dfs.shutdown();
+ dfs = null;
+ }
+ }
+
@Test
public void testDropDatabase() throws Exception {
dropDatabaseByOtherUser("-rwxrwxrwx", 0);
Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java (original)
+++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java Fri Nov 7 20:41:34 2014
@@ -140,7 +140,7 @@ public class TestCompactor {
executeStatementOnDriver("CREATE EXTERNAL TABLE " + tblNameStg + "(a INT, b STRING)" +
" ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n'" +
" STORED AS TEXTFILE" +
- " LOCATION '" + stagingFolder.newFolder() + "'", driver);
+ " LOCATION '" + stagingFolder.newFolder().toURI().getPath() + "'", driver);
executeStatementOnDriver("load data local inpath '" + BASIC_FILE_NAME +
"' overwrite into table " + tblNameStg, driver);
Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java (original)
+++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java Fri Nov 7 20:41:34 2014
@@ -43,9 +43,6 @@ public class TestZooKeeperTokenStore ext
private CuratorFramework zkClient = null;
private int zkPort = -1;
private ZooKeeperTokenStore ts;
- // connect timeout large enough for slower test environments
- private final int connectTimeoutMillis = 30000;
- private final int sessionTimeoutMillis = 3000;
@Override
protected void setUp() throws Exception {
@@ -55,10 +52,9 @@ public class TestZooKeeperTokenStore ext
}
this.zkCluster = new MiniZooKeeperCluster();
this.zkPort = this.zkCluster.startup(zkDataDir);
-
- this.zkClient = CuratorFrameworkFactory.builder().connectString("localhost:" + zkPort)
- .sessionTimeoutMs(sessionTimeoutMillis).connectionTimeoutMs(connectTimeoutMillis)
- .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
+ this.zkClient =
+ CuratorFrameworkFactory.builder().connectString("localhost:" + zkPort)
+ .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
this.zkClient.start();
}
@@ -74,15 +70,9 @@ public class TestZooKeeperTokenStore ext
private Configuration createConf(String zkPath) {
Configuration conf = new Configuration();
- conf.set(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR,
- "localhost:" + this.zkPort);
- conf.set(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE,
- zkPath);
- conf.setLong(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS,
- connectTimeoutMillis);
+ conf.set(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, "localhost:"
+ + this.zkPort);
+ conf.set(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE, zkPath);
return conf;
}
Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Fri Nov 7 20:41:34 2014
@@ -1805,6 +1805,7 @@ public void testParseUrlHttpMode() throw
ResultSet rs = stmt.executeQuery("SELECT 1 AS a, 2 AS a from " + tableName);
assertTrue(rs.next());
assertEquals(1, rs.getInt("a"));
+ rs.close();
}
Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java (original)
+++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java Fri Nov 7 20:41:34 2014
@@ -31,15 +31,21 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestSSL {
+ private static final Logger LOG = LoggerFactory.getLogger(TestSSL.class);
private static final String KEY_STORE_NAME = "keystore.jks";
private static final String TRUST_STORE_NAME = "truststore.jks";
private static final String KEY_STORE_PASSWORD = "HiveJdbc";
@@ -87,6 +93,73 @@ public class TestSSL {
System.clearProperty(JAVA_TRUST_STORE_PASS_PROP);
}
+ private int execCommand(String cmd) throws Exception {
+ int exitCode;
+ try {
+ String output = Shell.execCommand("bash", "-c", cmd);
+ LOG.info("Output from '" + cmd + "': " + output) ;
+ exitCode = 0;
+ } catch (Shell.ExitCodeException e) {
+ exitCode = e.getExitCode();
+ LOG.info("Error executing '" + cmd + "', exitCode = " + exitCode, e);
+ }
+ return exitCode;
+ }
+
+ /***
+ * Tests to ensure SSLv2 and SSLv3 are disabled
+ */
+ @Test
+ public void testSSLVersion() throws Exception {
+ Assume.assumeTrue(execCommand("which openssl") == 0); // we need openssl
+ Assume.assumeTrue(System.getProperty("os.name").toLowerCase()
+ .contains("linux")); // we depend on linux openssl exit codes
+
+ setSslConfOverlay(confOverlay);
+ // Test in binary mode
+ setBinaryConfOverlay(confOverlay);
+ // Start HS2 with SSL
+ miniHS2.start(confOverlay);
+
+ // make SSL connection
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL() + ";ssl=true;sslTrustStore=" +
+ dataFileDir + File.separator + TRUST_STORE_NAME + ";trustStorePassword=" +
+ KEY_STORE_PASSWORD, System.getProperty("user.name"), "bar");
+ hs2Conn.close();
+ Assert.assertEquals("Expected exit code of 1", 1,
+ execCommand("openssl s_client -connect " + miniHS2.getHost() + ":" + miniHS2.getBinaryPort()
+ + " -ssl2 < /dev/null"));
+ Assert.assertEquals("Expected exit code of 1", 1,
+ execCommand("openssl s_client -connect " + miniHS2.getHost() + ":" + miniHS2.getBinaryPort()
+ + " -ssl3 < /dev/null"));
+ miniHS2.stop();
+
+ // Test in http mode
+ setHttpConfOverlay(confOverlay);
+ miniHS2.start(confOverlay);
+ // make SSL connection
+ try {
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL() +
+ ";ssl=true;sslTrustStore=" + dataFileDir + File.separator +
+ TRUST_STORE_NAME + ";trustStorePassword=" + KEY_STORE_PASSWORD +
+ "?hive.server2.transport.mode=" + HS2_HTTP_MODE +
+ ";hive.server2.thrift.http.path=" + HS2_HTTP_ENDPOINT,
+ System.getProperty("user.name"), "bar");
+ Assert.fail("Expected SQLException during connect");
+ } catch (SQLException e) {
+ LOG.info("Expected exception: " + e, e);
+ Assert.assertEquals("08S01", e.getSQLState().trim());
+ Throwable cause = e.getCause();
+ Assert.assertNotNull(cause);
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+ Assert.assertEquals("org.apache.http.NoHttpResponseException", cause.getClass().getName());
+ Assert.assertEquals("The target server failed to respond", cause.getMessage());
+ }
+ miniHS2.stop();
+ }
+
/***
* Test SSL client with non-SSL server fails
* @throws Exception
Modified: hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties Fri Nov 7 20:41:34 2014
@@ -55,7 +55,20 @@ minitez.query.files.shared=alter_merge_2
bucket2.q,\
bucket3.q,\
bucket4.q,\
- cbo_correctness.q,\
+ cbo_gby.q,\
+ cbo_gby_empty.q,\
+ cbo_join.q,\
+ cbo_limit.q,\
+ cbo_semijoin.q,\
+ cbo_simple_select.q,\
+ cbo_stats.q,\
+ cbo_subq_exists.q,\
+ cbo_subq_in.q,\
+ cbo_subq_not_in.q,\
+ cbo_udf_udaf.q,\
+ cbo_union.q,\
+ cbo_views.q,\
+ cbo_windowing.q,\
correlationoptimizer1.q,\
count.q,\
create_merge_compressed.q,\
@@ -161,9 +174,25 @@ minitez.query.files.shared=alter_merge_2
vector_cast_constant.q,\
vector_char_4.q,\
vector_char_simple.q,\
+ vector_coalesce.q,\
vector_count_distinct.q,\
vector_data_types.q,\
+ vector_decimal_1.q,\
+ vector_decimal_10_0.q,\
+ vector_decimal_2.q,\
+ vector_decimal_3.q,\
+ vector_decimal_4.q,\
+ vector_decimal_5.q,\
+ vector_decimal_6.q,\
vector_decimal_aggregate.q,\
+ vector_decimal_cast.q,\
+ vector_decimal_expressions.q,\
+ vector_decimal_mapjoin.q,\
+ vector_decimal_math_funcs.q,\
+ vector_decimal_precision.q,\
+ vector_decimal_trailing.q,\
+ vector_decimal_udf.q,\
+ vector_decimal_udf2.q,\
vector_distinct_2.q,\
vector_elt.q,\
vector_groupby_3.q,\
@@ -196,6 +225,7 @@ minitez.query.files.shared=alter_merge_2
vectorization_9.q,\
vectorization_decimal_date.q,\
vectorization_div0.q,\
+ vectorization_limit.q,\
vectorization_nested_udf.q,\
vectorization_not.q,\
vectorization_part.q,\
@@ -204,7 +234,10 @@ minitez.query.files.shared=alter_merge_2
vectorization_short_regress.q,\
vectorized_bucketmapjoin1.q,\
vectorized_case.q,\
+ vectorized_casts.q,\
vectorized_context.q,\
+ vectorized_date_funcs.q,\
+ vectorized_distinct_gby.q,\
vectorized_mapjoin.q,\
vectorized_math_funcs.q,\
vectorized_nested_mapjoin.q,\
Modified: hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Fri Nov 7 20:41:34 2014
@@ -952,7 +952,7 @@ public class QTestUtil {
for (Task<? extends Serializable> plan : tasks) {
Utilities.serializePlan(plan, ofs, conf);
}
-
+ ofs.close();
fixXml4JDK7(outf.getPath());
maskPatterns(xmlPlanMask, outf.getPath());
@@ -964,6 +964,7 @@ public class QTestUtil {
return exitVal;
} finally {
conf.set(HiveConf.ConfVars.PLAN_SERIALIZATION.varname, "kryo");
+ IOUtils.closeQuietly(ofs);
}
}
Modified: hive/branches/spark/jdbc/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/pom.xml?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/jdbc/pom.xml (original)
+++ hive/branches/spark/jdbc/pom.xml Fri Nov 7 20:41:34 2014
@@ -29,6 +29,7 @@
<properties>
<hive.path.to.root>..</hive.path.to.root>
+ <packaging.minimizeJar>false</packaging.minimizeJar>
</properties>
<dependencies>
@@ -97,6 +98,11 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
</dependencies>
<profiles>
@@ -122,6 +128,12 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>dist</id>
+ <properties>
+ <packaging.minimizeJar>true</packaging.minimizeJar>
+ </properties>
+ </profile>
</profiles>
@@ -138,7 +150,7 @@
<goal>shade</goal>
</goals>
<configuration>
- <minimizeJar>true</minimizeJar>
+ <minimizeJar>${packaging.minimizeJar}</minimizeJar>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>${hive.jdbc.driver.classifier}</shadedClassifierName>
<filters>
@@ -163,6 +175,12 @@
</includes>
</filter>
<filter>
+ <artifact>commons-logging:commons-logging</artifact>
+ <includes>
+ <include>**</include>
+ </includes>
+ </filter>
+ <filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
Modified: hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java (original)
+++ hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java Fri Nov 7 20:41:34 2014
@@ -99,6 +99,7 @@ public class HiveConnection implements j
private JdbcConnectionParams connParams;
private final boolean isEmbeddedMode;
private TTransport transport;
+ private boolean assumeSubject;
// TODO should be replaced by CliServiceClient
private TCLIService.Iface client;
private boolean isClosed = true;
@@ -177,6 +178,9 @@ public class HiveConnection implements j
private void openTransport() throws SQLException {
while (true) {
try {
+ assumeSubject =
+ JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT.equals(sessConfMap
+ .get(JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE));
transport = isHttpTransportMode() ? createHttpTransport() : createBinaryTransport();
if (!transport.isOpen()) {
LOG.info("Will try to open client transport with JDBC Uri: " + jdbcUriString);
@@ -265,8 +269,9 @@ public class HiveConnection implements j
* In https mode, the entire information is encrypted
* TODO: Optimize this with a mix of kerberos + using cookie.
*/
- requestInterceptor = new HttpKerberosRequestInterceptor(
- sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL), host, getServerHttpUrl(false));
+ requestInterceptor =
+ new HttpKerberosRequestInterceptor(sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL),
+ host, getServerHttpUrl(useSsl), assumeSubject);
}
else {
/**
@@ -351,8 +356,6 @@ public class HiveConnection implements j
}
saslProps.put(Sasl.QOP, saslQOP.toString());
saslProps.put(Sasl.SERVER_AUTH, "true");
- boolean assumeSubject = JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT.equals(sessConfMap
- .get(JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE));
transport = KerberosSaslHelper.getKerberosTransport(
sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL), host,
HiveAuthFactory.getSocketTransport(host, port, loginTimeout), saslProps,
Modified: hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java (original)
+++ hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java Fri Nov 7 20:41:34 2014
@@ -39,15 +39,17 @@ public class HttpKerberosRequestIntercep
String principal;
String host;
String serverHttpUrl;
+ boolean assumeSubject;
// A fair reentrant lock
private static ReentrantLock kerberosLock = new ReentrantLock(true);
public HttpKerberosRequestInterceptor(String principal, String host,
- String serverHttpUrl) {
+ String serverHttpUrl, boolean assumeSubject) {
this.principal = principal;
this.host = host;
this.serverHttpUrl = serverHttpUrl;
+ this.assumeSubject = assumeSubject;
}
@Override
@@ -59,7 +61,7 @@ public class HttpKerberosRequestIntercep
// Locking ensures the tokens are unique in case of concurrent requests
kerberosLock.lock();
kerberosAuthHeader = HttpAuthUtils.getKerberosServiceTicket(
- principal, host, serverHttpUrl);
+ principal, host, serverHttpUrl, assumeSubject);
// Set the session key token (Base64 encoded) in the headers
httpRequest.addHeader(HttpAuthUtils.AUTHORIZATION + ": " +
HttpAuthUtils.NEGOTIATE + " ", kerberosAuthHeader);
Modified: hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/Utils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/Utils.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/Utils.java (original)
+++ hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/Utils.java Fri Nov 7 20:41:34 2014
@@ -100,8 +100,6 @@ public class Utils {
static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2";
// Non-configurable params:
- // ZOOKEEPER_SESSION_TIMEOUT is not exposed as client configurable
- static final int ZOOKEEPER_SESSION_TIMEOUT = 600 * 1000;
// Currently supports JKS keystore format
static final String SSL_TRUST_STORE_TYPE = "JKS";
Modified: hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java?rev=1637444&r1=1637443&r2=1637444&view=diff
==============================================================================
--- hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java (original)
+++ hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java Fri Nov 7 20:41:34 2014
@@ -25,9 +25,11 @@ import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.hive.jdbc.Utils.JdbcConnectionParams;
import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
public class ZooKeeperHiveClientHelper {
public static final Log LOG = LogFactory.getLog(ZooKeeperHiveClientHelper.class.getName());
@@ -59,14 +61,12 @@ public class ZooKeeperHiveClientHelper {
List<String> serverHosts;
Random randomizer = new Random();
String serverNode;
- ZooKeeper zooKeeperClient = null;
- // Pick a random HiveServer2 host from the ZooKeeper namspace
+ CuratorFramework zooKeeperClient =
+ CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble)
+ .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
+ zooKeeperClient.start();
try {
- zooKeeperClient =
- new ZooKeeper(zooKeeperEnsemble, JdbcConnectionParams.ZOOKEEPER_SESSION_TIMEOUT,
- new ZooKeeperHiveClientHelper.DummyWatcher());
- // All the HiveServer2 host nodes that are in ZooKeeper currently
- serverHosts = zooKeeperClient.getChildren("/" + zooKeeperNamespace, false);
+ serverHosts = zooKeeperClient.getChildren().forPath("/" + zooKeeperNamespace);
// Remove the znodes we've already tried from this list
serverHosts.removeAll(connParams.getRejectedHostZnodePaths());
if (serverHosts.isEmpty()) {
@@ -76,22 +76,18 @@ public class ZooKeeperHiveClientHelper {
// Now pick a host randomly
serverNode = serverHosts.get(randomizer.nextInt(serverHosts.size()));
connParams.setCurrentHostZnodePath(serverNode);
- // Read the value from the node (UTF-8 enoded byte array) and convert it to a String
String serverUri =
- new String(zooKeeperClient.getData("/" + zooKeeperNamespace + "/" + serverNode, false,
- null), Charset.forName("UTF-8"));
+ new String(
+ zooKeeperClient.getData().forPath("/" + zooKeeperNamespace + "/" + serverNode),
+ Charset.forName("UTF-8"));
LOG.info("Selected HiveServer2 instance with uri: " + serverUri);
return serverUri;
} catch (Exception e) {
throw new ZooKeeperHiveClientException("Unable to read HiveServer2 uri from ZooKeeper", e);
} finally {
- // Try to close the client connection with ZooKeeper
+ // Close the client connection with ZooKeeper
if (zooKeeperClient != null) {
- try {
- zooKeeperClient.close();
- } catch (Exception e) {
- // No-op
- }
+ zooKeeperClient.close();
}
}
}