You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/11/26 19:31:26 UTC
svn commit: r1641880 - in /hive/trunk:
contrib/src/java/org/apache/hadoop/hive/contrib/metastore/hooks/
contrib/src/test/queries/clientpositive/
hbase-handler/src/test/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/queries/positive/ hbase-handler...
Author: brock
Date: Wed Nov 26 18:31:26 2014
New Revision: 1641880
URL: http://svn.apache.org/r1641880
Log:
HIVE-8971 - HIVE-8965 exposed some classes which start with Test but are not tests (Brock reviewed by Xuefu)
Added:
hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/metastore/hooks/SampleURLHook.java
hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java
hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java
hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/AbstractTestGenericUDFOPNumeric.java
Removed:
hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/metastore/hooks/TestURLHook.java
hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory.java
hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory2.java
hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory3.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java
Modified:
hive/trunk/contrib/src/test/queries/clientpositive/url_hook.q
hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q
hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q
hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key3.q
hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out
hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out
hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInEmbed.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInRemote.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMinus.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMod.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMultiply.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPPlus.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFPosMod.java
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/metastore/hooks/SampleURLHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/metastore/hooks/SampleURLHook.java?rev=1641880&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/metastore/hooks/SampleURLHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/metastore/hooks/SampleURLHook.java Wed Nov 26 18:31:26 2014
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.contrib.metastore.hooks;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook;
+
+/**
+ * First returns a url for a blank DB, then returns a URL for the original DB.
+ * For testing the feature in url_hook.q
+ */
+public class SampleURLHook implements JDOConnectionURLHook {
+
+ private String originalUrl;
+
+ @Override
+ public String getJdoConnectionUrl(Configuration conf) throws Exception {
+ if (originalUrl == null) {
+ originalUrl = conf.get(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "");
+ return "jdbc:derby:;databaseName=target/tmp/junit_metastore_db_blank;create=true";
+ } else {
+ return originalUrl;
+ }
+
+ }
+
+ @Override
+ public void notifyBadConnectionUrl(String url) {
+
+ }
+
+}
Modified: hive/trunk/contrib/src/test/queries/clientpositive/url_hook.q
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/queries/clientpositive/url_hook.q?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/queries/clientpositive/url_hook.q (original)
+++ hive/trunk/contrib/src/test/queries/clientpositive/url_hook.q Wed Nov 26 18:31:26 2014
@@ -1,6 +1,6 @@
add jar ${system:maven.local.repository}/org/apache/hive/hive-contrib/${system:hive.version}/hive-contrib-${system:hive.version}.jar;
SHOW TABLES 'src';
-set hive.metastore.ds.connection.url.hook=org.apache.hadoop.hive.contrib.metastore.hooks.TestURLHook;
+set hive.metastore.ds.connection.url.hook=org.apache.hadoop.hive.contrib.metastore.hooks.SampleURLHook;
-- changes to dummy derby store.. should return empty result
SHOW TABLES 'src';
Added: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java?rev=1641880&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java (added)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java Wed Nov 26 18:31:26 2014
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.hbase;
+
+import org.apache.hadoop.hive.serde2.BaseStructObjectInspector;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class SampleHBaseKeyFactory extends DefaultHBaseKeyFactory {
+
+ private static final String DELIMITER_PATTERN = "\\$\\$";
+ private static final byte[] DELIMITER_BINARY = "$$".getBytes();
+
+ @Override
+ public ObjectInspector createKeyObjectInspector(TypeInfo type) {
+ return new SlashSeparatedOI((StructTypeInfo)type);
+ }
+
+ @Override
+ public LazyObjectBase createKey(ObjectInspector inspector) throws SerDeException {
+ return new DoubleDollarSeparated();
+ }
+
+ private final ByteStream.Output output = new ByteStream.Output();
+
+ @Override
+ public byte[] serializeKey(Object object, StructField field) throws IOException {
+ ObjectInspector inspector = field.getFieldObjectInspector();
+ if (inspector.getCategory() != ObjectInspector.Category.STRUCT) {
+ throw new IllegalStateException("invalid type value " + inspector.getTypeName());
+ }
+ output.reset();
+ for (Object element : ((StructObjectInspector)inspector).getStructFieldsDataAsList(object)) {
+ if (output.getLength() > 0) {
+ output.write(DELIMITER_BINARY);
+ }
+ output.write(String.valueOf(element).getBytes());
+ }
+ return output.getLength() > 0 ? output.toByteArray() : null;
+ }
+
+ private static class DoubleDollarSeparated implements LazyObjectBase {
+
+ private Object[] fields;
+
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ fields = new String(bytes.getData(), start, length).split(DELIMITER_PATTERN);
+ }
+
+ @Override
+ public Object getObject() {
+ return this;
+ }
+ }
+
+ private static class SlashSeparatedOI extends BaseStructObjectInspector {
+
+ private int length;
+
+ private SlashSeparatedOI(StructTypeInfo type) {
+ List<String> names = type.getAllStructFieldNames();
+ List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
+ for (int i = 0; i < names.size(); i++) {
+ ois.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ }
+ init(names, ois, null);
+ }
+
+ @Override
+ public Object getStructFieldData(Object data, StructField fieldRef) {
+ return ((DoubleDollarSeparated)data).fields[((MyField)fieldRef).getFieldID()];
+ }
+
+ @Override
+ public List<Object> getStructFieldsDataAsList(Object data) {
+ return Arrays.asList(((DoubleDollarSeparated)data).fields);
+ }
+ }
+}
Added: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java?rev=1641880&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java (added)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java Wed Nov 26 18:31:26 2014
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.hbase;
+
+import org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer;
+import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.serde2.BaseStructObjectInspector;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.mapred.JobConf;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SampleHBaseKeyFactory2 extends AbstractHBaseKeyFactory {
+
+ private static final int FIXED_LENGTH = 10;
+
+ @Override
+ public ObjectInspector createKeyObjectInspector(TypeInfo type) {
+ return new StringArrayOI((StructTypeInfo)type);
+ }
+
+ @Override
+ public LazyObjectBase createKey(ObjectInspector inspector) throws SerDeException {
+ return new FixedLengthed(FIXED_LENGTH);
+ }
+
+ private final ByteStream.Output output = new ByteStream.Output();
+
+ @Override
+ public byte[] serializeKey(Object object, StructField field) throws IOException {
+ ObjectInspector inspector = field.getFieldObjectInspector();
+ if (inspector.getCategory() != ObjectInspector.Category.STRUCT) {
+ throw new IllegalStateException("invalid type value " + inspector.getTypeName());
+ }
+ output.reset();
+ for (Object element : ((StructObjectInspector)inspector).getStructFieldsDataAsList(object)) {
+ output.write(toBinary(String.valueOf(element).getBytes(), FIXED_LENGTH, false, false));
+ }
+ return output.getLength() > 0 ? output.toByteArray() : null;
+ }
+
+ private byte[] toBinary(String value, int max, boolean end, boolean nextBA) {
+ return toBinary(value.getBytes(), max, end, nextBA);
+ }
+
+ private byte[] toBinary(byte[] value, int max, boolean end, boolean nextBA) {
+ byte[] bytes = new byte[max + 1];
+ System.arraycopy(value, 0, bytes, 0, Math.min(value.length, max));
+ if (end) {
+ Arrays.fill(bytes, value.length, max, (byte)0xff);
+ }
+ if (nextBA) {
+ bytes[max] = 0x01;
+ }
+ return bytes;
+ }
+
+ @Override
+ public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer,
+ ExprNodeDesc predicate) {
+ String keyColName = keyMapping.columnName;
+
+ IndexPredicateAnalyzer analyzer = IndexPredicateAnalyzer.createAnalyzer(false);
+ analyzer.allowColumnName(keyColName);
+ analyzer.setAcceptsFields(true);
+
+ DecomposedPredicate decomposed = new DecomposedPredicate();
+
+ List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
+ decomposed.residualPredicate =
+ (ExprNodeGenericFuncDesc)analyzer.analyzePredicate(predicate, searchConditions);
+ if (!searchConditions.isEmpty()) {
+ decomposed.pushedPredicate = analyzer.translateSearchConditions(searchConditions);
+ try {
+ decomposed.pushedPredicateObject = setupFilter(keyColName, searchConditions);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return decomposed;
+ }
+
+ private HBaseScanRange setupFilter(String keyColName, List<IndexSearchCondition> conditions)
+ throws IOException {
+ Map<String, List<IndexSearchCondition>> fieldConds =
+ new HashMap<String, List<IndexSearchCondition>>();
+ for (IndexSearchCondition condition : conditions) {
+ assert keyColName.equals(condition.getColumnDesc().getColumn());
+ String fieldName = condition.getFields()[0];
+ List<IndexSearchCondition> fieldCond = fieldConds.get(fieldName);
+ if (fieldCond == null) {
+ fieldConds.put(fieldName, fieldCond = new ArrayList<IndexSearchCondition>());
+ }
+ fieldCond.add(condition);
+ }
+ HBaseScanRange range = new HBaseScanRange();
+
+ ByteArrayOutputStream startRow = new ByteArrayOutputStream();
+ ByteArrayOutputStream stopRow = new ByteArrayOutputStream();
+
+ StructTypeInfo type = (StructTypeInfo) keyMapping.columnType;
+ for (String name : type.getAllStructFieldNames()) {
+ List<IndexSearchCondition> fieldCond = fieldConds.get(name);
+ if (fieldCond == null || fieldCond.size() > 2) {
+ continue;
+ }
+ byte[] startElement = null;
+ byte[] stopElement = null;
+ for (IndexSearchCondition condition : fieldCond) {
+ if (condition.getConstantDesc().getValue() == null) {
+ continue;
+ }
+ String comparisonOp = condition.getComparisonOp();
+ String constantVal = String.valueOf(condition.getConstantDesc().getValue());
+
+ if (comparisonOp.endsWith("UDFOPEqual")) {
+ startElement = toBinary(constantVal, FIXED_LENGTH, false, false);
+ stopElement = toBinary(constantVal, FIXED_LENGTH, true, true);
+ } else if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) {
+ startElement = toBinary(constantVal, FIXED_LENGTH, false, false);
+ } else if (comparisonOp.endsWith("UDFOPGreaterThan")) {
+ startElement = toBinary(constantVal, FIXED_LENGTH, false, true);
+ } else if (comparisonOp.endsWith("UDFOPEqualOrLessThan")) {
+ stopElement = toBinary(constantVal, FIXED_LENGTH, true, false);
+ } else if (comparisonOp.endsWith("UDFOPLessThan")) {
+ stopElement = toBinary(constantVal, FIXED_LENGTH, true, true);
+ } else {
+ throw new IOException(comparisonOp + " is not a supported comparison operator");
+ }
+ }
+ if (startRow != null) {
+ if (startElement != null) {
+ startRow.write(startElement);
+ } else {
+ if (startRow.size() > 0) {
+ range.setStartRow(startRow.toByteArray());
+ }
+ startRow = null;
+ }
+ }
+ if (stopRow != null) {
+ if (stopElement != null) {
+ stopRow.write(stopElement);
+ } else {
+ if (stopRow.size() > 0) {
+ range.setStopRow(stopRow.toByteArray());
+ }
+ stopRow = null;
+ }
+ }
+ if (startElement == null && stopElement == null) {
+ break;
+ }
+ }
+ if (startRow != null && startRow.size() > 0) {
+ range.setStartRow(startRow.toByteArray());
+ }
+ if (stopRow != null && stopRow.size() > 0) {
+ range.setStopRow(stopRow.toByteArray());
+ }
+ return range;
+ }
+
+ private static class FixedLengthed implements LazyObjectBase {
+
+ private final int fixedLength;
+ private final List<Object> fields = new ArrayList<Object>();
+
+ public FixedLengthed(int fixedLength) {
+ this.fixedLength = fixedLength;
+ }
+
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ fields.clear();
+ byte[] data = bytes.getData();
+ int rowStart = start;
+ int rowStop = rowStart + fixedLength;
+ for (; rowStart < length; rowStart = rowStop + 1, rowStop = rowStart + fixedLength) {
+ fields.add(new String(data, rowStart, rowStop - rowStart).trim());
+ }
+ }
+
+ @Override
+ public Object getObject() {
+ return this;
+ }
+ }
+
+ private static class StringArrayOI extends BaseStructObjectInspector {
+
+ private int length;
+
+ private StringArrayOI(StructTypeInfo type) {
+ List<String> names = type.getAllStructFieldNames();
+ List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
+ for (int i = 0; i < names.size(); i++) {
+ ois.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ }
+ init(names, ois, null);
+ }
+
+ @Override
+ public Object getStructFieldData(Object data, StructField fieldRef) {
+ return ((FixedLengthed)data).fields.get(((MyField)fieldRef).getFieldID());
+ }
+
+ @Override
+ public List<Object> getStructFieldsDataAsList(Object data) {
+ return ((FixedLengthed)data).fields;
+ }
+ }
+}
Added: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java?rev=1641880&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java (added)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java Wed Nov 26 18:31:26 2014
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.RowFilter;
+import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
+import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.mapred.JobConf;
+
+/**
+ * Simple extension of {@link SampleHBaseKeyFactory2} with exception of using filters instead of start
+ * and stop keys
+ * */
+public class SampleHBaseKeyFactory3 extends SampleHBaseKeyFactory2 {
+
+ @Override
+ public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer,
+ ExprNodeDesc predicate) {
+ SampleHBasePredicateDecomposer decomposedPredicate = new SampleHBasePredicateDecomposer(keyMapping);
+ return decomposedPredicate.decomposePredicate(keyMapping.columnName, predicate);
+ }
+}
+
+class SampleHBasePredicateDecomposer extends AbstractHBaseKeyPredicateDecomposer {
+
+ private static final int FIXED_LENGTH = 10;
+
+ private ColumnMapping keyMapping;
+
+ SampleHBasePredicateDecomposer(ColumnMapping keyMapping) {
+ this.keyMapping = keyMapping;
+ }
+
+ @Override
+ public HBaseScanRange getScanRange(List<IndexSearchCondition> searchConditions)
+ throws Exception {
+ Map<String, List<IndexSearchCondition>> fieldConds =
+ new HashMap<String, List<IndexSearchCondition>>();
+ for (IndexSearchCondition condition : searchConditions) {
+ String fieldName = condition.getFields()[0];
+ List<IndexSearchCondition> fieldCond = fieldConds.get(fieldName);
+ if (fieldCond == null) {
+ fieldConds.put(fieldName, fieldCond = new ArrayList<IndexSearchCondition>());
+ }
+ fieldCond.add(condition);
+ }
+ Filter filter = null;
+ HBaseScanRange range = new HBaseScanRange();
+
+ StructTypeInfo type = (StructTypeInfo) keyMapping.columnType;
+ for (String name : type.getAllStructFieldNames()) {
+ List<IndexSearchCondition> fieldCond = fieldConds.get(name);
+ if (fieldCond == null || fieldCond.size() > 2) {
+ continue;
+ }
+ for (IndexSearchCondition condition : fieldCond) {
+ if (condition.getConstantDesc().getValue() == null) {
+ continue;
+ }
+ String comparisonOp = condition.getComparisonOp();
+ String constantVal = String.valueOf(condition.getConstantDesc().getValue());
+
+ byte[] valueAsBytes = toBinary(constantVal, FIXED_LENGTH, false, false);
+
+ if (comparisonOp.endsWith("UDFOPEqual")) {
+ filter = new RowFilter(CompareOp.EQUAL, new BinaryComparator(valueAsBytes));
+ } else if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) {
+ filter = new RowFilter(CompareOp.GREATER_OR_EQUAL, new BinaryComparator(valueAsBytes));
+ } else if (comparisonOp.endsWith("UDFOPGreaterThan")) {
+ filter = new RowFilter(CompareOp.GREATER, new BinaryComparator(valueAsBytes));
+ } else if (comparisonOp.endsWith("UDFOPEqualOrLessThan")) {
+ filter = new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator(valueAsBytes));
+ } else if (comparisonOp.endsWith("UDFOPLessThan")) {
+ filter = new RowFilter(CompareOp.LESS, new BinaryComparator(valueAsBytes));
+ } else {
+ throw new IOException(comparisonOp + " is not a supported comparison operator");
+ }
+ }
+ }
+ if (filter != null) {
+ range.addFilter(filter);
+ }
+ return range;
+ }
+
+ private byte[] toBinary(String value, int max, boolean end, boolean nextBA) {
+ return toBinary(value.getBytes(), max, end, nextBA);
+ }
+
+ private byte[] toBinary(byte[] value, int max, boolean end, boolean nextBA) {
+ byte[] bytes = new byte[max + 1];
+ System.arraycopy(value, 0, bytes, 0, Math.min(value.length, max));
+ if (end) {
+ Arrays.fill(bytes, value.length, max, (byte) 0xff);
+ }
+ if (nextBA) {
+ bytes[max] = 0x01;
+ }
+ return bytes;
+ }
+}
\ No newline at end of file
Modified: hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q (original)
+++ hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q Wed Nov 26 18:31:26 2014
@@ -3,7 +3,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory");
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory");
CREATE EXTERNAL TABLE hbase_ck_2(key string, value string)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
Modified: hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q (original)
+++ hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q Wed Nov 26 18:31:26 2014
@@ -3,7 +3,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom2",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory2");
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory2");
from src tablesample (5 rows)
insert into table hbase_ck_4 select
Modified: hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key3.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key3.q?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key3.q (original)
+++ hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key3.q Wed Nov 26 18:31:26 2014
@@ -3,7 +3,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom3",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory3");
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory3");
from src tablesample (5 rows)
insert into table hbase_ck_5 select
Modified: hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out Wed Nov 26 18:31:26 2014
@@ -3,7 +3,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory")
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@hbase_ck_1
@@ -12,7 +12,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory")
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@hbase_ck_1
Modified: hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out Wed Nov 26 18:31:26 2014
@@ -3,7 +3,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom2",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory2")
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory2")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@hbase_ck_4
@@ -12,7 +12,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom2",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory2")
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory2")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@hbase_ck_4
Modified: hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out Wed Nov 26 18:31:26 2014
@@ -3,7 +3,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom3",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory3")
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory3")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@hbase_ck_5
@@ -12,7 +12,7 @@ STORED BY 'org.apache.hadoop.hive.hbase.
WITH SERDEPROPERTIES (
"hbase.table.name" = "hbase_custom3",
"hbase.columns.mapping" = ":key,cf:string",
- "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory3")
+ "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.SampleHBaseKeyFactory3")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@hbase_ck_5
Added: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java?rev=1641880&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java (added)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java Wed Nov 26 18:31:26 2014
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
+import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly;
+import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.junit.Test;
+
+/**
+ * Test case for {@link MetaStoreAuthzAPIAuthorizerEmbedOnly} The authorizer is
+ * supposed to allow api calls for metastore in embedded mode while disallowing
+ * them in remote metastore mode. Note that this is an abstract class, the
+ * subclasses that set the mode and the tests here get run as part of their
+ * testing.
+ */
+public abstract class AbstractTestAuthorizationApiAuthorizer {
+ protected static boolean isRemoteMetastoreMode;
+ private static HiveConf hiveConf;
+ private static HiveMetaStoreClient msc;
+
+ protected static void setup() throws Exception {
+ System.err.println("Running with remoteMode = " + isRemoteMetastoreMode);
+ System.setProperty("hive.metastore.pre.event.listeners",
+ AuthorizationPreEventListener.class.getName());
+ System.setProperty("hive.security.metastore.authorization.manager",
+ MetaStoreAuthzAPIAuthorizerEmbedOnly.class.getName());
+
+ hiveConf = new HiveConf();
+ if (isRemoteMetastoreMode) {
+ int port = MetaStoreUtils.findFreePort();
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+ }
+ hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
+ hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
+ hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
+ hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+
+ msc = new HiveMetaStoreClient(hiveConf, null);
+
+ }
+
+ interface FunctionInvoker {
+ public void invoke() throws Exception;
+ }
+
+ /**
+ * Test the if authorization failed/passed for FunctionInvoker that invokes a metastore client
+ * api call
+ * @param mscFunctionInvoker
+ * @throws Exception
+ */
+ private void testFunction(FunctionInvoker mscFunctionInvoker) throws Exception {
+ boolean caughtEx = false;
+ try {
+ try {
+ mscFunctionInvoker.invoke();
+ } catch (RuntimeException e) {
+ // A hack to verify that authorization check passed. Exception can be thrown be cause
+ // the functions are not being called with valid params.
+ // verify that exception has come from ObjectStore code, which means that the
+ // authorization checks passed.
+ String exStackString = ExceptionUtils.getStackTrace(e);
+ assertTrue("Verifying this exception came after authorization check",
+ exStackString.contains("org.apache.hadoop.hive.metastore.ObjectStore"));
+ // If its not an exception caused by auth check, ignore it
+ }
+ assertFalse("Authz Exception should have been thrown in remote mode", isRemoteMetastoreMode);
+ System.err.println("No auth exception thrown");
+ } catch (MetaException e) {
+ System.err.println("Caught exception");
+ caughtEx = true;
+ assertTrue(e.getMessage().contains(MetaStoreAuthzAPIAuthorizerEmbedOnly.errMsg));
+ }
+ if (!isRemoteMetastoreMode) {
+ assertFalse("No exception should be thrown in embedded mode", caughtEx);
+ }
+ }
+
+ @Test
+ public void testGrantPriv() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.grant_privileges(new PrivilegeBag(new ArrayList<HiveObjectPrivilege>()));
+ }
+ };
+ testFunction(invoker);
+ }
+
+ @Test
+ public void testRevokePriv() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.revoke_privileges(new PrivilegeBag(new ArrayList<HiveObjectPrivilege>()), false);
+ }
+ };
+ testFunction(invoker);
+ }
+
+ @Test
+ public void testGrantRole() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.grant_role(null, null, null, null, null, true);
+ }
+ };
+ testFunction(invoker);
+ }
+
+ @Test
+ public void testRevokeRole() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.revoke_role(null, null, null, false);
+ }
+ };
+ testFunction(invoker);
+ }
+
+ @Test
+ public void testCreateRole() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.create_role(new Role("role1", 0, "owner"));
+ }
+ };
+ testFunction(invoker);
+ }
+
+ @Test
+ public void testDropRole() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.drop_role(null);
+ }
+ };
+ testFunction(invoker);
+ }
+
+ @Test
+ public void testListRoles() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.list_roles(null, null);
+ }
+ };
+ testFunction(invoker);
+ }
+
+ @Test
+ public void testGetPrivSet() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.get_privilege_set(new HiveObjectRef(), null, new ArrayList<String>());
+ }
+ };
+ testFunction(invoker);
+ }
+
+ @Test
+ public void testListPriv() throws Exception {
+ FunctionInvoker invoker = new FunctionInvoker() {
+ @Override
+ public void invoke() throws Exception {
+ msc.list_privileges(null, PrincipalType.USER, new HiveObjectRef());
+ }
+ };
+ testFunction(invoker);
+ }
+
+
+
+}
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInEmbed.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInEmbed.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInEmbed.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInEmbed.java Wed Nov 26 18:31:26 2014
@@ -3,14 +3,14 @@ package org.apache.hadoop.hive.metastore
import org.junit.BeforeClass;
/**
- * Test {@link TestAuthorizationApiAuthorizer} in embedded mode of metastore
+ * Test {@link AbstractTestAuthorizationApiAuthorizer} in embedded mode of metastore
*/
-public class TestAuthzApiEmbedAuthorizerInEmbed extends TestAuthorizationApiAuthorizer {
+public class TestAuthzApiEmbedAuthorizerInEmbed extends AbstractTestAuthorizationApiAuthorizer {
@BeforeClass
public static void setup() throws Exception {
isRemoteMetastoreMode = false; // embedded metastore mode
- TestAuthorizationApiAuthorizer.setup();
+ AbstractTestAuthorizationApiAuthorizer.setup();
}
}
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInRemote.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInRemote.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInRemote.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthzApiEmbedAuthorizerInRemote.java Wed Nov 26 18:31:26 2014
@@ -3,14 +3,14 @@ package org.apache.hadoop.hive.metastore
import org.junit.BeforeClass;
/**
- * Test {@link TestAuthorizationApiAuthorizer} in remote mode of metastore
+ * Test {@link AbstractTestAuthorizationApiAuthorizer} in remote mode of metastore
*/
-public class TestAuthzApiEmbedAuthorizerInRemote extends TestAuthorizationApiAuthorizer {
+public class TestAuthzApiEmbedAuthorizerInRemote extends AbstractTestAuthorizationApiAuthorizer {
@BeforeClass
public static void setup() throws Exception {
isRemoteMetastoreMode = true; // remote metastore mode
- TestAuthorizationApiAuthorizer.setup();
+ AbstractTestAuthorizationApiAuthorizer.setup();
}
}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java?rev=1641880&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java Wed Nov 26 18:31:26 2014
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import javax.security.auth.login.LoginException;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.tez.dag.api.TezException;
+
+
+/**
+ * This class is needed for writing junit tests. For testing the multi-session
+ * use case from hive server 2, we need a session simulation.
+ *
+ */
+public class SampleTezSessionState extends TezSessionState {
+
+ private boolean open;
+ private final String sessionId;
+ private HiveConf hiveConf;
+ private String user;
+ private boolean doAsEnabled;
+
+ public SampleTezSessionState(String sessionId) {
+ super(sessionId);
+ this.sessionId = sessionId;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return open;
+ }
+
+ public void setOpen(boolean open) {
+ this.open = open;
+ }
+
+ @Override
+ public void open(HiveConf conf) throws IOException, LoginException, URISyntaxException,
+ TezException {
+ this.hiveConf = conf;
+ UserGroupInformation ugi;
+ ugi = ShimLoader.getHadoopShims().getUGIForConf(conf);
+ user = ShimLoader.getHadoopShims().getShortUserName(ugi);
+ this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS);
+ }
+
+ @Override
+ public void close(boolean keepTmpDir) throws TezException, IOException {
+ open = keepTmpDir;
+ }
+
+ @Override
+ public HiveConf getConf() {
+ return this.hiveConf;
+ }
+
+ @Override
+ public String getSessionId() {
+ return sessionId;
+ }
+
+ @Override
+ public String getUser() {
+ return user;
+ }
+
+ @Override
+ public boolean getDoAsEnabled() {
+ return this.doAsEnabled;
+ }
+}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java Wed Nov 26 18:31:26 2014
@@ -42,7 +42,7 @@ public class TestTezSessionPool {
@Override
public TezSessionState createSession(String sessionId) {
- return new TestTezSessionState(sessionId);
+ return new SampleTezSessionState(sessionId);
}
}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/AbstractTestGenericUDFOPNumeric.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/AbstractTestGenericUDFOPNumeric.java?rev=1641880&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/AbstractTestGenericUDFOPNumeric.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/AbstractTestGenericUDFOPNumeric.java Wed Nov 26 18:31:26 2014
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.junit.Assert;
+
+public abstract class AbstractTestGenericUDFOPNumeric {
+ public AbstractTestGenericUDFOPNumeric() {
+ // Arithmetic operations rely on getting conf from SessionState, need to initialize here.
+ SessionState ss = new SessionState(new HiveConf());
+ ss.getConf().setVar(HiveConf.ConfVars.HIVE_COMPAT, "latest");
+ SessionState.setCurrentSessionState(ss);
+ }
+
+ protected void verifyReturnType(GenericUDF udf,
+ String typeStr1, String typeStr2, String expectedTypeStr) throws HiveException {
+ // Lookup type infos for our input types and expected return type
+ PrimitiveTypeInfo type1 = TypeInfoFactory.getPrimitiveTypeInfo(typeStr1);
+ PrimitiveTypeInfo type2 = TypeInfoFactory.getPrimitiveTypeInfo(typeStr2);
+ PrimitiveTypeInfo expectedType = TypeInfoFactory.getPrimitiveTypeInfo(expectedTypeStr);
+
+ // Initialize UDF which will output the return type for the UDF.
+ ObjectInspector[] inputOIs = {
+ PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type1),
+ PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type2)
+ };
+ PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
+
+ Assert.assertEquals("Return type for " + udf.getDisplayString(new String[] {typeStr1, typeStr2}),
+ expectedType, oi.getTypeInfo());
+ }
+}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java Wed Nov 26 18:31:26 2014
@@ -40,7 +40,7 @@ import org.apache.hadoop.io.LongWritable
import org.junit.Assert;
import org.junit.Test;
-public class TestGenericUDFOPDivide extends TestGenericUDFOPNumeric {
+public class TestGenericUDFOPDivide extends AbstractTestGenericUDFOPNumeric {
@Test
public void testByteDivideShort() throws HiveException {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMinus.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMinus.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMinus.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMinus.java Wed Nov 26 18:31:26 2014
@@ -39,7 +39,7 @@ import org.apache.hadoop.io.LongWritable
import org.junit.Assert;
import org.junit.Test;
-public class TestGenericUDFOPMinus extends TestGenericUDFOPNumeric {
+public class TestGenericUDFOPMinus extends AbstractTestGenericUDFOPNumeric {
@Test
public void testByteMinusShort() throws HiveException {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMod.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMod.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMod.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMod.java Wed Nov 26 18:31:26 2014
@@ -36,7 +36,7 @@ import org.apache.hadoop.io.LongWritable
import org.junit.Assert;
import org.junit.Test;
-public class TestGenericUDFOPMod extends TestGenericUDFOPNumeric {
+public class TestGenericUDFOPMod extends AbstractTestGenericUDFOPNumeric {
@Test
public void testModByZero1() throws HiveException {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMultiply.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMultiply.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMultiply.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPMultiply.java Wed Nov 26 18:31:26 2014
@@ -39,7 +39,7 @@ import org.apache.hadoop.io.LongWritable
import org.junit.Assert;
import org.junit.Test;
-public class TestGenericUDFOPMultiply extends TestGenericUDFOPNumeric {
+public class TestGenericUDFOPMultiply extends AbstractTestGenericUDFOPNumeric {
@Test
public void testByteTimesShort() throws HiveException {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPPlus.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPPlus.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPPlus.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPPlus.java Wed Nov 26 18:31:26 2014
@@ -39,7 +39,7 @@ import org.apache.hadoop.io.LongWritable
import org.junit.Assert;
import org.junit.Test;
-public class TestGenericUDFOPPlus extends TestGenericUDFOPNumeric {
+public class TestGenericUDFOPPlus extends AbstractTestGenericUDFOPNumeric {
@Test
public void testBytePlusShort() throws HiveException {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFPosMod.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFPosMod.java?rev=1641880&r1=1641879&r2=1641880&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFPosMod.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFPosMod.java Wed Nov 26 18:31:26 2014
@@ -36,7 +36,7 @@ import org.apache.hadoop.io.LongWritable
import org.junit.Assert;
import org.junit.Test;
-public class TestGenericUDFPosMod extends TestGenericUDFOPNumeric {
+public class TestGenericUDFPosMod extends AbstractTestGenericUDFOPNumeric {
@Test
public void testPosModByZero1() throws HiveException {