You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2014/08/20 00:41:13 UTC
svn commit: r1619005 [4/9] - in /hive/trunk: ./ accumulo-handler/
accumulo-handler/src/ accumulo-handler/src/java/
accumulo-handler/src/java/org/ accumulo-handler/src/java/org/apache/
accumulo-handler/src/java/org/apache/hadoop/ accumulo-handler/src/ja...
Added: hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloSerDeParameters.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloSerDeParameters.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloSerDeParameters.java (added)
+++ hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloSerDeParameters.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo.serde;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Properties;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters;
+import org.apache.hadoop.hive.accumulo.columns.ColumnMapper;
+import org.apache.hadoop.hive.accumulo.columns.ColumnMapping;
+import org.apache.hadoop.hive.accumulo.columns.HiveAccumuloRowIdColumnMapping;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ *
+ */
+public class AccumuloSerDeParameters extends AccumuloConnectionParameters {
+ private static final Logger log = Logger.getLogger(AccumuloSerDeParameters.class);
+
+ public static final String COLUMN_MAPPINGS = "accumulo.columns.mapping";
+ public static final String ITERATOR_PUSHDOWN_KEY = "accumulo.iterator.pushdown";
+ public static final boolean ITERATOR_PUSHDOWN_DEFAULT = true;
+
+ public static final String DEFAULT_STORAGE_TYPE = "accumulo.default.storage";
+
+ public static final String VISIBILITY_LABEL_KEY = "accumulo.visibility.label";
+ public static final ColumnVisibility DEFAULT_VISIBILITY_LABEL = new ColumnVisibility();
+
+ public static final String AUTHORIZATIONS_KEY = "accumulo.authorizations";
+
+ public static final String COMPOSITE_ROWID_FACTORY = "accumulo.composite.rowid.factory";
+ public static final String COMPOSITE_ROWID_CLASS = "accumulo.composite.rowid";
+
+ protected final ColumnMapper columnMapper;
+
+ private Properties tableProperties;
+ private String serdeName;
+ private SerDeParameters lazySerDeParameters;
+ private AccumuloRowIdFactory rowIdFactory;
+
+ public AccumuloSerDeParameters(Configuration conf, Properties tableProperties, String serdeName)
+ throws SerDeException {
+ super(conf);
+ this.tableProperties = tableProperties;
+ this.serdeName = serdeName;
+
+ lazySerDeParameters = LazySimpleSerDe.initSerdeParams(conf, tableProperties, serdeName);
+
+ // The default encoding for this table when not otherwise specified
+ String defaultStorage = tableProperties.getProperty(DEFAULT_STORAGE_TYPE);
+
+ columnMapper = new ColumnMapper(getColumnMappingValue(), defaultStorage,
+ lazySerDeParameters.getColumnNames(), lazySerDeParameters.getColumnTypes());
+
+ log.info("Constructed column mapping " + columnMapper);
+
+ // Generate types for column mapping
+ if (null == getColumnTypeValue()) {
+ tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES, columnMapper.getTypesString());
+ }
+
+ if (columnMapper.size() < lazySerDeParameters.getColumnNames().size()) {
+ throw new TooManyHiveColumnsException("You have more " + COLUMN_MAPPINGS
+ + " fields than hive columns");
+ } else if (columnMapper.size() > lazySerDeParameters.getColumnNames().size()) {
+ throw new TooManyAccumuloColumnsException(
+ "You have more hive columns than fields mapped with " + COLUMN_MAPPINGS);
+ }
+
+ this.rowIdFactory = initRowIdFactory(conf, tableProperties);
+ }
+
+ protected AccumuloRowIdFactory initRowIdFactory(Configuration conf, Properties tbl)
+ throws SerDeException {
+ try {
+ AccumuloRowIdFactory keyFactory = createRowIdFactory(conf, tbl);
+ if (keyFactory != null) {
+ keyFactory.init(this, tbl);
+ }
+ return keyFactory;
+ } catch (Exception e) {
+ throw new SerDeException(e);
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ protected AccumuloRowIdFactory createRowIdFactory(Configuration job, Properties tbl)
+ throws Exception {
+ // Try to load the composite factory if one was provided
+ String factoryClassName = tbl.getProperty(COMPOSITE_ROWID_FACTORY);
+ if (factoryClassName != null) {
+ log.info("Loading CompositeRowIdFactory class " + factoryClassName);
+ Class<?> factoryClazz = Class.forName(factoryClassName);
+ return (AccumuloRowIdFactory) ReflectionUtils.newInstance(factoryClazz, job);
+ }
+
+ // See if a custom CompositeKey class was provided
+ String keyClassName = tbl.getProperty(COMPOSITE_ROWID_CLASS);
+ if (keyClassName != null) {
+ log.info("Loading CompositeRowId class " + keyClassName);
+ Class<?> keyClass = Class.forName(keyClassName);
+ Class<? extends AccumuloCompositeRowId> compositeRowIdClass = keyClass
+ .asSubclass(AccumuloCompositeRowId.class);
+ return new CompositeAccumuloRowIdFactory(compositeRowIdClass);
+ }
+
+ return new DefaultAccumuloRowIdFactory();
+ }
+
+ public SerDeParameters getSerDeParameters() {
+ return lazySerDeParameters;
+ }
+
+ public Properties getTableProperties() {
+ return tableProperties;
+ }
+
+ public String getColumnTypeValue() {
+ return tableProperties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+ }
+
+ public String getSerDeName() {
+ return serdeName;
+ }
+
+ public String getColumnMappingValue() {
+ return tableProperties.getProperty(COLUMN_MAPPINGS);
+ }
+
+ public HiveAccumuloRowIdColumnMapping getRowIdColumnMapping() {
+ return columnMapper.getRowIdMapping();
+ }
+
+ public boolean getIteratorPushdown() {
+ return conf.getBoolean(ITERATOR_PUSHDOWN_KEY, ITERATOR_PUSHDOWN_DEFAULT);
+ }
+
+ public List<String> getHiveColumnNames() {
+ return Collections.unmodifiableList(lazySerDeParameters.getColumnNames());
+ }
+
+ public List<TypeInfo> getHiveColumnTypes() {
+ return Collections.unmodifiableList(lazySerDeParameters.getColumnTypes());
+ }
+
+ public ColumnMapper getColumnMapper() {
+ return columnMapper;
+ }
+
+ public int getRowIdOffset() {
+ return columnMapper.getRowIdOffset();
+ }
+
+ public List<ColumnMapping> getColumnMappings() {
+ return columnMapper.getColumnMappings();
+ }
+
+ public AccumuloRowIdFactory getRowIdFactory() {
+ return rowIdFactory;
+ }
+
+ public String getRowIdHiveColumnName() {
+ int rowIdOffset = columnMapper.getRowIdOffset();
+ if (-1 == rowIdOffset) {
+ return null;
+ }
+
+ List<String> hiveColumnNames = lazySerDeParameters.getColumnNames();
+ if (0 > rowIdOffset || hiveColumnNames.size() <= rowIdOffset) {
+ throw new IllegalStateException("Tried to find rowID offset at position " + rowIdOffset
+ + " from Hive columns " + hiveColumnNames);
+ }
+
+ return hiveColumnNames.get(rowIdOffset);
+ }
+
+ public ColumnMapping getColumnMappingForHiveColumn(String hiveColumn) {
+ List<String> hiveColumnNames = lazySerDeParameters.getColumnNames();
+
+ for (int offset = 0; offset < hiveColumnNames.size() && offset < columnMapper.size(); offset++) {
+ String hiveColumnName = hiveColumnNames.get(offset);
+ if (hiveColumn.equals(hiveColumnName)) {
+ return columnMapper.get(offset);
+ }
+ }
+
+ throw new NoSuchElementException("Could not find column mapping for Hive column " + hiveColumn);
+ }
+
+ public TypeInfo getTypeForHiveColumn(String hiveColumn) {
+ List<String> hiveColumnNames = lazySerDeParameters.getColumnNames();
+ List<TypeInfo> hiveColumnTypes = lazySerDeParameters.getColumnTypes();
+
+ for (int i = 0; i < hiveColumnNames.size() && i < hiveColumnTypes.size(); i++) {
+ String columnName = hiveColumnNames.get(i);
+ if (hiveColumn.equals(columnName)) {
+ return hiveColumnTypes.get(i);
+ }
+ }
+
+ throw new NoSuchElementException("Could not find Hive column type for " + hiveColumn);
+ }
+
+ /**
+ * Extracts the table property to allow a custom ColumnVisibility label to be set on updates to be
+ * written to an Accumulo table. The value in the table property must be a properly formatted
+ * {@link ColumnVisibility}. If not value is present in the table properties, an empty
+ * ColumnVisibility is returned.
+ *
+ * @return The ColumnVisibility to be applied to all updates sent to Accumulo
+ */
+ public ColumnVisibility getTableVisibilityLabel() {
+ String visibilityLabel = tableProperties.getProperty(VISIBILITY_LABEL_KEY, null);
+ if (null == visibilityLabel || visibilityLabel.isEmpty()) {
+ return DEFAULT_VISIBILITY_LABEL;
+ }
+
+ return new ColumnVisibility(visibilityLabel);
+ }
+
+ /**
+ * Extracts the table property to allow dynamic Accumulo Authorizations to be used when reading
+ * data from an Accumulo table. If no Authorizations are provided in the table properties, null is
+ * returned to preserve the functionality to read all data that the current user has access to.
+ *
+ * @return The Authorizations that should be used to read data from Accumulo, null if no
+ * configuration is supplied.
+ */
+ public Authorizations getAuthorizations() {
+ String authorizationStr = tableProperties.getProperty(AUTHORIZATIONS_KEY, null);
+
+ return getAuthorizationsFromValue(authorizationStr);
+ }
+
+ /**
+ * Create an Authorizations object when the provided value is not null. Will return null,
+ * otherwise.
+ *
+ * @param authorizationStr
+ * Configuration value to parse
+ * @return Authorization object or null
+ */
+ protected static Authorizations getAuthorizationsFromValue(String authorizationStr) {
+ if (null == authorizationStr) {
+ return null;
+ }
+
+ return new Authorizations(authorizationStr);
+ }
+
+ /**
+ * Extract any configuration on Authorizations to be used from the provided Configuration. If a
+ * non-null value is not present in the configuration, a null object is returned
+ *
+ * @return Authorization built from configuration value, null if no value is present in conf
+ */
+ public static Authorizations getAuthorizationsFromConf(Configuration conf) {
+ Preconditions.checkNotNull(conf);
+
+ String authorizationStr = conf.get(AUTHORIZATIONS_KEY, null);
+
+ return getAuthorizationsFromValue(authorizationStr);
+ }
+}
Added: hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/CompositeAccumuloRowIdFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/CompositeAccumuloRowIdFactory.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/CompositeAccumuloRowIdFactory.java (added)
+++ hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/CompositeAccumuloRowIdFactory.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.accumulo.serde;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.accumulo.Utils;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.log4j.Logger;
+
+/**
+ * {@link AccumuloRowIdFactory} designed for injection of the {@link AccumuloCompositeRowId} to be
+ * used to generate the Accumulo rowId. Allows for custom {@link AccumuloCompositeRowId}s to be
+ * specified without overriding the entire ObjectInspector for the Hive row.
+ *
+ * @param <T>
+ */
+public class CompositeAccumuloRowIdFactory<T extends AccumuloCompositeRowId> extends
+ DefaultAccumuloRowIdFactory {
+
+ public static final Logger log = Logger.getLogger(CompositeAccumuloRowIdFactory.class);
+
+ private final Class<T> keyClass;
+ private final Constructor<T> constructor;
+
+ public CompositeAccumuloRowIdFactory(Class<T> keyClass) throws SecurityException,
+ NoSuchMethodException {
+ // see javadoc of AccumuloCompositeRowId
+ this.keyClass = keyClass;
+ this.constructor = keyClass.getDeclaredConstructor(LazySimpleStructObjectInspector.class,
+ Properties.class, Configuration.class);
+ }
+
+ @Override
+ public void addDependencyJars(Configuration jobConf) throws IOException {
+ // Make sure the jar containing the custom CompositeRowId is included
+ // in the mapreduce job's classpath (libjars)
+ Utils.addDependencyJars(jobConf, keyClass);
+ }
+
+ @Override
+ public T createRowId(ObjectInspector inspector) throws SerDeException {
+ try {
+ return (T) constructor.newInstance(inspector, this.properties,
+ this.accumuloSerDeParams.getConf());
+ } catch (Exception e) {
+ throw new SerDeException(e);
+ }
+ }
+}
Added: hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/DefaultAccumuloRowIdFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/DefaultAccumuloRowIdFactory.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/DefaultAccumuloRowIdFactory.java (added)
+++ hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/DefaultAccumuloRowIdFactory.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.accumulo.serde;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.accumulo.Utils;
+import org.apache.hadoop.hive.accumulo.columns.ColumnEncoding;
+import org.apache.hadoop.hive.accumulo.columns.HiveAccumuloRowIdColumnMapping;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+
+/**
+ * Default implementation of the AccumuloRowIdFactory which uses the normal
+ * {@link AccumuloRowSerializer} methods to serialize the field for storage into Accumulo.
+ */
+public class DefaultAccumuloRowIdFactory implements AccumuloRowIdFactory {
+
+ protected AccumuloSerDeParameters accumuloSerDeParams;
+ protected LazySimpleSerDe.SerDeParameters serdeParams;
+ protected Properties properties;
+ protected HiveAccumuloRowIdColumnMapping rowIdMapping;
+ protected AccumuloRowSerializer serializer;
+
+ @Override
+ public void init(AccumuloSerDeParameters accumuloSerDeParams, Properties properties)
+ throws SerDeException {
+ this.accumuloSerDeParams = accumuloSerDeParams;
+ this.serdeParams = accumuloSerDeParams.getSerDeParameters();
+ this.properties = properties;
+ this.serializer = new AccumuloRowSerializer(accumuloSerDeParams.getRowIdOffset(), serdeParams,
+ accumuloSerDeParams.getColumnMappings(), accumuloSerDeParams.getTableVisibilityLabel(),
+ this);
+ this.rowIdMapping = accumuloSerDeParams.getRowIdColumnMapping();
+ }
+
+ @Override
+ public void addDependencyJars(Configuration conf) throws IOException {
+ Utils.addDependencyJars(conf, getClass());
+ }
+
+ @Override
+ public ObjectInspector createRowIdObjectInspector(TypeInfo type) throws SerDeException {
+ return LazyFactory.createLazyObjectInspector(type, serdeParams.getSeparators(), 1,
+ serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar());
+ }
+
+ @Override
+ public LazyObjectBase createRowId(ObjectInspector inspector) throws SerDeException {
+ // LazyObject can only be binary when it's not a string as well
+// return LazyFactory.createLazyObject(inspector,
+// ColumnEncoding.BINARY == rowIdMapping.getEncoding());
+ return LazyFactory.createLazyObject(inspector,
+ inspector.getTypeName() != TypeInfoFactory.stringTypeInfo.getTypeName()
+ && ColumnEncoding.BINARY == rowIdMapping.getEncoding());
+ }
+
+ @Override
+ public byte[] serializeRowId(Object object, StructField field, ByteStream.Output output)
+ throws IOException {
+ return serializer.serializeRowId(object, field, rowIdMapping);
+ }
+
+}
Added: hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/TooManyAccumuloColumnsException.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/TooManyAccumuloColumnsException.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/TooManyAccumuloColumnsException.java (added)
+++ hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/TooManyAccumuloColumnsException.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo.serde;
+
+import org.apache.hadoop.hive.serde2.SerDeException;
+
+/**
+ *
+ */
+public class TooManyAccumuloColumnsException extends SerDeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public TooManyAccumuloColumnsException() {
+ super();
+ }
+
+ public TooManyAccumuloColumnsException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TooManyAccumuloColumnsException(String message) {
+ super(message);
+ }
+
+ public TooManyAccumuloColumnsException(Throwable cause) {
+ super(cause);
+ }
+
+}
Added: hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/TooManyHiveColumnsException.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/TooManyHiveColumnsException.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/TooManyHiveColumnsException.java (added)
+++ hive/trunk/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/TooManyHiveColumnsException.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo.serde;
+
+import org.apache.hadoop.hive.serde2.SerDeException;
+
+/**
+ *
+ */
+public class TooManyHiveColumnsException extends SerDeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public TooManyHiveColumnsException() {
+ super();
+ }
+
+ public TooManyHiveColumnsException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TooManyHiveColumnsException(String message) {
+ super(message);
+ }
+
+ public TooManyHiveColumnsException(Throwable cause) {
+ super(cause);
+ }
+
+}
Added: hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloConnectionParameters.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloConnectionParameters.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloConnectionParameters.java (added)
+++ hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloConnectionParameters.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ *
+ */
+public class TestAccumuloConnectionParameters {
+
+ @Test
+ public void testInstantiatesWithNullConfiguration() {
+ // TableDesc#getDeserializer() passes a null Configuration into the SerDe.
+ // We shouldn't fail immediately in this case
+ AccumuloConnectionParameters cnxnParams = new AccumuloConnectionParameters(null);
+
+ // We should fail if we try to get info out of the params
+ try {
+ cnxnParams.getAccumuloInstanceName();
+ Assert.fail("Should have gotten an NPE");
+ } catch (NullPointerException e) {}
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMissingInstanceName() {
+ Configuration conf = new Configuration(false);
+ conf.set(AccumuloConnectionParameters.ZOOKEEPERS, "localhost:2181");
+ conf.set(AccumuloConnectionParameters.USER_NAME, "user");
+ conf.set(AccumuloConnectionParameters.USER_PASS, "password");
+
+ AccumuloConnectionParameters cnxnParams = new AccumuloConnectionParameters(conf);
+ cnxnParams.getInstance();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMissingZooKeepers() {
+ Configuration conf = new Configuration(false);
+ conf.set(AccumuloConnectionParameters.INSTANCE_NAME, "accumulo");
+ conf.set(AccumuloConnectionParameters.USER_NAME, "user");
+ conf.set(AccumuloConnectionParameters.USER_PASS, "password");
+
+ AccumuloConnectionParameters cnxnParams = new AccumuloConnectionParameters(conf);
+ cnxnParams.getInstance();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMissingUserName() throws AccumuloException, AccumuloSecurityException {
+ Configuration conf = new Configuration(false);
+ conf.set(AccumuloConnectionParameters.INSTANCE_NAME, "accumulo");
+ conf.set(AccumuloConnectionParameters.ZOOKEEPERS, "localhost:2181");
+ conf.set(AccumuloConnectionParameters.USER_PASS, "password");
+
+ Instance instance = Mockito.mock(Instance.class);
+
+ AccumuloConnectionParameters cnxnParams = new AccumuloConnectionParameters(conf);
+
+ // Provide an instance of the code doesn't try to make a real Instance
+ // We just want to test that we fail before trying to make a connector
+ // with null username
+ cnxnParams.getConnector(instance);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMissingPassword() throws AccumuloException, AccumuloSecurityException {
+ Configuration conf = new Configuration(false);
+ conf.set(AccumuloConnectionParameters.INSTANCE_NAME, "accumulo");
+ conf.set(AccumuloConnectionParameters.ZOOKEEPERS, "localhost:2181");
+ conf.set(AccumuloConnectionParameters.USER_NAME, "user");
+
+ Instance instance = Mockito.mock(Instance.class);
+
+ AccumuloConnectionParameters cnxnParams = new AccumuloConnectionParameters(conf);
+
+ // Provide an instance of the code doesn't try to make a real Instance
+ // We just want to test that we fail before trying to make a connector
+ // with null password
+ cnxnParams.getConnector(instance);
+ }
+}
Added: hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloHiveRow.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloHiveRow.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloHiveRow.java (added)
+++ hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloHiveRow.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+/**
+ * Test basic operations on AccumuloHiveRow
+ */
+public class TestAccumuloHiveRow {
+
+ @Test
+ public void testHasFamilyAndQualifier() {
+ AccumuloHiveRow row = new AccumuloHiveRow("row1");
+
+ // Add some columns
+ for (int i = 1; i <= 5; i++) {
+ row.add("cf1", "cq" + i, Integer.toString(i).getBytes());
+ }
+
+ // Check that we don't find unexpected columns
+ assertFalse(row.hasFamAndQual(new Text(""), new Text("")));
+ assertFalse(row.hasFamAndQual(new Text("cf0"), new Text("cq1")));
+ assertFalse(row.hasFamAndQual(new Text("cf1"), new Text("cq0")));
+
+ // Check that we do find all expected columns
+ for (int i = 1; i <= 5; i++) {
+ assertTrue(row.hasFamAndQual(new Text("cf1"), new Text("cq" + i)));
+ }
+ }
+
+ @Test
+ public void testGetValueFromColumn() {
+ AccumuloHiveRow row = new AccumuloHiveRow("row1");
+
+ // Should return null when there is no column
+ assertNull(row.getValue(new Text(""), new Text("")));
+
+ for (int i = 1; i <= 5; i++) {
+ row.add("cf", "cq" + i, Integer.toString(i).getBytes());
+ }
+
+ assertNull(row.getValue(new Text("cf"), new Text("cq0")));
+
+ for (int i = 1; i <= 5; i++) {
+ assertArrayEquals(Integer.toString(i).getBytes(),
+ row.getValue(new Text("cf"), new Text("cq" + i)));
+ }
+ }
+
+ @Test
+ public void testWritableEmptyRow() throws IOException {
+ AccumuloHiveRow emptyRow = new AccumuloHiveRow();
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ emptyRow.write(out);
+ out.close();
+
+ AccumuloHiveRow emptyCopy = new AccumuloHiveRow();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream in = new DataInputStream(bais);
+ emptyCopy.readFields(in);
+
+ assertEquals(emptyRow, emptyCopy);
+ }
+
+ @Test
+ public void testWritableWithColumns() throws IOException {
+ AccumuloHiveRow rowWithColumns = new AccumuloHiveRow("row");
+ rowWithColumns.add("cf", "cq1", "1".getBytes());
+ rowWithColumns.add("cf", "cq2", "2".getBytes());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ rowWithColumns.write(out);
+ out.close();
+
+ AccumuloHiveRow copy = new AccumuloHiveRow();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream in = new DataInputStream(bais);
+ copy.readFields(in);
+
+ assertEquals(rowWithColumns, copy);
+ }
+}
Added: hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java (added)
+++ hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,536 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.hadoop.hive.accumulo.columns.ColumnEncoding;
+import org.apache.hadoop.hive.accumulo.serde.AccumuloSerDeParameters;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.mockito.Mockito;
+
+/**
+ *
+ */
+public class TestAccumuloStorageHandler {
+
+ protected AccumuloStorageHandler storageHandler;
+
+ @Rule
+ public TestName test = new TestName();
+
+ @Before
+ public void setup() {
+ storageHandler = new AccumuloStorageHandler();
+ }
+
+ @Test
+ public void testTablePropertiesPassedToOutputJobProperties() {
+ TableDesc tableDesc = Mockito.mock(TableDesc.class);
+ Properties props = new Properties();
+ Map<String,String> jobProperties = new HashMap<String,String>();
+
+ props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq1,cf:cq2,cf:cq3");
+ props.setProperty(AccumuloSerDeParameters.TABLE_NAME, "table");
+ props.setProperty(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY, "foo");
+
+ Mockito.when(tableDesc.getProperties()).thenReturn(props);
+
+ storageHandler.configureOutputJobProperties(tableDesc, jobProperties);
+
+ Assert.assertEquals(3, jobProperties.size());
+ Assert.assertTrue("Job properties did not contain column mappings",
+ jobProperties.containsKey(AccumuloSerDeParameters.COLUMN_MAPPINGS));
+ Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS),
+ jobProperties.get(AccumuloSerDeParameters.COLUMN_MAPPINGS));
+
+ Assert.assertTrue("Job properties did not contain accumulo table name",
+ jobProperties.containsKey(AccumuloSerDeParameters.TABLE_NAME));
+ Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.TABLE_NAME),
+ jobProperties.get(AccumuloSerDeParameters.TABLE_NAME));
+
+ Assert.assertTrue("Job properties did not contain visibility label",
+ jobProperties.containsKey(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY));
+ Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY),
+ jobProperties.get(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY));
+ }
+
+ @Test
+ public void testTablePropertiesPassedToInputJobProperties() {
+ TableDesc tableDesc = Mockito.mock(TableDesc.class);
+ Properties props = new Properties();
+ Map<String,String> jobProperties = new HashMap<String,String>();
+
+ props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq1,cf:cq2,cf:cq3");
+ props.setProperty(AccumuloSerDeParameters.TABLE_NAME, "table");
+ props.setProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY, "true");
+ props
+ .setProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE, ColumnEncoding.BINARY.getName());
+ props.setProperty(AccumuloSerDeParameters.AUTHORIZATIONS_KEY, "foo,bar");
+
+ Mockito.when(tableDesc.getProperties()).thenReturn(props);
+
+ storageHandler.configureInputJobProperties(tableDesc, jobProperties);
+
+ Assert.assertEquals(5, jobProperties.size());
+
+ Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.COLUMN_MAPPINGS));
+ Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS),
+ jobProperties.get(AccumuloSerDeParameters.COLUMN_MAPPINGS));
+
+ Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.TABLE_NAME));
+ Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.TABLE_NAME),
+ jobProperties.get(AccumuloSerDeParameters.TABLE_NAME));
+
+ Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY));
+ Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY),
+ jobProperties.get(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY));
+
+ Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE));
+ Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
+ jobProperties.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE));
+
+ Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.AUTHORIZATIONS_KEY));
+ Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.AUTHORIZATIONS_KEY),
+ jobProperties.get(AccumuloSerDeParameters.AUTHORIZATIONS_KEY));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNonBooleanIteratorPushdownValue() {
+ TableDesc tableDesc = Mockito.mock(TableDesc.class);
+ Properties props = new Properties();
+ Map<String,String> jobProperties = new HashMap<String,String>();
+
+ props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq1,cf:cq2,cf:cq3");
+ props.setProperty(AccumuloSerDeParameters.TABLE_NAME, "table");
+ props.setProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY, "foo");
+
+ Mockito.when(tableDesc.getProperties()).thenReturn(props);
+
+ storageHandler.configureInputJobProperties(tableDesc, jobProperties);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testEmptyIteratorPushdownValue() {
+ TableDesc tableDesc = Mockito.mock(TableDesc.class);
+ Properties props = new Properties();
+ Map<String,String> jobProperties = new HashMap<String,String>();
+
+ props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq1,cf:cq2,cf:cq3");
+ props.setProperty(AccumuloSerDeParameters.TABLE_NAME, "table");
+ props.setProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY, "");
+
+ Mockito.when(tableDesc.getProperties()).thenReturn(props);
+
+ storageHandler.configureInputJobProperties(tableDesc, jobProperties);
+ }
+
+ @Test
+ public void testTableJobPropertiesCallsInputAndOutputMethods() {
+ AccumuloStorageHandler mockStorageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ TableDesc tableDesc = Mockito.mock(TableDesc.class);
+ Map<String,String> jobProperties = new HashMap<String,String>();
+
+ Mockito.doCallRealMethod().when(mockStorageHandler)
+ .configureTableJobProperties(tableDesc, jobProperties);
+
+ // configureTableJobProperties shouldn't be getting called by Hive, but, if it somehow does,
+ // we should just set all of the configurations for input and output.
+ mockStorageHandler.configureTableJobProperties(tableDesc, jobProperties);
+
+ Mockito.verify(mockStorageHandler).configureInputJobProperties(tableDesc, jobProperties);
+ Mockito.verify(mockStorageHandler).configureOutputJobProperties(tableDesc, jobProperties);
+ }
+
+ @Test
+ public void testPreCreateTable() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ String tableName = "table";
+
+ // Define the SerDe Parameters
+ Map<String,String> params = new HashMap<String,String>();
+ params.put(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq");
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+ Table table = Mockito.mock(Table.class);
+ SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Not an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+
+ // Return the mocked StorageDescriptor
+ Mockito.when(table.getSd()).thenReturn(sd);
+
+ // No location expected with AccumuloStorageHandler
+ Mockito.when(sd.getLocation()).thenReturn(null);
+
+ // Return mocked SerDeInfo
+ Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
+
+ // Custom parameters
+ Mockito.when(serDeInfo.getParameters()).thenReturn(params);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.preCreateTable(table);
+
+ Assert.assertTrue("Table does not exist when we expect it to",
+ conn.tableOperations().exists(tableName));
+ }
+
+ @Test(expected = MetaException.class)
+ public void testMissingColumnMappingFails() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ String tableName = "table";
+
+ // Empty parameters are sent, no COLUMN_MAPPING
+ Map<String,String> params = new HashMap<String,String>();
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+ Table table = Mockito.mock(Table.class);
+ SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Not an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+
+ // Return the mocked StorageDescriptor
+ Mockito.when(table.getSd()).thenReturn(sd);
+
+ // No location expected with AccumuloStorageHandler
+ Mockito.when(sd.getLocation()).thenReturn(null);
+
+ // Return mocked SerDeInfo
+ Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
+
+ // Custom parameters
+ Mockito.when(serDeInfo.getParameters()).thenReturn(params);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.preCreateTable(table);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testNonNullLocation() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ String tableName = "table";
+
+ // Empty parameters are sent, no COLUMN_MAPPING
+ Map<String,String> params = new HashMap<String,String>();
+ params.put(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq");
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+ Table table = Mockito.mock(Table.class);
+ SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Not an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+
+ // Return the mocked StorageDescriptor
+ Mockito.when(table.getSd()).thenReturn(sd);
+
+ // No location expected with AccumuloStorageHandler
+ Mockito.when(sd.getLocation()).thenReturn("foobar");
+
+ // Return mocked SerDeInfo
+ Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
+
+ // Custom parameters
+ Mockito.when(serDeInfo.getParameters()).thenReturn(params);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.preCreateTable(table);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testExternalNonExistentTableFails() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ String tableName = "table";
+
+ // Define the SerDe Parameters
+ Map<String,String> params = new HashMap<String,String>();
+ params.put(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq");
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+ Table table = Mockito.mock(Table.class);
+ SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Is an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(true);
+
+ // Return the mocked StorageDescriptor
+ Mockito.when(table.getSd()).thenReturn(sd);
+
+ // No location expected with AccumuloStorageHandler
+ Mockito.when(sd.getLocation()).thenReturn(null);
+
+ // Return mocked SerDeInfo
+ Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
+
+ // Custom parameters
+ Mockito.when(serDeInfo.getParameters()).thenReturn(params);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.preCreateTable(table);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testNonExternalExistentTable() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ String tableName = "table";
+
+ // Create the table
+ conn.tableOperations().create(tableName);
+
+ // Define the SerDe Parameters
+ Map<String,String> params = new HashMap<String,String>();
+ params.put(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq");
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+ Table table = Mockito.mock(Table.class);
+ SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Is not an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+
+ // Return the mocked StorageDescriptor
+ Mockito.when(table.getSd()).thenReturn(sd);
+
+ // No location expected with AccumuloStorageHandler
+ Mockito.when(sd.getLocation()).thenReturn(null);
+
+ // Return mocked SerDeInfo
+ Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
+
+ // Custom parameters
+ Mockito.when(serDeInfo.getParameters()).thenReturn(params);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.preCreateTable(table);
+ }
+
+ @Test()
+ public void testRollbackCreateTableOnNonExistentTable() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ String tableName = "table";
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ Table table = Mockito.mock(Table.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).rollbackCreateTable(table);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Is not an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.rollbackCreateTable(table);
+ }
+
+ @Test()
+ public void testRollbackCreateTableDeletesExistentTable() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ String tableName = "table";
+
+ // Create the table
+ conn.tableOperations().create(tableName);
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ Table table = Mockito.mock(Table.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).rollbackCreateTable(table);
+ Mockito.doCallRealMethod().when(storageHandler).commitDropTable(table, true);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Is not an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.rollbackCreateTable(table);
+
+ Assert.assertFalse(conn.tableOperations().exists(tableName));
+ }
+
+ @Test()
+ public void testRollbackCreateTableDoesntDeleteExternalExistentTable() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ String tableName = "table";
+
+ // Create the table
+ conn.tableOperations().create(tableName);
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ Table table = Mockito.mock(Table.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).rollbackCreateTable(table);
+ Mockito.doCallRealMethod().when(storageHandler).commitDropTable(table, true);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Is not an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(true);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.rollbackCreateTable(table);
+
+ Assert.assertTrue(conn.tableOperations().exists(tableName));
+ }
+
+ @Test
+ public void testDropTableWithoutDeleteLeavesTableIntact() throws Exception {
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
+ String tableName = "table";
+
+ // Create the table
+ conn.tableOperations().create(tableName);
+
+ AccumuloConnectionParameters connectionParams = Mockito
+ .mock(AccumuloConnectionParameters.class);
+ Table table = Mockito.mock(Table.class);
+
+ // Call the real preCreateTable method
+ Mockito.doCallRealMethod().when(storageHandler).commitDropTable(table, false);
+
+ // Return our known table name
+ Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
+
+ // Is not an EXTERNAL table
+ Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+
+ // Return the MockInstance's Connector
+ Mockito.when(connectionParams.getConnector()).thenReturn(conn);
+
+ storageHandler.connectionParams = connectionParams;
+
+ storageHandler.rollbackCreateTable(table);
+
+ Assert.assertTrue(conn.tableOperations().exists(tableName));
+ }
+}
Added: hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloMap.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloMap.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloMap.java (added)
+++ hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloMap.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.hadoop.hive.accumulo.columns.ColumnEncoding;
+import org.apache.hadoop.hive.accumulo.columns.HiveAccumuloMapColumnMapping;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyInteger;
+import org.apache.hadoop.hive.serde2.lazy.LazyString;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestLazyAccumuloMap {
+
+ protected byte[] toBytes(int i) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ out.writeInt(i);
+ out.close();
+ return baos.toByteArray();
+ }
+
+ @Test
+ public void testStringMapWithProjection() throws SerDeException {
+ AccumuloHiveRow row = new AccumuloHiveRow("row");
+
+ row.add("cf1", "foo", "bar".getBytes());
+ row.add("cf1", "bar", "foo".getBytes());
+
+ row.add("cf2", "foo1", "bar1".getBytes());
+ row.add("cf3", "bar1", "foo1".getBytes());
+
+ HiveAccumuloMapColumnMapping mapping = new HiveAccumuloMapColumnMapping("cf1", null,
+ ColumnEncoding.STRING, ColumnEncoding.STRING, "column", TypeInfoFactory.getMapTypeInfo(
+ TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo).toString());
+
+ // Map of Integer to String
+ Text nullSequence = new Text("\\N");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils
+ .getTypeInfosFromTypeString("map<string,string>").get(0), new byte[] {(byte) 1, (byte) 2},
+ 0, nullSequence, false, (byte) 0);
+
+ LazyAccumuloMap map = new LazyAccumuloMap((LazyMapObjectInspector) oi);
+ map.init(row, mapping);
+
+ Assert.assertEquals(2, map.getMapSize());
+
+ Object o = map.getMapValueElement(new Text("foo"));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new Text("bar"), ((LazyString) o).getWritableObject());
+
+ o = map.getMapValueElement(new Text("bar"));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new Text("foo"), ((LazyString) o).getWritableObject());
+ }
+
+ @Test
+ public void testIntMap() throws SerDeException, IOException {
+ AccumuloHiveRow row = new AccumuloHiveRow("row");
+
+ row.add(new Text("cf1"), new Text("1"), "2".getBytes());
+ row.add(new Text("cf1"), new Text("2"), "4".getBytes());
+ row.add(new Text("cf1"), new Text("3"), "6".getBytes());
+
+ HiveAccumuloMapColumnMapping mapping = new HiveAccumuloMapColumnMapping("cf1", null,
+ ColumnEncoding.STRING, ColumnEncoding.STRING, "column", TypeInfoFactory.getMapTypeInfo(
+ TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo).toString());
+
+ // Map of Integer to Integer
+ Text nullSequence = new Text("\\N");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils
+ .getTypeInfosFromTypeString("map<int,int>").get(0), new byte[] {(byte) 1, (byte) 2}, 0,
+ nullSequence, false, (byte) 0);
+
+ LazyAccumuloMap map = new LazyAccumuloMap((LazyMapObjectInspector) oi);
+ map.init(row, mapping);
+
+ Assert.assertEquals(3, map.getMapSize());
+
+ Object o = map.getMapValueElement(new IntWritable(1));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(2), ((LazyInteger) o).getWritableObject());
+
+ o = map.getMapValueElement(new IntWritable(2));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(4), ((LazyInteger) o).getWritableObject());
+
+ o = map.getMapValueElement(new IntWritable(3));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(6), ((LazyInteger) o).getWritableObject());
+ }
+
+ @Test
+ public void testBinaryIntMap() throws SerDeException, IOException {
+ AccumuloHiveRow row = new AccumuloHiveRow("row");
+
+ row.add(new Text("cf1"), new Text(toBytes(1)), toBytes(2));
+ row.add(new Text("cf1"), new Text(toBytes(2)), toBytes(4));
+ row.add(new Text("cf1"), new Text(toBytes(3)), toBytes(6));
+
+ HiveAccumuloMapColumnMapping mapping = new HiveAccumuloMapColumnMapping("cf1", null,
+ ColumnEncoding.BINARY, ColumnEncoding.BINARY, "column", TypeInfoFactory.getMapTypeInfo(
+ TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo).toString());
+
+ // Map of Integer to String
+ Text nullSequence = new Text("\\N");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils
+ .getTypeInfosFromTypeString("map<int,int>").get(0), new byte[] {(byte) 1, (byte) 2}, 0,
+ nullSequence, false, (byte) 0);
+
+ LazyAccumuloMap map = new LazyAccumuloMap((LazyMapObjectInspector) oi);
+ map.init(row, mapping);
+
+ Assert.assertEquals(3, map.getMapSize());
+
+ Object o = map.getMapValueElement(new IntWritable(1));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(2), ((LazyInteger) o).getWritableObject());
+
+ o = map.getMapValueElement(new IntWritable(2));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(4), ((LazyInteger) o).getWritableObject());
+
+ o = map.getMapValueElement(new IntWritable(3));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(6), ((LazyInteger) o).getWritableObject());
+ }
+
+ @Test
+ public void testMixedSerializationMap() throws SerDeException, IOException {
+ AccumuloHiveRow row = new AccumuloHiveRow("row");
+
+ row.add(new Text("cf1"), new Text(toBytes(1)), "2".getBytes());
+ row.add(new Text("cf1"), new Text(toBytes(2)), "4".getBytes());
+ row.add(new Text("cf1"), new Text(toBytes(3)), "6".getBytes());
+
+ HiveAccumuloMapColumnMapping mapping = new HiveAccumuloMapColumnMapping("cf1", null,
+ ColumnEncoding.BINARY, ColumnEncoding.STRING, "column", TypeInfoFactory.getMapTypeInfo(
+ TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo).toString());
+
+ // Map of Integer to String
+ Text nullSequence = new Text("\\N");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils
+ .getTypeInfosFromTypeString("map<int,int>").get(0), new byte[] {(byte) 1, (byte) 2}, 0,
+ nullSequence, false, (byte) 0);
+
+ LazyAccumuloMap map = new LazyAccumuloMap((LazyMapObjectInspector) oi);
+ map.init(row, mapping);
+
+ Assert.assertEquals(3, map.getMapSize());
+
+ Object o = map.getMapValueElement(new IntWritable(1));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(2), ((LazyInteger) o).getWritableObject());
+
+ o = map.getMapValueElement(new IntWritable(2));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(4), ((LazyInteger) o).getWritableObject());
+
+ o = map.getMapValueElement(new IntWritable(3));
+ Assert.assertNotNull(o);
+ Assert.assertEquals(new IntWritable(6), ((LazyInteger) o).getWritableObject());
+ }
+
+}
Added: hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloRow.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloRow.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloRow.java (added)
+++ hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloRow.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.accumulo.columns.ColumnEncoding;
+import org.apache.hadoop.hive.accumulo.columns.ColumnMapper;
+import org.apache.hadoop.hive.accumulo.serde.AccumuloSerDe;
+import org.apache.hadoop.hive.accumulo.serde.AccumuloSerDeParameters;
+import org.apache.hadoop.hive.accumulo.serde.DefaultAccumuloRowIdFactory;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyInteger;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazyString;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
+import org.apache.hadoop.hive.serde2.lazydio.LazyDioInteger;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.base.Joiner;
+
+/**
+ *
+ */
+public class TestLazyAccumuloRow {
+
+ @Test
+ public void testExpectedDeserializationOfColumns() throws Exception {
+ List<String> columns = Arrays.asList("row", "given_name", "surname", "age", "weight", "height");
+ List<TypeInfo> types = Arrays.<TypeInfo> asList(TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo);
+
+ LazySimpleStructObjectInspector objectInspector = (LazySimpleStructObjectInspector) LazyFactory
+ .createLazyStructInspector(columns, types, LazySimpleSerDe.DefaultSeparators, new Text(
+ "\\N"), false, false, (byte) '\\');
+
+ DefaultAccumuloRowIdFactory rowIdFactory = new DefaultAccumuloRowIdFactory();
+
+ Properties props = new Properties();
+ props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,
+ ":rowid,personal:given_name,personal:surname,personal:age,personal:weight,personal:height");
+ props.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columns));
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(',').join(types));
+
+ AccumuloSerDeParameters params = new AccumuloSerDeParameters(new Configuration(), props,
+ AccumuloSerDe.class.getName());
+
+ rowIdFactory.init(params, props);
+
+ LazyAccumuloRow lazyRow = new LazyAccumuloRow(objectInspector);
+ AccumuloHiveRow hiveRow = new AccumuloHiveRow("1");
+ hiveRow.add("personal", "given_name", "Bob".getBytes());
+ hiveRow.add("personal", "surname", "Stevens".getBytes());
+ hiveRow.add("personal", "age", "30".getBytes());
+ hiveRow.add("personal", "weight", "200".getBytes());
+ hiveRow.add("personal", "height", "72".getBytes());
+
+ ColumnMapper columnMapper = params.getColumnMapper();
+
+ lazyRow.init(hiveRow, columnMapper.getColumnMappings(), rowIdFactory);
+
+ Object o = lazyRow.getField(0);
+ Assert.assertEquals(LazyString.class, o.getClass());
+ Assert.assertEquals("1", ((LazyString) o).toString());
+
+ o = lazyRow.getField(1);
+ Assert.assertEquals(LazyString.class, o.getClass());
+ Assert.assertEquals("Bob", ((LazyString) o).toString());
+
+ o = lazyRow.getField(2);
+ Assert.assertEquals(LazyString.class, o.getClass());
+ Assert.assertEquals("Stevens", ((LazyString) o).toString());
+
+ o = lazyRow.getField(3);
+ Assert.assertEquals(LazyInteger.class, o.getClass());
+ Assert.assertEquals("30", ((LazyInteger) o).toString());
+
+ o = lazyRow.getField(4);
+ Assert.assertEquals(LazyInteger.class, o.getClass());
+ Assert.assertEquals("200", ((LazyInteger) o).toString());
+
+ o = lazyRow.getField(5);
+ Assert.assertEquals(LazyInteger.class, o.getClass());
+ Assert.assertEquals("72", ((LazyInteger) o).toString());
+ }
+
+ @Test
+ public void testDeserializationOfBinaryEncoding() throws Exception {
+ List<String> columns = Arrays.asList("row", "given_name", "surname", "age", "weight", "height");
+ List<TypeInfo> types = Arrays.<TypeInfo> asList(TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo,
+ TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo);
+
+ LazySimpleStructObjectInspector objectInspector = (LazySimpleStructObjectInspector) LazyFactory
+ .createLazyStructInspector(columns, types, LazySimpleSerDe.DefaultSeparators, new Text(
+ "\\N"), false, false, (byte) '\\');
+
+ DefaultAccumuloRowIdFactory rowIdFactory = new DefaultAccumuloRowIdFactory();
+
+ Properties props = new Properties();
+ props
+ .setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,
+ ":rowid#s,personal:given_name#s,personal:surname#s,personal:age,personal:weight,personal:height");
+ props.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columns));
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(',').join(types));
+ props
+ .setProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE, ColumnEncoding.BINARY.getName());
+
+ AccumuloSerDeParameters params = new AccumuloSerDeParameters(new Configuration(), props,
+ AccumuloSerDe.class.getName());
+
+ rowIdFactory.init(params, props);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+
+ LazyAccumuloRow lazyRow = new LazyAccumuloRow(objectInspector);
+ AccumuloHiveRow hiveRow = new AccumuloHiveRow("1");
+ hiveRow.add("personal", "given_name", "Bob".getBytes());
+ hiveRow.add("personal", "surname", "Stevens".getBytes());
+
+ out.writeInt(30);
+ hiveRow.add("personal", "age", baos.toByteArray());
+
+ baos.reset();
+ out.writeInt(200);
+ hiveRow.add("personal", "weight", baos.toByteArray());
+
+ baos.reset();
+ out.writeInt(72);
+ hiveRow.add("personal", "height", baos.toByteArray());
+
+ ColumnMapper columnMapper = params.getColumnMapper();
+
+ lazyRow.init(hiveRow, columnMapper.getColumnMappings(), rowIdFactory);
+
+ Object o = lazyRow.getField(0);
+ Assert.assertNotNull(o);
+ Assert.assertEquals(LazyString.class, o.getClass());
+ Assert.assertEquals("1", ((LazyString) o).toString());
+
+ o = lazyRow.getField(1);
+ Assert.assertNotNull(o);
+ Assert.assertEquals(LazyString.class, o.getClass());
+ Assert.assertEquals("Bob", ((LazyString) o).toString());
+
+ o = lazyRow.getField(2);
+ Assert.assertNotNull(o);
+ Assert.assertEquals(LazyString.class, o.getClass());
+ Assert.assertEquals("Stevens", ((LazyString) o).toString());
+
+ o = lazyRow.getField(3);
+ Assert.assertNotNull(o);
+ Assert.assertEquals(LazyDioInteger.class, o.getClass());
+ Assert.assertEquals("30", ((LazyDioInteger) o).toString());
+
+ o = lazyRow.getField(4);
+ Assert.assertNotNull(o);
+ Assert.assertEquals(LazyDioInteger.class, o.getClass());
+ Assert.assertEquals("200", ((LazyDioInteger) o).toString());
+
+ o = lazyRow.getField(5);
+ Assert.assertNotNull(o);
+ Assert.assertEquals(LazyDioInteger.class, o.getClass());
+ Assert.assertEquals("72", ((LazyDioInteger) o).toString());
+ }
+
+ @Test
+ public void testNullInit() throws SerDeException {
+ List<String> columns = Arrays.asList("row", "1", "2", "3");
+ List<TypeInfo> types = Arrays.<TypeInfo> asList(
+ TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME),
+ TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME),
+ TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME),
+ TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME));
+
+ LazySimpleStructObjectInspector objectInspector = (LazySimpleStructObjectInspector) LazyFactory
+ .createLazyStructInspector(columns, types, LazySimpleSerDe.DefaultSeparators, new Text(
+ "\\N"), false, false, (byte) '\\');
+
+ DefaultAccumuloRowIdFactory rowIdFactory = new DefaultAccumuloRowIdFactory();
+
+ Properties props = new Properties();
+ props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, ":rowid,cf:cq1,cf:cq2,cf:cq3");
+ props.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columns));
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(',').join(types));
+
+ AccumuloSerDeParameters params = new AccumuloSerDeParameters(new Configuration(), props,
+ AccumuloSerDe.class.getName());
+
+ rowIdFactory.init(params, props);
+
+ ColumnMapper columnMapper = params.getColumnMapper();
+
+ LazyAccumuloRow lazyRow = new LazyAccumuloRow(objectInspector);
+ AccumuloHiveRow hiveRow = new AccumuloHiveRow("1");
+ hiveRow.add("cf", "cq1", "foo".getBytes());
+ hiveRow.add("cf", "cq3", "bar".getBytes());
+
+ lazyRow.init(hiveRow, columnMapper.getColumnMappings(), rowIdFactory);
+
+ // Noticed that we also suffer from the same issue as HIVE-3179
+ // Only want to call a field init'ed when it's non-NULL
+ // Check it twice, make sure we get null both times
+ Assert.assertEquals("{'row':'1','1':'foo','2':null,'3':'bar'}".replace('\'', '"'),
+ SerDeUtils.getJSONString(lazyRow, objectInspector));
+ Assert.assertEquals("{'row':'1','1':'foo','2':null,'3':'bar'}".replace('\'', '"'),
+ SerDeUtils.getJSONString(lazyRow, objectInspector));
+ }
+}
Added: hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnEncoding.java
URL: http://svn.apache.org/viewvc/hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnEncoding.java?rev=1619005&view=auto
==============================================================================
--- hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnEncoding.java (added)
+++ hive/trunk/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnEncoding.java Tue Aug 19 22:41:10 2014
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.accumulo.columns;
+
+import java.util.Map.Entry;
+
+import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+/**
+ *
+ */
+public class TestColumnEncoding {
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidCodeThrowsException() {
+ ColumnEncoding.fromCode("foo");
+ }
+
+ @Test
+ public void testStringEncoding() {
+ Assert.assertEquals(ColumnEncoding.STRING, ColumnEncoding.fromCode("s"));
+ }
+
+ @Test
+ public void testBinaryEncoding() {
+ Assert.assertEquals(ColumnEncoding.BINARY, ColumnEncoding.fromCode("b"));
+ }
+
+ @Test
+ public void testMissingColumnEncoding() {
+ Assert.assertFalse(ColumnEncoding.hasColumnEncoding("foo:bar"));
+ }
+
+ @Test
+ public void testColumnEncodingSpecified() {
+ Assert.assertTrue(ColumnEncoding.hasColumnEncoding("foo:bar#s"));
+ }
+
+ @Test
+ public void testEscapedPoundIsNoEncodingSpecified() {
+ Assert.assertFalse(ColumnEncoding.hasColumnEncoding("foo:b\\#ar"));
+ }
+
+ @Test
+ public void testEscapedPoundWithRealPound() {
+ Assert.assertTrue(ColumnEncoding.hasColumnEncoding("foo:b\\#ar#b"));
+ }
+
+ @Test
+ public void testParse() {
+ Assert.assertEquals(ColumnEncoding.STRING, ColumnEncoding.getFromMapping("foo:bar#s"));
+ }
+
+ @Test
+ public void testParseWithEscapedPound() {
+ Assert.assertEquals(ColumnEncoding.BINARY, ColumnEncoding.getFromMapping("fo\\#o:bar#b"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testMissingEncodingOnParse() {
+ ColumnEncoding.getFromMapping("foo:bar");
+ }
+
+ @Test
+ public void testStripCode() {
+ String mapping = "foo:bar";
+ Assert.assertEquals(
+ mapping,
+ ColumnEncoding.stripCode(mapping + AccumuloHiveConstants.POUND
+ + ColumnEncoding.BINARY.getCode()));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testStripNonExistentCodeFails() {
+ ColumnEncoding.stripCode("foo:bar");
+ }
+
+ @Test
+ public void testStripCodeWithEscapedPound() {
+ String mapping = "foo:ba\\#r";
+
+ Assert.assertEquals(
+ mapping,
+ ColumnEncoding.stripCode(mapping + AccumuloHiveConstants.POUND
+ + ColumnEncoding.BINARY.getCode()));
+ }
+
+ @Test
+ public void testMapEncoding() {
+ Assert.assertFalse(ColumnEncoding.isMapEncoding("s"));
+ Assert.assertFalse(ColumnEncoding.isMapEncoding("string"));
+ Assert.assertFalse(ColumnEncoding.isMapEncoding("binary"));
+
+ Assert.assertTrue(ColumnEncoding.isMapEncoding("s:s"));
+ Assert.assertTrue(ColumnEncoding.isMapEncoding("s:string"));
+ Assert.assertTrue(ColumnEncoding.isMapEncoding("string:s"));
+ Assert.assertTrue(ColumnEncoding.isMapEncoding("string:string"));
+ }
+
+ @Test
+ public void testMapEncodingParsing() {
+ Entry<ColumnEncoding,ColumnEncoding> stringString = Maps.immutableEntry(ColumnEncoding.STRING,
+ ColumnEncoding.STRING), stringBinary = Maps.immutableEntry(ColumnEncoding.STRING,
+ ColumnEncoding.BINARY), binaryBinary = Maps.immutableEntry(ColumnEncoding.BINARY,
+ ColumnEncoding.BINARY), binaryString = Maps.immutableEntry(ColumnEncoding.BINARY,
+ ColumnEncoding.STRING);
+
+ Assert.assertEquals(stringString, ColumnEncoding.getMapEncoding("s:s"));
+ Assert.assertEquals(stringString, ColumnEncoding.getMapEncoding("s:string"));
+ Assert.assertEquals(stringString, ColumnEncoding.getMapEncoding("string:s"));
+ Assert.assertEquals(stringString, ColumnEncoding.getMapEncoding("string:string"));
+
+ Assert.assertEquals(stringBinary, ColumnEncoding.getMapEncoding("s:b"));
+ Assert.assertEquals(stringBinary, ColumnEncoding.getMapEncoding("string:b"));
+ Assert.assertEquals(stringBinary, ColumnEncoding.getMapEncoding("s:binary"));
+ Assert.assertEquals(stringBinary, ColumnEncoding.getMapEncoding("string:binary"));
+
+ Assert.assertEquals(binaryString, ColumnEncoding.getMapEncoding("b:s"));
+ Assert.assertEquals(binaryString, ColumnEncoding.getMapEncoding("b:string"));
+ Assert.assertEquals(binaryString, ColumnEncoding.getMapEncoding("binary:s"));
+ Assert.assertEquals(binaryString, ColumnEncoding.getMapEncoding("binary:string"));
+
+ Assert.assertEquals(binaryBinary, ColumnEncoding.getMapEncoding("b:b"));
+ Assert.assertEquals(binaryBinary, ColumnEncoding.getMapEncoding("binary:b"));
+ Assert.assertEquals(binaryBinary, ColumnEncoding.getMapEncoding("b:binary"));
+ Assert.assertEquals(binaryBinary, ColumnEncoding.getMapEncoding("binary:binary"));
+ }
+}