You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2011/04/12 17:30:12 UTC
svn commit: r1091509 [3/8] - in /incubator/hcatalog/trunk: ./ bin/ ivy/ src/
src/docs/ src/docs/src/ src/docs/src/documentation/
src/docs/src/documentation/classes/ src/docs/src/documentation/conf/
src/docs/src/documentation/content/ src/docs/src/docum...
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/AuthUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/AuthUtils.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/AuthUtils.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/AuthUtils.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.common;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import javax.security.auth.login.LoginException;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+
+public class AuthUtils {
+
+ /**
+ * @param path non-null
+ * @param action non-null
+ * @param conf
+ * @throws SemanticException
+ * @throws HCatException
+ *
+ * This method validates only for existing path. If path doesn't exist
+ * there is nothing to validate. So, make sure that path passed in is non-null.
+ */
+
+ @SuppressWarnings("deprecation")
+ public static void authorize(final Path path, final FsAction action, final Configuration conf) throws SemanticException, HCatException{
+
+ if(path == null) {
+ throw new HCatException(ErrorType.ERROR_INTERNAL_EXCEPTION);
+ }
+ final FileStatus stat;
+
+ try {
+ stat = path.getFileSystem(conf).getFileStatus(path);
+ } catch (FileNotFoundException fnfe){
+ // File named by path doesn't exist; nothing to validate.
+ return;
+ }
+ catch (AccessControlException ace) {
+ throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL, ace);
+ } catch (org.apache.hadoop.fs.permission.AccessControlException ace){
+ // Older hadoop version will throw this @deprecated Exception.
+ throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL, ace);
+ } catch (IOException ioe){
+ throw new SemanticException(ioe);
+ }
+
+ final UserGroupInformation ugi;
+ try {
+ ugi = ShimLoader.getHadoopShims().getUGIForConf(conf);
+ } catch (LoginException le) {
+ throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL,le);
+ } catch (IOException ioe) {
+ throw new SemanticException(ioe);
+ }
+
+ final FsPermission dirPerms = stat.getPermission();
+
+ final String user = HiveConf.getBoolVar(conf, ConfVars.METASTORE_USE_THRIFT_SASL) ?
+ ugi.getShortUserName() : ugi.getUserName();
+ final String grp = stat.getGroup();
+ if(user.equals(stat.getOwner())){
+ if(dirPerms.getUserAction().implies(action)){
+ return;
+ }
+ throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL);
+ }
+ if(ArrayUtils.contains(ugi.getGroupNames(), grp)){
+ if(dirPerms.getGroupAction().implies(action)){
+ return;
+ }
+ throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL);
+
+ }
+ if(dirPerms.getOtherAction().implies(action)){
+ return;
+ }
+ throw new HCatException(ErrorType.ERROR_ACCESS_CONTROL);
+
+
+ }
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/ErrorType.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/ErrorType.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/ErrorType.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/ErrorType.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.common;
+
+/**
+ * Enum type representing the various errors throws by Howl.
+ */
+public enum ErrorType {
+
+ /* Howl Input Format related errors 1000 - 1999 */
+ ERROR_DB_INIT (1000, "Error initializing database session"),
+ ERROR_EXCEED_MAXPART (1001, "Query result exceeded maximum number of partitions allowed"),
+
+
+ /* Howl Output Format related errors 2000 - 2999 */
+ ERROR_INVALID_TABLE (2000, "Table specified does not exist"),
+ ERROR_SET_OUTPUT (2001, "Error setting output information"),
+ ERROR_DUPLICATE_PARTITION (2002, "Partition already present with given partition key values"),
+ ERROR_NON_EMPTY_TABLE (2003, "Non-partitioned table already contains data"),
+ ERROR_NOT_INITIALIZED (2004, "HowlOutputFormat not initialized, setOutput has to be called"),
+ ERROR_INIT_STORAGE_DRIVER (2005, "Error initializing output storage driver instance"),
+ ERROR_PUBLISHING_PARTITION (2006, "Error adding partition to metastore"),
+ ERROR_SCHEMA_COLUMN_MISMATCH (2007, "Invalid column position in partition schema"),
+ ERROR_SCHEMA_PARTITION_KEY (2008, "Partition key cannot be present in the partition data"),
+ ERROR_SCHEMA_TYPE_MISMATCH (2009, "Invalid column type in partition schema"),
+ ERROR_INVALID_PARTITION_VALUES (2010, "Invalid partition values specified"),
+ ERROR_MISSING_PARTITION_KEY (2011, "Partition key value not provided for publish"),
+ ERROR_MOVE_FAILED (2012, "Moving of data failed during commit"),
+
+
+ /* Authorization Errors 3000 - 3999 */
+ ERROR_ACCESS_CONTROL (3000, "Permission denied"),
+
+ /* Miscellaneous errors, range 9000 - 9998 */
+ ERROR_UNIMPLEMENTED (9000, "Functionality currently unimplemented"),
+ ERROR_INTERNAL_EXCEPTION (9001, "Exception occurred while processing Howl request");
+
+ /** The error code. */
+ private int errorCode;
+
+ /** The error message. */
+ private String errorMessage;
+
+ /** Should the causal exception message be appended to the error message, yes by default*/
+ private boolean appendCauseMessage = true;
+
+ /** Is this a retriable error, no by default. */
+ private boolean isRetriable = false;
+
+ /**
+ * Instantiates a new error type.
+ * @param errorCode the error code
+ * @param errorMessage the error message
+ */
+ private ErrorType(int errorCode, String errorMessage) {
+ this.errorCode = errorCode;
+ this.errorMessage = errorMessage;
+ }
+
+ /**
+ * Instantiates a new error type.
+ * @param errorCode the error code
+ * @param errorMessage the error message
+ * @param appendCauseMessage should causal exception message be appended to error message
+ */
+ private ErrorType(int errorCode, String errorMessage, boolean appendCauseMessage) {
+ this.errorCode = errorCode;
+ this.errorMessage = errorMessage;
+ this.appendCauseMessage = appendCauseMessage;
+ }
+
+ /**
+ * Instantiates a new error type.
+ * @param errorCode the error code
+ * @param errorMessage the error message
+ * @param appendCauseMessage should causal exception message be appended to error message
+ * @param isRetriable is this a retriable error
+ */
+ private ErrorType(int errorCode, String errorMessage, boolean appendCauseMessage, boolean isRetriable) {
+ this.errorCode = errorCode;
+ this.errorMessage = errorMessage;
+ this.appendCauseMessage = appendCauseMessage;
+ this.isRetriable = isRetriable;
+ }
+
+ /**
+ * Gets the error code.
+ * @return the error code
+ */
+ public int getErrorCode() {
+ return errorCode;
+ }
+
+ /**
+ * Gets the error message.
+ * @return the error message
+ */
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+ /**
+ * Checks if this is a retriable error.
+ * @return true, if is a retriable error, false otherwise
+ */
+ public boolean isRetriable() {
+ return isRetriable;
+ }
+
+ /**
+ * Whether the cause of the exception should be added to the error message.
+ * @return true, if the cause should be added to the message, false otherwise
+ */
+ public boolean appendCauseMessage() {
+ return appendCauseMessage;
+ }
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatConstants.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.common;
+
+public final class HCatConstants {
+
+ /** The key for the input storage driver class name */
+ public static final String HCAT_ISD_CLASS = "hcat.isd";
+
+ /** The key for the output storage driver class name */
+ public static final String HCAT_OSD_CLASS = "hcat.osd";
+
+ public static final String HIVE_RCFILE_IF_CLASS = "org.apache.hadoop.hive.ql.io.RCFileInputFormat";
+ public static final String HIVE_RCFILE_OF_CLASS = "org.apache.hadoop.hive.ql.io.RCFileOutputFormat";
+ public static final String HCAT_RCFILE_ISD_CLASS = "org.apache.hcatalog.rcfile.RCFileInputDriver";
+ public static final String HCAT_RCFILE_OSD_CLASS = "org.apache.hcatalog.rcfile.RCFileOutputDriver";
+
+ //The keys used to store info into the job Configuration
+ public static final String HCAT_KEY_BASE = "mapreduce.lib.hcat";
+
+ public static final String HCAT_KEY_OUTPUT_SCHEMA = HCAT_KEY_BASE + ".output.schema";
+
+ public static final String HCAT_KEY_JOB_INFO = HCAT_KEY_BASE + ".job.info";
+
+ private HCatConstants() { // restrict instantiation
+ }
+
+ public static final String HCAT_TABLE_SCHEMA = "hcat.table.schema";
+
+ public static final String HCAT_METASTORE_URI = "hcat.metastore.uri";
+
+ public static final String HCAT_PERMS = "hcat.perms";
+
+ public static final String HCAT_GROUP = "hcat.group";
+
+ public static final String HCAT_CREATE_TBL_NAME = "hcat.create.tbl.name";
+
+ public static final String HCAT_CREATE_DB_NAME = "hcat.create.db.name";
+
+ public static final String HCAT_METASTORE_PRINCIPAL = "hcat.metastore.principal";
+
+ // IMPORTANT IMPORTANT IMPORTANT!!!!!
+ //The keys used to store info into the job Configuration.
+ //If any new keys are added, the HowlStorer needs to be updated. The HowlStorer
+ //updates the job configuration in the backend to insert these keys to avoid
+ //having to call setOutput from the backend (which would cause a metastore call
+ //from the map jobs)
+ public static final String HCAT_KEY_OUTPUT_BASE = "mapreduce.lib.hcatoutput";
+ public static final String HCAT_KEY_OUTPUT_INFO = HCAT_KEY_OUTPUT_BASE + ".info";
+ public static final String HCAT_KEY_HIVE_CONF = HCAT_KEY_OUTPUT_BASE + ".hive.conf";
+ public static final String HCAT_KEY_TOKEN_SIGNATURE = HCAT_KEY_OUTPUT_BASE + ".token.sig";
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatException.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatException.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatException.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatException.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.common;
+
+import java.io.IOException;
+
+/**
+ * Class representing exceptions thrown by Howl.
+ */
+public class HCatException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ /** The error type enum for this exception. */
+ private final ErrorType errorType;
+
+ /**
+ * Instantiates a new howl exception.
+ * @param errorType the error type
+ */
+ public HCatException(ErrorType errorType) {
+ this(errorType, null, null);
+ }
+
+
+ /**
+ * Instantiates a new howl exception.
+ * @param errorType the error type
+ * @param cause the cause
+ */
+ public HCatException(ErrorType errorType, Throwable cause) {
+ this(errorType, null, cause);
+ }
+
+ /**
+ * Instantiates a new howl exception.
+ * @param errorType the error type
+ * @param extraMessage extra messages to add to the message string
+ */
+ public HCatException(ErrorType errorType, String extraMessage) {
+ this(errorType, extraMessage, null);
+ }
+
+ /**
+ * Instantiates a new howl exception.
+ * @param errorType the error type
+ * @param extraMessage extra messages to add to the message string
+ * @param cause the cause
+ */
+ public HCatException(ErrorType errorType, String extraMessage, Throwable cause) {
+ super(buildErrorMessage(
+ errorType,
+ extraMessage,
+ cause), cause);
+ this.errorType = errorType;
+ }
+
+
+ //TODO : remove default error type constructors after all exceptions
+ //are changed to use error types
+ /**
+ * Instantiates a new howl exception.
+ * @param message the error message
+ */
+ public HCatException(String message) {
+ this(ErrorType.ERROR_INTERNAL_EXCEPTION, message, null);
+ }
+
+ /**
+ * Instantiates a new howl exception.
+ * @param message the error message
+ * @param cause the cause
+ */
+ public HCatException(String message, Throwable cause) {
+ this(ErrorType.ERROR_INTERNAL_EXCEPTION, message, cause);
+ }
+
+
+ /**
+ * Builds the error message string. The error type message is appended with the extra message. If appendCause
+ * is true for the error type, then the message of the cause also is added to the message.
+ * @param type the error type
+ * @param extraMessage the extra message string
+ * @param cause the cause for the exception
+ * @return the exception message string
+ */
+ public static String buildErrorMessage(ErrorType type, String extraMessage, Throwable cause) {
+
+ //Initial message is just the error type message
+ StringBuffer message = new StringBuffer(HCatException.class.getName());
+ message.append(" : " + type.getErrorCode());
+ message.append(" : " + type.getErrorMessage());
+
+ if( extraMessage != null ) {
+ //Add the extra message value to buffer
+ message.append(" : " + extraMessage);
+ }
+
+ if( type.appendCauseMessage() ) {
+ if( cause != null && cause.getMessage() != null ) {
+ //Add the cause message to buffer
+ message.append(". Cause : " + cause.toString());
+ }
+ }
+
+ return message.toString();
+ }
+
+
+ /**
+ * Is this a retriable error.
+ * @return is it retriable
+ */
+ public boolean isRetriable() {
+ return errorType.isRetriable();
+ }
+
+ /**
+ * Gets the error type.
+ * @return the error type enum
+ */
+ public ErrorType getErrorType() {
+ return errorType;
+ }
+
+ /**
+ * Gets the error code.
+ * @return the error code
+ */
+ public int getErrorCode() {
+ return errorType.getErrorCode();
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Throwable#toString()
+ */
+ @Override
+ public String toString() {
+ return getMessage();
+ }
+
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.common;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.data.schema.HCatSchema;
+import org.apache.hcatalog.data.schema.HCatSchemaUtils;
+
+public class HCatUtil {
+
+ public static boolean checkJobContextIfRunningFromBackend(JobContext j){
+ if (j.getConfiguration().get("mapred.task.id", "").equals("")){
+ return false;
+ }
+ return true;
+ }
+
+ public static String serialize(Serializable obj) throws IOException {
+ if (obj == null) {
+ return "";
+ }
+ try {
+ ByteArrayOutputStream serialObj = new ByteArrayOutputStream();
+ ObjectOutputStream objStream = new ObjectOutputStream(serialObj);
+ objStream.writeObject(obj);
+ objStream.close();
+ return encodeBytes(serialObj.toByteArray());
+ } catch (Exception e) {
+ throw new IOException("Serialization error: " + e.getMessage(), e);
+ }
+ }
+
+ public static Object deserialize(String str) throws IOException {
+ if (str == null || str.length() == 0) {
+ return null;
+ }
+ try {
+ ByteArrayInputStream serialObj = new ByteArrayInputStream(decodeBytes(str));
+ ObjectInputStream objStream = new ObjectInputStream(serialObj);
+ return objStream.readObject();
+ } catch (Exception e) {
+ throw new IOException("Deserialization error: " + e.getMessage(), e);
+ }
+ }
+
+ public static String encodeBytes(byte[] bytes) {
+ StringBuffer strBuf = new StringBuffer();
+
+ for (int i = 0; i < bytes.length; i++) {
+ strBuf.append((char) (((bytes[i] >> 4) & 0xF) + ('a')));
+ strBuf.append((char) (((bytes[i]) & 0xF) + ('a')));
+ }
+
+ return strBuf.toString();
+ }
+
+ public static byte[] decodeBytes(String str) {
+ byte[] bytes = new byte[str.length() / 2];
+ for (int i = 0; i < str.length(); i+=2) {
+ char c = str.charAt(i);
+ bytes[i/2] = (byte) ((c - 'a') << 4);
+ c = str.charAt(i+1);
+ bytes[i/2] += (c - 'a');
+ }
+ return bytes;
+ }
+
+ public static List<HCatFieldSchema> getHCatFieldSchemaList(List<FieldSchema> fields) throws HCatException {
+ if(fields == null) {
+ return null;
+ } else {
+ List<HCatFieldSchema> result = new ArrayList<HCatFieldSchema>();
+ for(FieldSchema f: fields) {
+ result.add(HCatSchemaUtils.getHCatFieldSchema(f));
+ }
+ return result;
+ }
+ }
+
+
+ public static HCatSchema extractSchemaFromStorageDescriptor(StorageDescriptor sd) throws HCatException {
+ if (sd == null){
+ throw new HCatException("Cannot construct partition info from an empty storage descriptor.");
+ }
+ HCatSchema schema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(sd.getCols()));
+ return schema;
+ }
+
+ public static List<FieldSchema> getFieldSchemaList(List<HCatFieldSchema> howlFields) {
+ if(howlFields == null) {
+ return null;
+ } else {
+ List<FieldSchema> result = new ArrayList<FieldSchema>();
+ for(HCatFieldSchema f: howlFields) {
+ result.add(HCatSchemaUtils.getFieldSchema(f));
+ }
+ return result;
+ }
+ }
+
+ public static Table getTable(HiveMetaStoreClient client, String dbName, String tableName) throws Exception{
+ return client.getTable(dbName,tableName);
+ }
+
+ public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException{
+ HCatSchema tableSchema = extractSchemaFromStorageDescriptor(table.getSd());
+
+ if( table.getPartitionKeys().size() != 0 ) {
+
+ // add partition keys to table schema
+ // NOTE : this assumes that we do not ever have ptn keys as columns inside the table schema as well!
+ for (FieldSchema fs : table.getPartitionKeys()){
+ tableSchema.append(HCatSchemaUtils.getHCatFieldSchema(fs));
+ }
+ }
+ return tableSchema;
+ }
+
+ /**
+ * Validate partition schema, checks if the column types match between the partition
+ * and the existing table schema. Returns the list of columns present in the partition
+ * but not in the table.
+ * @param table the table
+ * @param partitionSchema the partition schema
+ * @return the list of newly added fields
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ public static List<FieldSchema> validatePartitionSchema(Table table, HCatSchema partitionSchema) throws IOException {
+ Map<String, FieldSchema> partitionKeyMap = new HashMap<String, FieldSchema>();
+
+ for(FieldSchema field : table.getPartitionKeys()) {
+ partitionKeyMap.put(field.getName().toLowerCase(), field);
+ }
+
+ List<FieldSchema> tableCols = table.getSd().getCols();
+ List<FieldSchema> newFields = new ArrayList<FieldSchema>();
+
+ for(int i = 0;i < partitionSchema.getFields().size();i++) {
+
+ FieldSchema field = HCatSchemaUtils.getFieldSchema(partitionSchema.getFields().get(i));
+
+ FieldSchema tableField;
+ if( i < tableCols.size() ) {
+ tableField = tableCols.get(i);
+
+ if( ! tableField.getName().equalsIgnoreCase(field.getName())) {
+ throw new HCatException(ErrorType.ERROR_SCHEMA_COLUMN_MISMATCH, "Expected column <" + tableField.getName() +
+ "> at position " + (i + 1) + ", found column <" + field.getName() + ">");
+ }
+ } else {
+ tableField = partitionKeyMap.get(field.getName().toLowerCase());
+
+ if( tableField != null ) {
+ throw new HCatException(ErrorType.ERROR_SCHEMA_PARTITION_KEY, "Key <" + field.getName() + ">");
+ }
+ }
+
+ if( tableField == null ) {
+ //field present in partition but not in table
+ newFields.add(field);
+ } else {
+ //field present in both. validate type has not changed
+ TypeInfo partitionType = TypeInfoUtils.getTypeInfoFromTypeString(field.getType());
+ TypeInfo tableType = TypeInfoUtils.getTypeInfoFromTypeString(tableField.getType());
+
+ if( ! partitionType.equals(tableType) ) {
+ throw new HCatException(ErrorType.ERROR_SCHEMA_TYPE_MISMATCH, "Column <" + field.getName() + ">, expected <" +
+ tableType.getTypeName() + ">, got <" + partitionType.getTypeName() + ">");
+ }
+ }
+ }
+
+ return newFields;
+ }
+
+ /**
+ * Test if the first FsAction is more permissive than the second. This is useful in cases where
+ * we want to ensure that a file owner has more permissions than the group they belong to, for eg.
+ * More completely(but potentially more cryptically)
+ * owner-r >= group-r >= world-r : bitwise and-masked with 0444 => 444 >= 440 >= 400 >= 000
+ * owner-w >= group-w >= world-w : bitwise and-masked with &0222 => 222 >= 220 >= 200 >= 000
+ * owner-x >= group-x >= world-x : bitwise and-masked with &0111 => 111 >= 110 >= 100 >= 000
+ * @return true if first FsAction is more permissive than the second, false if not.
+ */
+ public static boolean validateMorePermissive(FsAction first, FsAction second) {
+ if ((first == FsAction.ALL) ||
+ (second == FsAction.NONE) ||
+ (first == second)) {
+ return true;
+ }
+ switch (first){
+ case READ_EXECUTE : return ((second == FsAction.READ) || (second == FsAction.EXECUTE));
+ case READ_WRITE : return ((second == FsAction.READ) || (second == FsAction.WRITE));
+ case WRITE_EXECUTE : return ((second == FsAction.WRITE) || (second == FsAction.EXECUTE));
+ }
+ return false;
+ }
+
+ /**
+ * Ensure that read or write permissions are not granted without also granting execute permissions.
+ * Essentially, r-- , rw- and -w- are invalid,
+ * r-x, -wx, rwx, ---, --x are valid
+ *
+ * @param perms The FsAction to verify
+ * @return true if the presence of read or write permission is accompanied by execute permissions
+ */
+ public static boolean validateExecuteBitPresentIfReadOrWrite(FsAction perms){
+ if ((perms == FsAction.READ) || (perms == FsAction.WRITE) || (perms == FsAction.READ_WRITE)){
+ return false;
+ }
+ return true;
+ }
+
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DataType.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DataType.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DataType.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DataType.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
+
+public abstract class DataType {
+
+ public static final byte NULL = 1;
+ public static final byte BOOLEAN = 5;
+ public static final byte BYTE = 6;
+ public static final byte INTEGER = 10;
+ public static final byte SHORT = 11;
+ public static final byte LONG = 15;
+ public static final byte FLOAT = 20;
+ public static final byte DOUBLE = 25;
+ public static final byte STRING = 55;
+
+ public static final byte MAP = 100;
+ public static final byte STRUCT = 110;
+ public static final byte LIST = 120;
+ public static final byte ERROR = -1;
+
+ /**
+ * Determine the datatype of an object.
+ * @param o Object to test.
+ * @return byte code of the type, or ERROR if we don't know.
+ */
+ public static byte findType(Object o) {
+ if (o == null) {
+ return NULL;
+ }
+
+ Class<?> clazz = o.getClass();
+
+ // Try to put the most common first
+ if (clazz == String.class) {
+ return STRING;
+ } else if (clazz == Integer.class) {
+ return INTEGER;
+ } else if (clazz == Long.class) {
+ return LONG;
+ } else if (clazz == Float.class) {
+ return FLOAT;
+ } else if (clazz == Double.class) {
+ return DOUBLE;
+ } else if (clazz == Boolean.class) {
+ return BOOLEAN;
+ } else if (clazz == Byte.class) {
+ return BYTE;
+ } else if (clazz == Short.class) {
+ return SHORT;
+ } else if (o instanceof List<?>) {
+ return LIST;
+ } else if (o instanceof Map<?,?>) {
+ return MAP;
+ } else {return ERROR;}
+ }
+
+ public static int compare(Object o1, Object o2) {
+
+ return compare(o1, o2, findType(o1),findType(o2));
+ }
+
+ public static int compare(Object o1, Object o2, byte dt1, byte dt2) {
+ if (dt1 == dt2) {
+ switch (dt1) {
+ case NULL:
+ return 0;
+
+ case BOOLEAN:
+ return ((Boolean)o1).compareTo((Boolean)o2);
+
+ case BYTE:
+ return ((Byte)o1).compareTo((Byte)o2);
+
+ case INTEGER:
+ return ((Integer)o1).compareTo((Integer)o2);
+
+ case LONG:
+ return ((Long)o1).compareTo((Long)o2);
+
+ case FLOAT:
+ return ((Float)o1).compareTo((Float)o2);
+
+ case DOUBLE:
+ return ((Double)o1).compareTo((Double)o2);
+
+ case STRING:
+ return ((String)o1).compareTo((String)o2);
+
+ case SHORT:
+ return ((Short)o1).compareTo((Short)o2);
+
+ case LIST:
+ List<?> l1 = (List<?>)o1;
+ List<?> l2 = (List<?>)o2;
+ int len = l1.size();
+ if(len != l2.size()) {
+ return len - l2.size();
+ } else{
+ for(int i =0; i < len; i++){
+ int cmpVal = compare(l1.get(i), l2.get(i));
+ if(cmpVal != 0) {
+ return cmpVal;
+ }
+ }
+ return 0;
+ }
+
+ case MAP: {
+ Map<?,?> m1 = (Map<?,?>)o1;
+ Map<?,?> m2 = (Map<?,?>)o2;
+ int sz1 = m1.size();
+ int sz2 = m2.size();
+ if (sz1 < sz2) {
+ return -1;
+ } else if (sz1 > sz2) {
+ return 1;
+ } else {
+ // This is bad, but we have to sort the keys of the maps in order
+ // to be commutative.
+ TreeMap<Object,Object> tm1 = new TreeMap<Object,Object>(m1);
+ TreeMap<Object, Object> tm2 = new TreeMap<Object,Object>(m2);
+ Iterator<Entry<Object, Object>> i1 = tm1.entrySet().iterator();
+ Iterator<Entry<Object, Object> > i2 = tm2.entrySet().iterator();
+ while (i1.hasNext()) {
+ Map.Entry<Object, Object> entry1 = i1.next();
+ Map.Entry<Object, Object> entry2 = i2.next();
+ int c = compare(entry1.getValue(), entry2.getValue());
+ if (c != 0) {
+ return c;
+ } else {
+ c = compare(entry1.getValue(), entry2.getValue());
+ if (c != 0) {
+ return c;
+ }
+ }
+ }
+ return 0;
+ }
+ }
+
+ default:
+ throw new RuntimeException("Unkown type " + dt1 +
+ " in compare");
+ }
+ } else {
+ return dt1 < dt2 ? -1 : 1;
+ }
+ }
+}
\ No newline at end of file
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatSchema;
+
+public class DefaultHCatRecord extends HCatRecord {
+
+ private final List<Object> contents;
+
+ public DefaultHCatRecord(){
+ contents = new ArrayList<Object>();
+ }
+
+ public DefaultHCatRecord(int size){
+ contents = new ArrayList<Object>(size);
+ for(int i=0; i < size; i++){
+ contents.add(null);
+ }
+ }
+
+ @Override
+ public void remove(int idx) throws HCatException {
+ contents.remove(idx);
+ }
+
+ public DefaultHCatRecord(List<Object> list) {
+ contents = list;
+ }
+
+ @Override
+ public Object get(int fieldNum) {
+ return contents.get(fieldNum);
+ }
+
+ @Override
+ public List<Object> getAll() {
+ return contents;
+ }
+
+ @Override
+ public void set(int fieldNum, Object val) {
+ contents.set(fieldNum, val);
+ }
+
+ @Override
+ public int size() {
+ return contents.size();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+
+ contents.clear();
+ int len = in.readInt();
+ for(int i =0; i < len; i++){
+ contents.add(ReaderWriter.readDatum(in));
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ int sz = size();
+ out.writeInt(sz);
+ for (int i = 0; i < sz; i++) {
+ ReaderWriter.writeDatum(out, contents.get(i));
+ }
+
+ }
+
+ @Override
+ public int compareTo(Object that) {
+
+ if(that instanceof HCatRecord) {
+ HCatRecord other = (HCatRecord)that;
+ int mySz = this.size();
+ int urSz = other.size();
+ if(mySz != urSz) {
+ return mySz - urSz;
+ } else{
+ for (int i = 0; i < mySz;i++) {
+ int c = DataType.compare(get(i), other.get(i));
+ if (c != 0) {
+ return c;
+ }
+ }
+ }
+ return 0;
+ } else {
+ return DataType.compare(this, that);
+ }
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return (compareTo(other) == 0);
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 1;
+ for (Object o : contents) {
+ if (o != null) {
+ hash = 31 * hash + o.hashCode();
+ }
+ }
+ return hash;
+ }
+
+ @Override
+ public String toString() {
+
+ StringBuilder sb = new StringBuilder();
+ for(Object o : contents) {
+ sb.append(o+"\t");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public Object get(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return get(recordSchema.getPosition(fieldName));
+ }
+
+ @Override
+ public void set(String fieldName, HCatSchema recordSchema, Object value) throws HCatException {
+ set(recordSchema.getPosition(fieldName),value);
+ }
+
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatArrayBag.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatArrayBag.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatArrayBag.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatArrayBag.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DefaultBagFactory;
+import org.apache.pig.data.DefaultDataBag;
+import org.apache.pig.data.DefaultTuple;
+import org.apache.pig.data.Tuple;
+
+public class HCatArrayBag<T> implements DataBag {
+
+ private static final long DUMMY_SIZE = 40;
+ List<T> rawItemList = null;
+ DataBag convertedBag = null;
+// List<Tuple> tupleList = null;
+
+ public class HowlArrayBagIterator implements Iterator<Tuple> {
+
+ Iterator<T> iter = null;
+
+ public HowlArrayBagIterator(List<T> rawItemList) {
+ iter = rawItemList.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public Tuple next() {
+ Tuple t = new DefaultTuple();
+ t.append(iter.next());
+ return t;
+ }
+
+ @Override
+ public void remove() {
+ iter.remove();
+ }
+
+ }
+
+ public HCatArrayBag(List<T> list) {
+ rawItemList = list;
+ }
+
+ private void convertFromRawToTupleForm(){
+ if (convertedBag == null){
+ List<Tuple> ltuples = new ArrayList<Tuple>();
+ for (T item : rawItemList){
+ Tuple t = new DefaultTuple();
+ t.append(item);
+ ltuples.add(t);
+ }
+ convertedBag = DefaultBagFactory.getInstance().newDefaultBag(ltuples);
+ }else{
+ // TODO : throw exception or be silent? Currently going with silence, but needs revisiting.
+ }
+ }
+
+ @Override
+ public void add(Tuple t) {
+ if (convertedBag == null){
+ convertFromRawToTupleForm();
+ }
+ convertedBag.add(t);
+ }
+
+ @Override
+ public void addAll(DataBag db) {
+ Tuple t;
+ for (Iterator<Tuple> dbi = db.iterator() ; dbi.hasNext();){
+ this.add(dbi.next());
+ }
+ }
+
+ @Override
+ public void clear() {
+ rawItemList = null;
+ if (convertedBag != null){
+ convertedBag.clear();
+ convertedBag = null;
+ }
+ }
+
+ @Override
+ public boolean isDistinct() {
+ return false;
+ }
+
+ @Override
+ public boolean isSorted() {
+ return false;
+ }
+
+ @Override
+ public Iterator<Tuple> iterator() {
+ if (convertedBag != null){
+ return convertedBag.iterator();
+ }else{
+ return new HowlArrayBagIterator(rawItemList);
+ }
+ }
+
+ @Override
+ public void markStale(boolean arg0) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public long size() {
+ return (convertedBag == null ? (rawItemList == null ? 0 : rawItemList.size()) : convertedBag.size() );
+ }
+
+ @Override
+ public long getMemorySize() {
+ // FIXME: put in actual impl
+ if (convertedBag != null){
+ return convertedBag.getMemorySize() + DUMMY_SIZE;
+ }else {
+ return DUMMY_SIZE;
+ }
+ }
+
+ @Override
+ public long spill() {
+ // FIXME: put in actual spill impl even for the list case
+ if (convertedBag != null){
+ return convertedBag.spill();
+ }
+ return 0;
+ }
+
+ @Override
+ public void readFields(DataInput arg0) throws IOException {
+ convertedBag = new DefaultDataBag();
+ convertedBag.readFields(arg0);
+ }
+
+ @Override
+ public void write(DataOutput arg0) throws IOException {
+ convertFromRawToTupleForm();
+ convertedBag.write(arg0);
+ }
+
+ @Override
+ public int compareTo(Object arg0) {
+ // TODO Auto-generated method stub - really need to put in a better implementation here, also, equality case not considered yet
+ return arg0.hashCode() < this.hashCode() ? -1 : 1;
+ }
+
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatSchema;
+
+/**
+ * Abstract class exposing get and set semantics for basic record usage.
+ * Note :
+ * HowlRecord is designed only to be used as in-memory representation only.
+ * Don't use it to store data on the physical device.
+ */
+public abstract class HCatRecord implements HCatRecordable {
+
+ public abstract Object get(String fieldName, HCatSchema recordSchema) throws HCatException;
+ public abstract void set(String fieldName, HCatSchema recordSchema, Object value ) throws HCatException;
+ public abstract void remove(int idx) throws HCatException;
+
+ protected Object get(String fieldName, HCatSchema recordSchema, Class clazz) throws HCatException{
+ // TODO : if needed, verify that recordschema entry for fieldname matches appropriate type.
+ return get(fieldName,recordSchema);
+ }
+
+ public Boolean getBoolean(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (Boolean) get(fieldName, recordSchema, Boolean.class);
+ }
+
+ public void setBoolean(String fieldName, HCatSchema recordSchema, Boolean value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public Byte getByte(String fieldName, HCatSchema recordSchema) throws HCatException {
+ //TINYINT
+ return (Byte) get(fieldName, recordSchema, Byte.class);
+ }
+
+ public void setByte(String fieldName, HCatSchema recordSchema, Byte value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public Short getShort(String fieldName, HCatSchema recordSchema) throws HCatException {
+ // SMALLINT
+ return (Short) get(fieldName, recordSchema, Short.class);
+ }
+
+ public void setShort(String fieldName, HCatSchema recordSchema, Short value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public Integer getInteger(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (Integer) get(fieldName,recordSchema, Integer.class);
+ }
+
+ public void setInteger(String fieldName, HCatSchema recordSchema, Integer value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public Long getLong(String fieldName, HCatSchema recordSchema) throws HCatException {
+ // BIGINT
+ return (Long) get(fieldName,recordSchema,Long.class);
+ }
+
+ public void setLong(String fieldName, HCatSchema recordSchema, Long value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public Float getFloat(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (Float) get(fieldName,recordSchema,Float.class);
+ }
+
+ public void setFloat(String fieldName, HCatSchema recordSchema, Float value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public Double getDouble(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (Double) get(fieldName,recordSchema,Double.class);
+ }
+
+ public void setDouble(String fieldName, HCatSchema recordSchema, Double value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public String getString(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (String) get(fieldName,recordSchema,String.class);
+ }
+
+ public void setString(String fieldName, HCatSchema recordSchema, String value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<? extends Object> getStruct(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (List<? extends Object>) get(fieldName,recordSchema,List.class);
+ }
+
+ public void setStruct(String fieldName, HCatSchema recordSchema, List<? extends Object> value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public List<?> getList(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (List<?>) get(fieldName,recordSchema,List.class);
+ }
+
+ public void setList(String fieldName, HCatSchema recordSchema, List<?> value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+ public Map<?,?> getMap(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (Map<?,?>) get(fieldName,recordSchema,Map.class);
+ }
+
+ public void setMap(String fieldName, HCatSchema recordSchema, Map<?,?> value) throws HCatException {
+ set(fieldName,recordSchema,value);
+ }
+
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data;
+
+import java.util.List;
+
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * Interface that determines whether we can implement a HowlRecord on top of it
+ */
+public interface HCatRecordable extends WritableComparable<Object> {
+
+ /**
+ * Gets the field at the specified index.
+ * @param fieldNum the field number
+ * @return the object at the specified index
+ */
+ Object get(int fieldNum);
+
+ /**
+ * Gets all the fields of the howl record.
+ * @return the list of fields
+ */
+ List<Object> getAll();
+
+ /**
+ * Sets the field at the specified index.
+ * @param fieldNum the field number
+ * @param value the value to set
+ */
+ void set(int fieldNum, Object value);
+
+ /**
+ * Gets the size of the howl record.
+ * @return the size
+ */
+ int size();
+
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/Pair.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/Pair.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/Pair.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/Pair.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data;
+
+import java.io.Serializable;
+
+/**
+ * Copy of C++ STL pair container.
+ */
+public class Pair<T, U> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ public T first;
+ public U second;
+
+ /**
+ * @param f First element in pair.
+ * @param s Second element in pair.
+ */
+ public Pair(T f, U s) {
+ first = f;
+ second = s;
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ return "[" + first.toString() +"," + second.toString() + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ return (((this.first == null ? 1 : this.first.hashCode()) * 17)
+ + (this.second == null ? 1 : this.second.hashCode()) * 19);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if(other == null) {
+ return false;
+ }
+
+ if(! (other instanceof Pair)) {
+ return false;
+ }
+
+ Pair otherPair = (Pair) other;
+
+ if(this.first == null) {
+ if(otherPair.first != null) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ if(this.second == null) {
+ if(otherPair.second != null) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ if(this.first.equals(otherPair.first) && this.second.equals(otherPair.second)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/ReaderWriter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/ReaderWriter.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/ReaderWriter.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/ReaderWriter.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.io.VIntWritable;
+import org.apache.hadoop.io.VLongWritable;
+
+
+public abstract class ReaderWriter {
+
+ private static final String UTF8 = "UTF-8";
+
+ public static Object readDatum(DataInput in) throws IOException {
+
+ byte type = in.readByte();
+ switch (type) {
+
+ case DataType.STRING:
+ byte[] buffer = new byte[in.readInt()];
+ in.readFully(buffer);
+ return new String(buffer,UTF8);
+
+ case DataType.INTEGER:
+ VIntWritable vint = new VIntWritable();
+ vint.readFields(in);
+ return vint.get();
+
+ case DataType.LONG:
+ VLongWritable vlong = new VLongWritable();
+ vlong.readFields(in);
+ return vlong.get();
+
+ case DataType.FLOAT:
+ return in.readFloat();
+
+ case DataType.DOUBLE:
+ return in.readDouble();
+
+ case DataType.BOOLEAN:
+ return in.readBoolean();
+
+ case DataType.BYTE:
+ return in.readByte();
+
+ case DataType.SHORT:
+ return in.readShort();
+
+ case DataType.NULL:
+ return null;
+
+ case DataType.MAP:
+ int size = in.readInt();
+ Map<Object,Object> m = new HashMap<Object, Object>(size);
+ for (int i = 0; i < size; i++) {
+ m.put(readDatum(in), readDatum(in));
+ }
+ return m;
+
+ case DataType.LIST:
+ int sz = in.readInt();
+ List<Object> list = new ArrayList<Object>(sz);
+ for(int i=0; i < sz; i++) {
+ list.add(readDatum(in));
+ }
+ return list;
+
+ default:
+ throw new IOException("Unexpected data type " + type +
+ " found in stream.");
+ }
+ }
+
+ public static void writeDatum(DataOutput out, Object val) throws IOException {
+ // write the data type
+ byte type = DataType.findType(val);
+ switch (type) {
+ case DataType.LIST:
+ out.writeByte(DataType.LIST);
+ List<?> list = (List<?>)val;
+ int sz = list.size();
+ out.writeInt(sz);
+ for (int i = 0; i < sz; i++) {
+ writeDatum(out, list.get(i));
+ }
+ return;
+
+ case DataType.MAP:
+ out.writeByte(DataType.MAP);
+ Map<?,?> m = (Map<?, ?>)val;
+ out.writeInt(m.size());
+ Iterator<?> i =
+ m.entrySet().iterator();
+ while (i.hasNext()) {
+ Entry<?,?> entry = (Entry<?, ?>) i.next();
+ writeDatum(out, entry.getKey());
+ writeDatum(out, entry.getValue());
+ }
+ return;
+
+ case DataType.INTEGER:
+ out.writeByte(DataType.INTEGER);
+ new VIntWritable((Integer)val).write(out);
+ return;
+
+ case DataType.LONG:
+ out.writeByte(DataType.LONG);
+ new VLongWritable((Long)val).write(out);
+ return;
+
+ case DataType.FLOAT:
+ out.writeByte(DataType.FLOAT);
+ out.writeFloat((Float)val);
+ return;
+
+ case DataType.DOUBLE:
+ out.writeByte(DataType.DOUBLE);
+ out.writeDouble((Double)val);
+ return;
+
+ case DataType.BOOLEAN:
+ out.writeByte(DataType.BOOLEAN);
+ out.writeBoolean((Boolean)val);
+ return;
+
+ case DataType.BYTE:
+ out.writeByte(DataType.BYTE);
+ out.writeByte((Byte)val);
+ return;
+
+ case DataType.SHORT:
+ out.writeByte(DataType.SHORT);
+ out.writeShort((Short)val);
+ return;
+
+ case DataType.STRING:
+ String s = (String)val;
+ byte[] utfBytes = s.getBytes(ReaderWriter.UTF8);
+ out.writeByte(DataType.STRING);
+ out.writeInt(utfBytes.length);
+ out.write(utfBytes);
+ return;
+
+
+ case DataType.NULL:
+ out.writeByte(DataType.NULL);
+ return;
+
+ default:
+ throw new IOException("Unexpected data type " + type +
+ " found in stream.");
+ }
+ }
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data.schema;
+
+import java.io.Serializable;
+
+import org.apache.hcatalog.common.HCatException;
+
+public class HCatFieldSchema implements Serializable {
+
+ public enum Type {
+ INT,
+ TINYINT,
+ SMALLINT,
+ BIGINT,
+ BOOLEAN,
+ FLOAT,
+ DOUBLE,
+ STRING,
+ ARRAY,
+ MAP,
+ STRUCT,
+ }
+
+ public enum Category {
+ PRIMITIVE,
+ ARRAY,
+ MAP,
+ STRUCT;
+
+ public static Category fromType(Type type) {
+ if (Type.ARRAY == type){
+ return ARRAY;
+ }else if(Type.STRUCT == type){
+ return STRUCT;
+ }else if (Type.MAP == type){
+ return MAP;
+ }else{
+ return PRIMITIVE;
+ }
+ }
+ };
+
+ public boolean isComplex(){
+ return (category == Category.PRIMITIVE) ? false : true;
+ }
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ String fieldName = null;
+ String comment = null;
+ Type type = null;
+ Category category = null;
+
+ // Populated if column is struct, array or map types.
+ // If struct type, contains schema of the struct.
+ // If array type, contains schema of one of the elements.
+ // If map type, contains schema of the value element.
+ HCatSchema subSchema = null;
+
+ // populated if column is Map type
+ Type mapKeyType = null;
+
+ private String typeString = null;
+
+ @SuppressWarnings("unused")
+ private HCatFieldSchema(){
+ // preventing empty ctor from being callable
+ }
+
+ /**
+ * Returns type of the field
+ * @return type of the field
+ */
+ public Type getType(){
+ return type;
+ }
+
+ /**
+ * Returns category of the field
+ * @return category of the field
+ */
+ public Category getCategory(){
+ return category;
+ }
+
+ /**
+ * Returns name of the field
+ * @return name of the field
+ */
+ public String getName(){
+ return fieldName;
+ }
+
+ public String getComment(){
+ return comment;
+ }
+
+ /**
+ * Constructor constructing a primitive datatype HCatFieldSchema
+ * @param fieldName Name of the primitive field
+ * @param type Type of the primitive field
+ * @throws HCatException if call made on non-primitive types
+ */
+ public HCatFieldSchema(String fieldName, Type type, String comment) throws HCatException {
+ assertTypeInCategory(type,Category.PRIMITIVE);
+ this.fieldName = fieldName;
+ this.type = type;
+ this.category = Category.PRIMITIVE;
+ this.comment = comment;
+ }
+
+ /**
+ * Constructor for constructing a ARRAY type or STRUCT type HCatFieldSchema, passing type and subschema
+ * @param fieldName Name of the array or struct field
+ * @param type Type of the field - either Type.ARRAY or Type.STRUCT
+ * @param subSchema - subschema of the struct, or element schema of the elements in the array
+ * @throws HCatException if call made on Primitive or Map types
+ */
+ public HCatFieldSchema(String fieldName, Type type, HCatSchema subSchema,String comment) throws HCatException{
+ assertTypeNotInCategory(type,Category.PRIMITIVE);
+ assertTypeNotInCategory(type,Category.MAP);
+ this.fieldName = fieldName;
+ this.type = type;
+ this.category = Category.fromType(type);
+ this.subSchema = subSchema;
+ if(type == Type.ARRAY){
+ this.subSchema.get(0).setName(null);
+ }
+ this.comment = comment;
+ }
+
+ private void setName(String name) {
+ this.fieldName = name;
+ }
+
+ /**
+ * Constructor for constructing a MAP type HCatFieldSchema, passing type of key and value
+ * @param fieldName Name of the array or struct field
+ * @param type Type of the field - must be Type.MAP
+ * @param mapKeyType - key type of the Map
+ * @param mapValueSchema - subschema of the value of the Map
+ * @throws HCatException if call made on non-Map types
+ */
+ public HCatFieldSchema(String fieldName, Type type, Type mapKeyType, HCatSchema mapValueSchema, String comment) throws HCatException{
+ assertTypeInCategory(type,Category.MAP);
+ assertTypeInCategory(mapKeyType,Category.PRIMITIVE);
+ this.fieldName = fieldName;
+ this.type = Type.MAP;
+ this.category = Category.MAP;
+ this.mapKeyType = mapKeyType;
+ this.subSchema = mapValueSchema;
+ this.subSchema.get(0).setName(null);
+ this.comment = comment;
+ }
+
+ public HCatSchema getStructSubSchema() throws HCatException {
+ assertTypeInCategory(this.type,Category.STRUCT);
+ return subSchema;
+ }
+
+ public HCatSchema getArrayElementSchema() throws HCatException {
+ assertTypeInCategory(this.type,Category.ARRAY);
+ return subSchema;
+ }
+
+ public Type getMapKeyType() throws HCatException {
+ assertTypeInCategory(this.type,Category.MAP);
+ return mapKeyType;
+ }
+
+ public HCatSchema getMapValueSchema() throws HCatException {
+ assertTypeInCategory(this.type,Category.MAP);
+ return subSchema;
+ }
+
+ private static void assertTypeInCategory(Type type, Category category) throws HCatException {
+ Category typeCategory = Category.fromType(type);
+ if (typeCategory != category){
+ throw new HCatException("Type category mismatch. Expected "+category+" but type "+type+" in category "+typeCategory);
+ }
+ }
+
+ private static void assertTypeNotInCategory(Type type, Category category) throws HCatException {
+ Category typeCategory = Category.fromType(type);
+ if (typeCategory == category){
+ throw new HCatException("Type category mismatch. Expected type "+type+" not in category "+category+" but was so.");
+ }
+ }
+
+ @Override
+ public String toString(){
+ return getTypeString();
+ }
+
+ public String getTypeString(){
+ if (typeString != null){
+ return typeString;
+ }
+
+ StringBuilder sb = new StringBuilder();
+ if (Category.PRIMITIVE == category){
+ sb.append(type);
+ }else if (Category.STRUCT == category){
+ sb.append("struct<");
+ sb.append(subSchema.toString());
+ sb.append(">");
+ }else if (Category.ARRAY == category){
+ sb.append("array<");
+ sb.append(subSchema.toString());
+ sb.append(">");
+ }else if (Category.MAP == category){
+ sb.append("map<");
+ sb.append(mapKeyType);
+ sb.append(",");
+ sb.append(subSchema.toString());
+ sb.append(">");
+ }
+ return (typeString = sb.toString().toLowerCase());
+ }
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchema.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data.schema;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hcatalog.common.HCatException;
+
+/**
+ * HCatSchema. This class is NOT thread-safe.
+ */
+
+public class HCatSchema implements Serializable{
+
+ private static final long serialVersionUID = 1L;
+
+ private final List<HCatFieldSchema> fieldSchemas;
+ private final Map<String,Integer> fieldPositionMap;
+ private final List<String> fieldNames;
+
+ /**
+ *
+ * @param fieldSchemas is now owned by HCatSchema. Any subsequent modifications
+ * on fieldSchemas won't get reflected in HCatSchema.
+ */
+ public HCatSchema(final List<HCatFieldSchema> fieldSchemas){
+ this.fieldSchemas = new ArrayList<HCatFieldSchema>(fieldSchemas);
+ int idx = 0;
+ fieldPositionMap = new HashMap<String,Integer>();
+ fieldNames = new ArrayList<String>();
+ for (HCatFieldSchema field : fieldSchemas){
+ fieldPositionMap.put(field.getName(), idx);
+ fieldNames.add(field.getName());
+ idx++;
+ }
+ }
+
+ public void append(final HCatFieldSchema hfs) throws HCatException{
+
+ if(hfs == null || fieldSchemas == null){
+ throw new HCatException("Attempt to append null HCatFieldSchema in HCatSchema.");
+ }
+ //TODO Addition of existing field should not be allowed in Schema.
+ //Need to enforce that. For that to happen, field schema needs to implement Comparable.
+ // Also, HCatSchema needs to implement Comparable.
+
+ this.fieldSchemas.add(hfs);
+ String fieldName = hfs.getName();
+ this.fieldNames.add(fieldName);
+ this.fieldPositionMap.put(fieldName, this.size()-1);
+ }
+
+ /**
+ * Users are not allowed to modify the list directly, since HCatSchema
+ * maintains internal state. Use append/remove to modify the schema.
+ */
+ public List<HCatFieldSchema> getFields(){
+ return Collections.unmodifiableList(this.fieldSchemas);
+ }
+
+ /**
+ * @param fieldName
+ * @return the index of field named fieldName in Schema. If field is not
+ * present, returns null.
+ */
+ public Integer getPosition(String fieldName) {
+ return fieldPositionMap.get(fieldName);
+ }
+
+ public HCatFieldSchema get(String fieldName) throws HCatException {
+ return get(getPosition(fieldName));
+ }
+
+ public List<String> getFieldNames(){
+ return this.fieldNames;
+ }
+
+ public HCatFieldSchema get(int position) {
+ return fieldSchemas.get(position);
+ }
+
+ public int size(){
+ return fieldSchemas.size();
+ }
+
+ public void remove(final HCatFieldSchema howlFieldSchema) throws HCatException {
+
+ if(!fieldSchemas.contains(howlFieldSchema)){
+ throw new HCatException("Attempt to delete a non-existent column from Howl Schema: "+ howlFieldSchema);
+ }
+
+ fieldSchemas.remove(howlFieldSchema);
+ fieldPositionMap.remove(howlFieldSchema);
+ fieldNames.remove(howlFieldSchema.getName());
+ }
+
+ @Override
+ public String toString() {
+ boolean first = true;
+ StringBuilder sb = new StringBuilder();
+ for (HCatFieldSchema hfs : fieldSchemas){
+ if (!first){
+ sb.append(",");
+ }else{
+ first = false;
+ }
+ if (hfs.getName() != null){
+ sb.append(hfs.getName());
+ sb.append(":");
+ }
+ sb.append(hfs.toString());
+ }
+ return sb.toString();
+ }
+}
Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java?rev=1091509&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java Tue Apr 12 17:30:08 2011
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.data.schema;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Schema;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.data.schema.HCatFieldSchema.Type;
+
+
+public class HCatSchemaUtils {
+
+ private static HCatSchemaUtils ref = new HCatSchemaUtils();
+
+ public static CollectionBuilder getStructSchemaBuilder(){
+ return ref.new CollectionBuilder();
+ }
+
+ public static CollectionBuilder getListSchemaBuilder(){
+ return ref.new CollectionBuilder();
+ }
+
+ public static MapBuilder getMapSchemaBuilder(){
+ return ref.new MapBuilder();
+ }
+
+
+ public abstract class HCatSchemaBuilder {
+ public abstract HCatSchema build() throws HCatException;
+ }
+
+ public class CollectionBuilder extends HCatSchemaBuilder { // for STRUCTS(multiple-add-calls) and LISTS(single-add-call)
+ List<HCatFieldSchema> fieldSchemas = null;
+
+ CollectionBuilder(){
+ fieldSchemas = new ArrayList<HCatFieldSchema>();
+ }
+
+ public CollectionBuilder addField(FieldSchema fieldSchema) throws HCatException{
+ return this.addField(getHCatFieldSchema(fieldSchema));
+ }
+
+ public CollectionBuilder addField(HCatFieldSchema fieldColumnSchema){
+ fieldSchemas.add(fieldColumnSchema);
+ return this;
+ }
+
+ @Override
+ public HCatSchema build() throws HCatException{
+ return new HCatSchema(fieldSchemas);
+ }
+
+ }
+
+ public class MapBuilder extends HCatSchemaBuilder {
+
+ Type keyType = null;
+ HCatSchema valueSchema = null;
+
+ @Override
+ public HCatSchema build() throws HCatException {
+ List<HCatFieldSchema> fslist = new ArrayList<HCatFieldSchema>();
+ fslist.add(new HCatFieldSchema(null,Type.MAP,keyType,valueSchema,null));
+ return new HCatSchema(fslist);
+ }
+
+ public MapBuilder withValueSchema(HCatSchema valueSchema) {
+ this.valueSchema = valueSchema;
+ return this;
+ }
+
+ public MapBuilder withKeyType(Type keyType) {
+ this.keyType = keyType;
+ return this;
+ }
+
+ }
+
+
+ /**
+ * Convert a HCatFieldSchema to a FieldSchema
+ * @param fs FieldSchema to convert
+ * @return HCatFieldSchema representation of FieldSchema
+ * @throws HCatException
+ */
+ public static HCatFieldSchema getHCatFieldSchema(FieldSchema fs) throws HCatException {
+ String fieldName = fs.getName();
+ TypeInfo baseTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fs.getType());
+ return getHCatFieldSchema(fieldName, baseTypeInfo);
+ }
+
+ private static HCatFieldSchema getHCatFieldSchema(String fieldName, TypeInfo fieldTypeInfo) throws HCatException {
+ Category typeCategory = fieldTypeInfo.getCategory();
+ if (Category.PRIMITIVE == typeCategory){
+ return new HCatFieldSchema(fieldName,getPrimitiveHType(fieldTypeInfo),null);
+ } else if (Category.STRUCT == typeCategory) {
+ HCatSchema subSchema = constructHCatSchema((StructTypeInfo)fieldTypeInfo);
+ return new HCatFieldSchema(fieldName,HCatFieldSchema.Type.STRUCT,subSchema,null);
+ } else if (Category.LIST == typeCategory) {
+ HCatSchema subSchema = getHCatSchema(((ListTypeInfo)fieldTypeInfo).getListElementTypeInfo());
+ return new HCatFieldSchema(fieldName,HCatFieldSchema.Type.ARRAY,subSchema,null);
+ } else if (Category.MAP == typeCategory) {
+ HCatFieldSchema.Type mapKeyType = getPrimitiveHType(((MapTypeInfo)fieldTypeInfo).getMapKeyTypeInfo());
+ HCatSchema subSchema = getHCatSchema(((MapTypeInfo)fieldTypeInfo).getMapValueTypeInfo());
+ return new HCatFieldSchema(fieldName,HCatFieldSchema.Type.MAP,mapKeyType,subSchema,null);
+ } else{
+ throw new TypeNotPresentException(fieldTypeInfo.getTypeName(),null);
+ }
+ }
+
+ private static Type getPrimitiveHType(TypeInfo basePrimitiveTypeInfo) {
+ switch(((PrimitiveTypeInfo)basePrimitiveTypeInfo).getPrimitiveCategory()) {
+ case BOOLEAN:
+ return Type.BOOLEAN;
+ case BYTE:
+ return Type.TINYINT;
+ case DOUBLE:
+ return Type.DOUBLE;
+ case FLOAT:
+ return Type.FLOAT;
+ case INT:
+ return Type.INT;
+ case LONG:
+ return Type.BIGINT;
+ case SHORT:
+ return Type.SMALLINT;
+ case STRING:
+ return Type.STRING;
+ default:
+ throw new TypeNotPresentException(((PrimitiveTypeInfo)basePrimitiveTypeInfo).getTypeName(), null);
+ }
+ }
+
+ public static HCatSchema getHCatSchema(Schema schema) throws HCatException{
+ return getHCatSchema(schema.getFieldSchemas());
+ }
+
+ public static HCatSchema getHCatSchema(List<? extends FieldSchema> fslist) throws HCatException{
+ CollectionBuilder builder = getStructSchemaBuilder();
+ for (FieldSchema fieldSchema : fslist){
+ builder.addField(fieldSchema);
+ }
+ return builder.build();
+ }
+
+ private static HCatSchema constructHCatSchema(StructTypeInfo stypeInfo) throws HCatException {
+ CollectionBuilder builder = getStructSchemaBuilder();
+ for (String fieldName : ((StructTypeInfo)stypeInfo).getAllStructFieldNames()){
+ builder.addField(getHCatFieldSchema(fieldName,((StructTypeInfo)stypeInfo).getStructFieldTypeInfo(fieldName)));
+ }
+ return builder.build();
+ }
+
+ public static HCatSchema getHCatSchema(TypeInfo typeInfo) throws HCatException {
+ Category typeCategory = typeInfo.getCategory();
+ if (Category.PRIMITIVE == typeCategory){
+ return getStructSchemaBuilder().addField(new HCatFieldSchema(null,getPrimitiveHType(typeInfo),null)).build();
+ } else if (Category.STRUCT == typeCategory) {
+ HCatSchema subSchema = constructHCatSchema((StructTypeInfo) typeInfo);
+ return getStructSchemaBuilder().addField(new HCatFieldSchema(null,Type.STRUCT,subSchema,null)).build();
+ } else if (Category.LIST == typeCategory) {
+ CollectionBuilder builder = getStructSchemaBuilder();
+ builder.addField(getHCatFieldSchema(null,((ListTypeInfo)typeInfo).getListElementTypeInfo()));
+ return builder.build();
+ } else if (Category.MAP == typeCategory) {
+ HCatFieldSchema.Type mapKeyType = getPrimitiveHType(((MapTypeInfo)typeInfo).getMapKeyTypeInfo());
+ HCatSchema subSchema = getHCatSchema(((MapTypeInfo)typeInfo).getMapValueTypeInfo());
+ MapBuilder builder = getMapSchemaBuilder();
+ return builder.withKeyType(mapKeyType).withValueSchema(subSchema).build();
+ } else{
+ throw new TypeNotPresentException(typeInfo.getTypeName(),null);
+ }
+ }
+
+ public static HCatSchema getHCatSchemaFromTypeString(String typeString) throws HCatException {
+ return getHCatSchema(TypeInfoUtils.getTypeInfoFromTypeString(typeString));
+ }
+
+ public static HCatSchema getHCatSchema(String schemaString) throws HCatException {
+ if ((schemaString == null) || (schemaString.trim().isEmpty())){
+ return new HCatSchema(new ArrayList<HCatFieldSchema>()); // empty HSchema construct
+ }
+ HCatSchema outerSchema = getHCatSchemaFromTypeString("struct<"+schemaString+">");
+ return outerSchema.get(0).getStructSubSchema();
+ }
+
+ public static FieldSchema getFieldSchema(HCatFieldSchema howlFieldSchema){
+ return new FieldSchema(howlFieldSchema.getName(),howlFieldSchema.getTypeString(),howlFieldSchema.getComment());
+ }
+
+ public static List<FieldSchema> getFieldSchemas(List<HCatFieldSchema> howlFieldSchemas){
+ List<FieldSchema> lfs = new ArrayList<FieldSchema>();
+ for (HCatFieldSchema hfs : howlFieldSchemas){
+ lfs.add(getFieldSchema(hfs));
+ }
+ return lfs;
+ }
+}