You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by kw...@apache.org on 2016/09/30 02:14:42 UTC
[25/61] [partial] incubator-impala git commit: IMPALA-3786: Replace
"cloudera" with "apache" (part 1)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/Type.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/Type.java b/fe/src/main/java/com/cloudera/impala/catalog/Type.java
deleted file mode 100644
index 0162ec6..0000000
--- a/fe/src/main/java/com/cloudera/impala/catalog/Type.java
+++ /dev/null
@@ -1,781 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.catalog;
-
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.cloudera.impala.analysis.CreateTableStmt;
-import com.cloudera.impala.analysis.SqlParser;
-import com.cloudera.impala.analysis.SqlScanner;
-import com.cloudera.impala.analysis.TypeDef;
-import com.cloudera.impala.common.Pair;
-import com.cloudera.impala.thrift.TColumnType;
-import com.cloudera.impala.thrift.TPrimitiveType;
-import com.cloudera.impala.thrift.TScalarType;
-import com.cloudera.impala.thrift.TStructField;
-import com.cloudera.impala.thrift.TTypeNode;
-import com.cloudera.impala.thrift.TTypeNodeType;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * Abstract class describing an Impala data type (scalar/complex type).
- * Mostly contains static type instances and helper methods for convenience, as well
- * as abstract methods that subclasses must implement.
- */
-public abstract class Type {
- // Maximum nesting depth of a type. This limit was determined experimentally by
- // generating and scanning deeply nested Parquet and Avro files. In those experiments,
- // we exceeded the stack space in the scanner (which uses recursion for dealing with
- // nested types) at a nesting depth between 200 and 300 (200 worked, 300 crashed).
- public static int MAX_NESTING_DEPTH = 100;
-
- // Static constant types for scalar types that don't require additional information.
- public static final ScalarType INVALID = new ScalarType(PrimitiveType.INVALID_TYPE);
- public static final ScalarType NULL = new ScalarType(PrimitiveType.NULL_TYPE);
- public static final ScalarType BOOLEAN = new ScalarType(PrimitiveType.BOOLEAN);
- public static final ScalarType TINYINT = new ScalarType(PrimitiveType.TINYINT);
- public static final ScalarType SMALLINT = new ScalarType(PrimitiveType.SMALLINT);
- public static final ScalarType INT = new ScalarType(PrimitiveType.INT);
- public static final ScalarType BIGINT = new ScalarType(PrimitiveType.BIGINT);
- public static final ScalarType FLOAT = new ScalarType(PrimitiveType.FLOAT);
- public static final ScalarType DOUBLE = new ScalarType(PrimitiveType.DOUBLE);
- public static final ScalarType STRING = new ScalarType(PrimitiveType.STRING);
- public static final ScalarType BINARY = new ScalarType(PrimitiveType.BINARY);
- public static final ScalarType TIMESTAMP = new ScalarType(PrimitiveType.TIMESTAMP);
- public static final ScalarType DATE = new ScalarType(PrimitiveType.DATE);
- public static final ScalarType DATETIME = new ScalarType(PrimitiveType.DATETIME);
- public static final ScalarType DEFAULT_DECIMAL = (ScalarType)
- ScalarType.createDecimalType(ScalarType.DEFAULT_PRECISION,
- ScalarType.DEFAULT_SCALE);
- public static final ScalarType DECIMAL =
- (ScalarType) ScalarType.createDecimalTypeInternal(-1, -1);
- public static final ScalarType DEFAULT_VARCHAR = ScalarType.createVarcharType(-1);
- public static final ScalarType VARCHAR = ScalarType.createVarcharType(-1);
- public static final ScalarType CHAR = (ScalarType) ScalarType.createCharType(-1);
-
- private static ArrayList<ScalarType> integerTypes;
- private static ArrayList<ScalarType> numericTypes;
- private static ArrayList<ScalarType> supportedTypes;
- private static ArrayList<ScalarType> unsupportedTypes;
-
- static {
- integerTypes = Lists.newArrayList();
- integerTypes.add(TINYINT);
- integerTypes.add(SMALLINT);
- integerTypes.add(INT);
- integerTypes.add(BIGINT);
-
- numericTypes = Lists.newArrayList();
- numericTypes.add(TINYINT);
- numericTypes.add(SMALLINT);
- numericTypes.add(INT);
- numericTypes.add(BIGINT);
- numericTypes.add(FLOAT);
- numericTypes.add(DOUBLE);
- numericTypes.add(DECIMAL);
-
- supportedTypes = Lists.newArrayList();
- supportedTypes.add(NULL);
- supportedTypes.add(BOOLEAN);
- supportedTypes.add(TINYINT);
- supportedTypes.add(SMALLINT);
- supportedTypes.add(INT);
- supportedTypes.add(BIGINT);
- supportedTypes.add(FLOAT);
- supportedTypes.add(DOUBLE);
- supportedTypes.add(STRING);
- supportedTypes.add(VARCHAR);
- supportedTypes.add(CHAR);
- supportedTypes.add(TIMESTAMP);
- supportedTypes.add(DECIMAL);
-
- unsupportedTypes = Lists.newArrayList();
- unsupportedTypes.add(BINARY);
- unsupportedTypes.add(DATE);
- unsupportedTypes.add(DATETIME);
- }
-
- public static ArrayList<ScalarType> getIntegerTypes() {
- return integerTypes;
- }
- public static ArrayList<ScalarType> getNumericTypes() {
- return numericTypes;
- }
- public static ArrayList<ScalarType> getSupportedTypes() {
- return supportedTypes;
- }
- public static ArrayList<ScalarType> getUnsupportedTypes() {
- return unsupportedTypes;
- }
-
- /**
- * The output of this is stored directly in the hive metastore as the column type.
- * The string must match exactly.
- */
- public final String toSql() { return toSql(0); }
-
- /**
- * Recursive helper for toSql() to be implemented by subclasses. Keeps track of the
- * nesting depth and terminates the recursion if MAX_NESTING_DEPTH is reached.
- */
- protected abstract String toSql(int depth);
-
- /**
- * Same as toSql() but adds newlines and spaces for better readability of nested types.
- */
- public String prettyPrint() { return prettyPrint(0); }
-
- /**
- * Pretty prints this type with lpad number of leading spaces. Used to implement
- * prettyPrint() with space-indented nested types.
- */
- protected abstract String prettyPrint(int lpad);
-
- public boolean isInvalid() { return isScalarType(PrimitiveType.INVALID_TYPE); }
- public boolean isValid() { return !isInvalid(); }
- public boolean isNull() { return isScalarType(PrimitiveType.NULL_TYPE); }
- public boolean isBoolean() { return isScalarType(PrimitiveType.BOOLEAN); }
- public boolean isTimestamp() { return isScalarType(PrimitiveType.TIMESTAMP); }
- public boolean isDecimal() { return isScalarType(PrimitiveType.DECIMAL); }
- public boolean isDecimalOrNull() { return isDecimal() || isNull(); }
- public boolean isFullySpecifiedDecimal() { return false; }
- public boolean isWildcardDecimal() { return false; }
- public boolean isWildcardVarchar() { return false; }
- public boolean isWildcardChar() { return false; }
-
- public boolean isStringType() {
- return isScalarType(PrimitiveType.STRING) || isScalarType(PrimitiveType.VARCHAR) ||
- isScalarType(PrimitiveType.CHAR);
- }
-
- public boolean isScalarType() { return this instanceof ScalarType; }
- public boolean isScalarType(PrimitiveType t) {
- return isScalarType() && ((ScalarType) this).getPrimitiveType() == t;
- }
-
- public boolean isFixedPointType() {
- return isScalarType(PrimitiveType.TINYINT) || isScalarType(PrimitiveType.SMALLINT) ||
- isScalarType(PrimitiveType.INT) || isScalarType(PrimitiveType.BIGINT) ||
- isScalarType(PrimitiveType.DECIMAL);
- }
-
- public boolean isFloatingPointType() {
- return isScalarType(PrimitiveType.FLOAT) || isScalarType(PrimitiveType.DOUBLE);
- }
-
- public boolean isIntegerType() {
- return isScalarType(PrimitiveType.TINYINT) || isScalarType(PrimitiveType.SMALLINT)
- || isScalarType(PrimitiveType.INT) || isScalarType(PrimitiveType.BIGINT);
- }
-
- // TODO: Handle complex types properly. Some instances may be fixed length.
- public boolean isFixedLengthType() { return false; }
-
- public boolean isNumericType() {
- return isFixedPointType() || isFloatingPointType() || isDecimal();
- }
-
- public boolean isDateType() {
- return isScalarType(PrimitiveType.DATE) || isScalarType(PrimitiveType.DATETIME)
- || isScalarType(PrimitiveType.TIMESTAMP);
- }
-
- public boolean isComplexType() { return isStructType() || isCollectionType(); }
- public boolean isCollectionType() { return isMapType() || isArrayType(); }
- public boolean isMapType() { return this instanceof MapType; }
- public boolean isArrayType() { return this instanceof ArrayType; }
- public boolean isStructType() { return this instanceof StructType; }
-
- /**
- * Returns true if Impala supports this type in the metdata. It does not mean we
- * can manipulate data of this type. For tables that contain columns with these
- * types, we can safely skip over them.
- */
- public boolean isSupported() { return true; }
-
- /**
- * Indicates whether we support partitioning tables on columns of this type.
- */
- public boolean supportsTablePartitioning() { return false; }
-
- public PrimitiveType getPrimitiveType() { return PrimitiveType.INVALID_TYPE; }
-
- /**
- * Returns the size in bytes of the fixed-length portion that a slot of this type
- * occupies in a tuple.
- */
- public int getSlotSize() {
- // 8-byte pointer and 4-byte length indicator (12 bytes total).
- // Per struct alignment rules, there is an extra 4 bytes of padding to align to 8
- // bytes so 16 bytes total.
- if (isCollectionType()) return 16;
- throw new IllegalStateException("getSlotSize() not implemented for type " + toSql());
- }
-
- public TColumnType toThrift() {
- TColumnType container = new TColumnType();
- container.setTypes(new ArrayList<TTypeNode>());
- toThrift(container);
- return container;
- }
-
- /**
- * Subclasses should override this method to add themselves to the thrift container.
- */
- public abstract void toThrift(TColumnType container);
-
- /**
- * Returns true if this type is equal to t, or if t is a wildcard variant of this
- * type. Subclasses should override this as appropriate. The default implementation
- * here is to avoid special-casing logic in callers for concrete types.
- */
- public boolean matchesType(Type t) { return false; }
-
- /**
- * Gets the ColumnType from the given FieldSchema by using Impala's SqlParser.
- * Returns null if the FieldSchema could not be parsed.
- * The type can either be:
- * - Supported by Impala, in which case the type is returned.
- * - A type Impala understands but is not yet implemented (e.g. date), the type is
- * returned but type.IsSupported() returns false.
- * - A type Impala can't understand at all in which case null is returned.
- */
- public static Type parseColumnType(String typeStr) {
- // Wrap the type string in a CREATE TABLE stmt and use Impala's Parser
- // to get the ColumnType.
- // Pick a table name that can't be used.
- String stmt = String.format("CREATE TABLE $DUMMY ($DUMMY %s)", typeStr);
- SqlScanner input = new SqlScanner(new StringReader(stmt));
- SqlParser parser = new SqlParser(input);
- CreateTableStmt createTableStmt;
- try {
- Object o = parser.parse().value;
- if (!(o instanceof CreateTableStmt)) {
- // Should never get here.
- throw new IllegalStateException("Couldn't parse create table stmt.");
- }
- createTableStmt = (CreateTableStmt) o;
- if (createTableStmt.getColumnDefs().isEmpty()) {
- // Should never get here.
- throw new IllegalStateException("Invalid create table stmt.");
- }
- } catch (Exception e) {
- return null;
- }
- TypeDef typeDef = createTableStmt.getColumnDefs().get(0).getTypeDef();
- return typeDef.getType();
- }
-
- /**
- * Returns true if t1 can be implicitly cast to t2 according to Impala's casting rules.
- * Implicit casts are always allowed when no loss of precision would result (i.e. every
- * value of t1 can be represented exactly by a value of t2). Implicit casts are allowed
- * in certain other cases such as casting numeric types to floating point types and
- * converting strings to timestamps.
- * If strict is true, only consider casts that result in no loss of precision.
- * TODO: Support casting of non-scalar types.
- */
- public static boolean isImplicitlyCastable(Type t1, Type t2, boolean strict) {
- if (t1.isScalarType() && t2.isScalarType()) {
- return ScalarType.isImplicitlyCastable(
- (ScalarType) t1, (ScalarType) t2, strict);
- }
- return false;
- }
-
- /**
- * Return type t such that values from both t1 and t2 can be assigned to t without an
- * explicit cast. If strict, does not consider conversions that would result in loss
- * of precision (e.g. converting decimal to float). Returns INVALID_TYPE if there is
- * no such type or if any of t1 and t2 is INVALID_TYPE.
- * TODO: Support non-scalar types.
- */
- public static Type getAssignmentCompatibleType(Type t1, Type t2, boolean strict) {
- if (t1.isScalarType() && t2.isScalarType()) {
- return ScalarType.getAssignmentCompatibleType(
- (ScalarType) t1, (ScalarType) t2, strict);
- }
- return ScalarType.INVALID;
- }
-
- /**
- * Returns true if this type exceeds the MAX_NESTING_DEPTH, false otherwise.
- */
- public boolean exceedsMaxNestingDepth() { return exceedsMaxNestingDepth(0); }
-
- /**
- * Helper for exceedsMaxNestingDepth(). Recursively computes the max nesting depth,
- * terminating early if MAX_NESTING_DEPTH is reached. Returns true if this type
- * exceeds the MAX_NESTING_DEPTH, false otherwise.
- *
- * Examples of types and their nesting depth:
- * INT --> 1
- * STRUCT<f1:INT> --> 2
- * STRUCT<f1:STRUCT<f2:INT>> --> 3
- * ARRAY<INT> --> 2
- * ARRAY<STRUCT<f1:INT>> --> 3
- * MAP<STRING,INT> --> 2
- * MAP<STRING,STRUCT<f1:INT>> --> 3
- */
- private boolean exceedsMaxNestingDepth(int d) {
- if (d >= MAX_NESTING_DEPTH) return true;
- if (isStructType()) {
- StructType structType = (StructType) this;
- for (StructField f: structType.getFields()) {
- if (f.getType().exceedsMaxNestingDepth(d + 1)) return true;
- }
- } else if (isArrayType()) {
- ArrayType arrayType = (ArrayType) this;
- if (arrayType.getItemType().exceedsMaxNestingDepth(d + 1)) return true;
- } else if (isMapType()) {
- MapType mapType = (MapType) this;
- if (mapType.getValueType().exceedsMaxNestingDepth(d + 1)) return true;
- } else {
- Preconditions.checkState(isScalarType());
- }
- return false;
- }
-
- public static List<TColumnType> toThrift(Type[] types) {
- return toThrift(Lists.newArrayList(types));
- }
-
- public static List<TColumnType> toThrift(ArrayList<Type> types) {
- ArrayList<TColumnType> result = Lists.newArrayList();
- for (Type t: types) {
- result.add(t.toThrift());
- }
- return result;
- }
-
- public static Type fromThrift(TColumnType thrift) {
- Preconditions.checkState(thrift.types.size() > 0);
- Pair<Type, Integer> t = fromThrift(thrift, 0);
- Preconditions.checkState(t.second.equals(thrift.getTypesSize()));
- return t.first;
- }
-
- /**
- * Constructs a ColumnType rooted at the TTypeNode at nodeIdx in TColumnType.
- * Returned pair: The resulting ColumnType and the next nodeIdx that is not a child
- * type of the result.
- */
- protected static Pair<Type, Integer> fromThrift(TColumnType col, int nodeIdx) {
- TTypeNode node = col.getTypes().get(nodeIdx);
- Type type = null;
- switch (node.getType()) {
- case SCALAR: {
- Preconditions.checkState(node.isSetScalar_type());
- TScalarType scalarType = node.getScalar_type();
- if (scalarType.getType() == TPrimitiveType.CHAR) {
- Preconditions.checkState(scalarType.isSetLen());
- type = ScalarType.createCharType(scalarType.getLen());
- } else if (scalarType.getType() == TPrimitiveType.VARCHAR) {
- Preconditions.checkState(scalarType.isSetLen());
- type = ScalarType.createVarcharType(scalarType.getLen());
- } else if (scalarType.getType() == TPrimitiveType.DECIMAL) {
- Preconditions.checkState(scalarType.isSetPrecision()
- && scalarType.isSetScale());
- type = ScalarType.createDecimalType(scalarType.getPrecision(),
- scalarType.getScale());
- } else {
- type = ScalarType.createType(
- PrimitiveType.fromThrift(scalarType.getType()));
- }
- ++nodeIdx;
- break;
- }
- case ARRAY: {
- Preconditions.checkState(nodeIdx + 1 < col.getTypesSize());
- Pair<Type, Integer> childType = fromThrift(col, nodeIdx + 1);
- type = new ArrayType(childType.first);
- nodeIdx = childType.second;
- break;
- }
- case MAP: {
- Preconditions.checkState(nodeIdx + 2 < col.getTypesSize());
- Pair<Type, Integer> keyType = fromThrift(col, nodeIdx + 1);
- Pair<Type, Integer> valueType = fromThrift(col, keyType.second);
- type = new MapType(keyType.first, valueType.first);
- nodeIdx = valueType.second;
- break;
- }
- case STRUCT: {
- Preconditions.checkState(nodeIdx + node.getStruct_fieldsSize() < col.getTypesSize());
- ArrayList<StructField> structFields = Lists.newArrayList();
- ++nodeIdx;
- for (int i = 0; i < node.getStruct_fieldsSize(); ++i) {
- TStructField thriftField = node.getStruct_fields().get(i);
- String name = thriftField.getName();
- String comment = null;
- if (thriftField.isSetComment()) comment = thriftField.getComment();
- Pair<Type, Integer> res = fromThrift(col, nodeIdx);
- nodeIdx = res.second.intValue();
- structFields.add(new StructField(name, res.first, comment));
- }
- type = new StructType(structFields);
- break;
- }
- }
- return new Pair<Type, Integer>(type, nodeIdx);
- }
-
- /**
- * Utility function to get the primitive type of a thrift type that is known
- * to be scalar.
- */
- public TPrimitiveType getTPrimitiveType(TColumnType ttype) {
- Preconditions.checkState(ttype.getTypesSize() == 1);
- Preconditions.checkState(ttype.types.get(0).getType() == TTypeNodeType.SCALAR);
- return ttype.types.get(0).scalar_type.getType();
- }
-
- /**
- * JDBC data type description
- * Returns the column size for this type.
- * For numeric data this is the maximum precision.
- * For character data this is the length in characters.
- * For datetime types this is the length in characters of the String representation
- * (assuming the maximum allowed precision of the fractional seconds component).
- * For binary data this is the length in bytes.
- * Null is returned for for data types where the column size is not applicable.
- */
- public Integer getColumnSize() {
- if (!isScalarType()) return null;
- if (isNumericType()) return getPrecision();
- ScalarType t = (ScalarType) this;
- switch (t.getPrimitiveType()) {
- case STRING:
- return Integer.MAX_VALUE;
- case TIMESTAMP:
- return 29;
- case CHAR:
- case VARCHAR:
- return t.getLength();
- default:
- return null;
- }
- }
-
- /**
- * JDBC data type description
- * For numeric types, returns the maximum precision for this type.
- * For non-numeric types, returns null.
- */
- public Integer getPrecision() {
- if (!isScalarType()) return null;
- ScalarType t = (ScalarType) this;
- switch (t.getPrimitiveType()) {
- case TINYINT:
- return 3;
- case SMALLINT:
- return 5;
- case INT:
- return 10;
- case BIGINT:
- return 19;
- case FLOAT:
- return 7;
- case DOUBLE:
- return 15;
- case DECIMAL:
- return t.decimalPrecision();
- default:
- return null;
- }
- }
-
- /**
- * JDBC data type description
- * Returns the number of fractional digits for this type, or null if not applicable.
- * For timestamp/time types, returns the number of digits in the fractional seconds
- * component.
- */
- public Integer getDecimalDigits() {
- if (!isScalarType()) return null;
- ScalarType t = (ScalarType) this;
- switch (t.getPrimitiveType()) {
- case BOOLEAN:
- case TINYINT:
- case SMALLINT:
- case INT:
- case BIGINT:
- return 0;
- case FLOAT:
- return 7;
- case DOUBLE:
- return 15;
- case TIMESTAMP:
- return 9;
- case DECIMAL:
- return t.decimalScale();
- default:
- return null;
- }
- }
-
- /**
- * JDBC data type description
- * For numeric data types, either 10 or 2. If it is 10, the values in COLUMN_SIZE
- * and DECIMAL_DIGITS give the number of decimal digits allowed for the column.
- * For example, a DECIMAL(12,5) column would return a NUM_PREC_RADIX of 10,
- * a COLUMN_SIZE of 12, and a DECIMAL_DIGITS of 5; a FLOAT column could return
- * a NUM_PREC_RADIX of 10, a COLUMN_SIZE of 15, and a DECIMAL_DIGITS of NULL.
- * If it is 2, the values in COLUMN_SIZE and DECIMAL_DIGITS give the number of bits
- * allowed in the column. For example, a FLOAT column could return a RADIX of 2,
- * a COLUMN_SIZE of 53, and a DECIMAL_DIGITS of NULL. NULL is returned for data
- * types where NUM_PREC_RADIX is not applicable.
- */
- public Integer getNumPrecRadix() {
- if (!isScalarType()) return null;
- ScalarType t = (ScalarType) this;
- switch (t.getPrimitiveType()) {
- case TINYINT:
- case SMALLINT:
- case INT:
- case BIGINT:
- case FLOAT:
- case DOUBLE:
- case DECIMAL:
- return 10;
- default:
- // everything else (including boolean and string) is null
- return null;
- }
- }
-
- /**
- * JDBC data type description
- * Returns the java SQL type enum
- */
- public int getJavaSqlType() {
- if (isStructType()) return java.sql.Types.STRUCT;
- // Both MAP and ARRAY are reported as ARRAY, since there is no better matching
- // Java SQL type. This behavior is consistent with Hive.
- if (isCollectionType()) return java.sql.Types.ARRAY;
-
- Preconditions.checkState(isScalarType(), "Invalid non-scalar type: " + toSql());
- ScalarType t = (ScalarType) this;
- switch (t.getPrimitiveType()) {
- case NULL_TYPE: return java.sql.Types.NULL;
- case BOOLEAN: return java.sql.Types.BOOLEAN;
- case TINYINT: return java.sql.Types.TINYINT;
- case SMALLINT: return java.sql.Types.SMALLINT;
- case INT: return java.sql.Types.INTEGER;
- case BIGINT: return java.sql.Types.BIGINT;
- case FLOAT: return java.sql.Types.FLOAT;
- case DOUBLE: return java.sql.Types.DOUBLE;
- case TIMESTAMP: return java.sql.Types.TIMESTAMP;
- case STRING: return java.sql.Types.VARCHAR;
- case CHAR: return java.sql.Types.CHAR;
- case VARCHAR: return java.sql.Types.VARCHAR;
- case BINARY: return java.sql.Types.BINARY;
- case DECIMAL: return java.sql.Types.DECIMAL;
- default:
- Preconditions.checkArgument(false, "Invalid primitive type " +
- t.getPrimitiveType().name());
- return 0;
- }
- }
-
- /**
- * Matrix that records "smallest" assignment-compatible type of two types
- * (INVALID_TYPE if no such type exists, ie, if the input types are fundamentally
- * incompatible). A value of any of the two types could be assigned to a slot
- * of the assignment-compatible type. For strict compatibility, this can be done
- * without any loss of precision. For non-strict compatibility, there may be loss of
- * precision, e.g. if converting from BIGINT to FLOAT.
- *
- * We chose not to follow MySQL's type casting behavior as described here:
- * http://dev.mysql.com/doc/refman/5.0/en/type-conversion.html
- * for the following reasons:
- * conservative casting in arithmetic exprs: TINYINT + TINYINT -> BIGINT
- * comparison of many types as double: INT < FLOAT -> comparison as DOUBLE
- * special cases when dealing with dates and timestamps.
- */
- protected static PrimitiveType[][] compatibilityMatrix;
-
- /**
- * If we are checking in strict mode, any non-null entry in this matrix overrides
- * compatibilityMatrix. If the entry is null, the entry in compatibility matrix
- * is valid.
- */
- protected static PrimitiveType[][] strictCompatibilityMatrix;
-
- static {
- compatibilityMatrix = new
- PrimitiveType[PrimitiveType.values().length][PrimitiveType.values().length];
- strictCompatibilityMatrix = new
- PrimitiveType[PrimitiveType.values().length][PrimitiveType.values().length];
-
- for (int i = 0; i < PrimitiveType.values().length; ++i) {
- // Each type is compatible with itself.
- compatibilityMatrix[i][i] = PrimitiveType.values()[i];
- // BINARY is not supported.
- compatibilityMatrix[BINARY.ordinal()][i] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[i][BINARY.ordinal()] = PrimitiveType.INVALID_TYPE;
- }
-
- compatibilityMatrix[BOOLEAN.ordinal()][TINYINT.ordinal()] = PrimitiveType.TINYINT;
- compatibilityMatrix[BOOLEAN.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT;
- compatibilityMatrix[BOOLEAN.ordinal()][INT.ordinal()] = PrimitiveType.INT;
- compatibilityMatrix[BOOLEAN.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
- compatibilityMatrix[BOOLEAN.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
- compatibilityMatrix[BOOLEAN.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
- compatibilityMatrix[BOOLEAN.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BOOLEAN.ordinal()][DATETIME.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BOOLEAN.ordinal()][TIMESTAMP.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BOOLEAN.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- compatibilityMatrix[TINYINT.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT;
- compatibilityMatrix[TINYINT.ordinal()][INT.ordinal()] = PrimitiveType.INT;
- compatibilityMatrix[TINYINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
- // 8 bit integer fits in mantissa of both float and double.
- compatibilityMatrix[TINYINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
- compatibilityMatrix[TINYINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
- compatibilityMatrix[TINYINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[TINYINT.ordinal()][DATETIME.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[TINYINT.ordinal()][TIMESTAMP.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[TINYINT.ordinal()][VARCHAR.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[TINYINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- compatibilityMatrix[SMALLINT.ordinal()][INT.ordinal()] = PrimitiveType.INT;
- compatibilityMatrix[SMALLINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
- // 16 bit integer fits in mantissa of both float and double.
- compatibilityMatrix[SMALLINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
- compatibilityMatrix[SMALLINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
- compatibilityMatrix[SMALLINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[SMALLINT.ordinal()][DATETIME.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[SMALLINT.ordinal()][TIMESTAMP.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[SMALLINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- compatibilityMatrix[INT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
- // 32 bit integer fits only mantissa of double.
- // TODO: arguably we should promote INT + FLOAT to DOUBLE to avoid loss of precision,
- // but we depend on it remaining FLOAT for some use cases, e.g.
- // "insert into tbl (float_col) select int_col + float_col from ..."
- compatibilityMatrix[INT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
- strictCompatibilityMatrix[INT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE;
- compatibilityMatrix[INT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
- compatibilityMatrix[INT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[INT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[INT.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[INT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[INT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[INT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- // 64 bit integer does not fit in mantissa of double or float.
- // TODO: arguably we should always promote BIGINT + FLOAT to double here to keep as
- // much precision as possible, but we depend on this implicit cast for some use
- // cases, similarly to INT + FLOAT.
- compatibilityMatrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
- strictCompatibilityMatrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE;
- // TODO: we're breaking the definition of strict compatibility for BIGINT + DOUBLE,
- // but this forces function overloading to consider the DOUBLE overload first.
- compatibilityMatrix[BIGINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
- compatibilityMatrix[BIGINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BIGINT.ordinal()][DATETIME.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BIGINT.ordinal()][TIMESTAMP.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BIGINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[BIGINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- compatibilityMatrix[FLOAT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
- compatibilityMatrix[FLOAT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[FLOAT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[FLOAT.ordinal()][TIMESTAMP.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[FLOAT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[FLOAT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- compatibilityMatrix[DOUBLE.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DOUBLE.ordinal()][DATETIME.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DOUBLE.ordinal()][TIMESTAMP.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DOUBLE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DOUBLE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- compatibilityMatrix[DATE.ordinal()][DATETIME.ordinal()] = PrimitiveType.DATETIME;
- compatibilityMatrix[DATE.ordinal()][TIMESTAMP.ordinal()] = PrimitiveType.TIMESTAMP;
- compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DATE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DATE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- compatibilityMatrix[DATETIME.ordinal()][TIMESTAMP.ordinal()] =
- PrimitiveType.TIMESTAMP;
- compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DATETIME.ordinal()][VARCHAR.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] =
- PrimitiveType.INVALID_TYPE;
-
- // We can convert some but not all string values to timestamps.
- compatibilityMatrix[TIMESTAMP.ordinal()][STRING.ordinal()] =
- PrimitiveType.TIMESTAMP;
- strictCompatibilityMatrix[TIMESTAMP.ordinal()][STRING.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[TIMESTAMP.ordinal()][VARCHAR.ordinal()] =
- PrimitiveType.INVALID_TYPE;
- compatibilityMatrix[TIMESTAMP.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- compatibilityMatrix[STRING.ordinal()][VARCHAR.ordinal()] = PrimitiveType.STRING;
- compatibilityMatrix[STRING.ordinal()][CHAR.ordinal()] = PrimitiveType.STRING;
-
- compatibilityMatrix[VARCHAR.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
-
- // Check all of the necessary entries that should be filled.
- for (int i = 0; i < PrimitiveType.values().length; ++i) {
- for (int j = i; j < PrimitiveType.values().length; ++j) {
- PrimitiveType t1 = PrimitiveType.values()[i];
- PrimitiveType t2 = PrimitiveType.values()[j];
- // DECIMAL, NULL, and INVALID_TYPE are handled separately.
- if (t1 == PrimitiveType.INVALID_TYPE ||
- t2 == PrimitiveType.INVALID_TYPE) continue;
- if (t1 == PrimitiveType.NULL_TYPE || t2 == PrimitiveType.NULL_TYPE) continue;
- if (t1 == PrimitiveType.DECIMAL || t2 == PrimitiveType.DECIMAL) continue;
- Preconditions.checkNotNull(compatibilityMatrix[i][j]);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/View.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/View.java b/fe/src/main/java/com/cloudera/impala/catalog/View.java
deleted file mode 100644
index cc82f95..0000000
--- a/fe/src/main/java/com/cloudera/impala/catalog/View.java
+++ /dev/null
@@ -1,207 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.catalog;
-
-import java.io.StringReader;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-
-import com.cloudera.impala.analysis.ParseNode;
-import com.cloudera.impala.analysis.QueryStmt;
-import com.cloudera.impala.analysis.SqlParser;
-import com.cloudera.impala.analysis.SqlScanner;
-import com.cloudera.impala.thrift.TCatalogObjectType;
-import com.cloudera.impala.thrift.TTable;
-import com.cloudera.impala.thrift.TTableDescriptor;
-import com.cloudera.impala.thrift.TTableType;
-import com.google.common.collect.Lists;
-
-/**
- * Table metadata representing a catalog view or a local view from a WITH clause.
- * Most methods inherited from Table are not supposed to be called on this class because
- * views are substituted with their underlying definition during analysis of a statement.
- *
- * Refreshing or invalidating a view will reload the view's definition but will not
- * affect the metadata of the underlying tables (if any).
- */
-public class View extends Table {
-
- // The original SQL-string given as view definition. Set during analysis.
- // Corresponds to Hive's viewOriginalText.
- private String originalViewDef_;
-
- // Query statement (as SQL string) that defines the View for view substitution.
- // It is a transformation of the original view definition, e.g., to enforce the
- // explicit column definitions even if the original view definition has explicit
- // column aliases.
- // If column definitions were given, then this "expanded" view definition
- // wraps the original view definition in a select stmt as follows.
- //
- // SELECT viewName.origCol1 AS colDesc1, viewName.origCol2 AS colDesc2, ...
- // FROM (originalViewDef) AS viewName
- //
- // Corresponds to Hive's viewExpandedText, but is not identical to the SQL
- // Hive would produce in view creation.
- private String inlineViewDef_;
-
- // View definition created by parsing inlineViewDef_ into a QueryStmt.
- private QueryStmt queryStmt_;
-
- // Set if this View is from a WITH clause and not persisted in the catalog.
- private final boolean isLocalView_;
-
- // Set if this View is from a WITH clause with column labels.
- private List<String> colLabels_;
-
- public View(TableId id, org.apache.hadoop.hive.metastore.api.Table msTable,
- Db db, String name, String owner) {
- super(id, msTable, db, name, owner);
- isLocalView_ = false;
- }
-
- /**
- * C'tor for WITH-clause views that already have a parsed QueryStmt and an optional
- * list of column labels.
- */
- public View(String alias, QueryStmt queryStmt, List<String> colLabels) {
- super(null, null, null, alias, null);
- isLocalView_ = true;
- queryStmt_ = queryStmt;
- colLabels_ = colLabels;
- }
-
- /**
- * Creates a view for testig purposes.
- */
- private View(Db db, String name, QueryStmt queryStmt) {
- super(null, null, db, name, null);
- isLocalView_ = false;
- queryStmt_ = queryStmt;
- colLabels_ = null;
- }
-
- @Override
- public void load(boolean reuseMetadata, IMetaStoreClient client,
- org.apache.hadoop.hive.metastore.api.Table msTbl) throws TableLoadingException {
- try {
- clearColumns();
- msTable_ = msTbl;
- // Load columns.
- List<FieldSchema> fieldSchemas = client.getFields(db_.getName(), name_);
- for (int i = 0; i < fieldSchemas.size(); ++i) {
- FieldSchema s = fieldSchemas.get(i);
- Type type = parseColumnType(s);
- Column col = new Column(s.getName(), type, s.getComment(), i);
- addColumn(col);
- }
- // These fields are irrelevant for views.
- numClusteringCols_ = 0;
- numRows_ = -1;
- init();
- } catch (TableLoadingException e) {
- throw e;
- } catch (Exception e) {
- throw new TableLoadingException("Failed to load metadata for view: " + name_, e);
- }
- }
-
- @Override
- protected void loadFromThrift(TTable t) throws TableLoadingException {
- super.loadFromThrift(t);
- init();
- }
-
- /**
- * Initializes the originalViewDef_, inlineViewDef_, and queryStmt_ members
- * by parsing the expanded view definition SQL-string.
- * Throws a TableLoadingException if there was any error parsing the
- * the SQL or if the view definition did not parse into a QueryStmt.
- */
- private void init() throws TableLoadingException {
- // Set view-definition SQL strings.
- originalViewDef_ = getMetaStoreTable().getViewOriginalText();
- inlineViewDef_ = getMetaStoreTable().getViewExpandedText();
- // Parse the expanded view definition SQL-string into a QueryStmt and
- // populate a view definition.
- SqlScanner input = new SqlScanner(new StringReader(inlineViewDef_));
- SqlParser parser = new SqlParser(input);
- ParseNode node = null;
- try {
- node = (ParseNode) parser.parse().value;
- } catch (Exception e) {
- // Do not pass e as the exception cause because it might reveal the existence
- // of tables that the user triggering this load may not have privileges on.
- throw new TableLoadingException(
- String.format("Failed to parse view-definition statement of view: " +
- "%s.%s", db_.getName(), name_));
- }
- // Make sure the view definition parses to a query statement.
- if (!(node instanceof QueryStmt)) {
- throw new TableLoadingException(String.format("View definition of %s.%s " +
- "is not a query statement", db_.getName(), name_));
- }
- queryStmt_ = (QueryStmt) node;
- }
-
- @Override
- public TCatalogObjectType getCatalogObjectType() { return TCatalogObjectType.VIEW; }
- public QueryStmt getQueryStmt() { return queryStmt_; }
- public String getOriginalViewDef() { return originalViewDef_; }
- public String getInlineViewDef() { return inlineViewDef_; }
- public boolean isLocalView() { return isLocalView_; }
-
- /**
- * Returns the column labels the user specified in the WITH-clause.
- */
- public List<String> getOriginalColLabels() { return colLabels_; }
-
- /**
- * Returns the explicit column labels for this view, or null if they need to be derived
- * entirely from the underlying query statement. The returned list has at least as many
- * elements as the number of column labels in the query stmt.
- */
- public List<String> getColLabels() {
- if (colLabels_ == null) return null;
- if (colLabels_.size() >= queryStmt_.getColLabels().size()) return colLabels_;
- List<String> explicitColLabels = Lists.newArrayList(colLabels_);
- explicitColLabels.addAll(queryStmt_.getColLabels().subList(
- colLabels_.size(), queryStmt_.getColLabels().size()));
- return explicitColLabels;
- }
-
- public boolean hasColLabels() { return colLabels_ != null; }
-
- @Override
- public TTableDescriptor toThriftDescriptor(Set<Long> referencedPartitions) {
- throw new IllegalStateException("Cannot call toThriftDescriptor() on a view.");
- }
-
- @Override
- public TTable toThrift() {
- TTable view = super.toThrift();
- view.setTable_type(TTableType.VIEW);
- return view;
- }
-
- public static View createTestView(Db db, String name, QueryStmt viewDefStmt) {
- return new View(db, name, viewDefStmt);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/delegates/DdlDelegate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/delegates/DdlDelegate.java b/fe/src/main/java/com/cloudera/impala/catalog/delegates/DdlDelegate.java
deleted file mode 100644
index a21bd90..0000000
--- a/fe/src/main/java/com/cloudera/impala/catalog/delegates/DdlDelegate.java
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.catalog.delegates;
-
-import java.util.List;
-
-import com.cloudera.impala.thrift.TDistributeParam;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-import com.cloudera.impala.common.ImpalaRuntimeException;
-import com.cloudera.impala.thrift.TAlterTableParams;
-
-/**
- * Abstract class to implement the storage specific portion of DDL requests.
- *
- * During catalog DDL operations the CatalogOpExecutor will instantiate the correct
- * subclass of this class to handle the DDL operation to the storage backend. See,
- * CatalogOpExecutor::createDDLDelegate() for details.
- *
- */
-public abstract class DdlDelegate {
-
- protected Table msTbl_;
- protected TAlterTableParams tAlterTableParams_;
- protected List<TDistributeParam> distributeParams_;
-
- /**
- * Creates a new delegate to modify Table 'msTbl'.
- */
- public DdlDelegate setMsTbl(Table msTbl) {
- msTbl_ = msTbl;
- return this;
- }
-
- public DdlDelegate setAlterTableParams(TAlterTableParams p) {
- tAlterTableParams_ = p;
- return this;
- }
-
- public DdlDelegate setDistributeParams(List<TDistributeParam> p) {
- distributeParams_ = p;
- return this;
- }
-
- /**
- * Creates the table.
- */
- public abstract void createTable() throws ImpalaRuntimeException;
-
- /**
- * Drops the table.
- */
- public abstract void dropTable() throws ImpalaRuntimeException;
-
- /**
- * Performs an alter table with the parameters set with setAlterTableParams().
- */
- public abstract boolean alterTable() throws ImpalaRuntimeException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java b/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java
deleted file mode 100644
index ecfeb1a..0000000
--- a/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java
+++ /dev/null
@@ -1,190 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.catalog.delegates;
-
-import static com.cloudera.impala.util.KuduUtil.compareSchema;
-import static com.cloudera.impala.util.KuduUtil.fromImpalaType;
-import static com.cloudera.impala.util.KuduUtil.parseKeyColumns;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.kudu.ColumnSchema;
-import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder;
-import org.apache.kudu.Schema;
-import org.apache.kudu.Type;
-import org.apache.kudu.client.CreateTableOptions;
-import org.apache.kudu.client.KuduClient;
-import org.apache.kudu.client.PartialRow;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.impala.catalog.KuduTable;
-import com.cloudera.impala.common.ImpalaRuntimeException;
-import com.cloudera.impala.thrift.TDistributeParam;
-import com.cloudera.impala.util.KuduUtil;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-
-/**
- * Implementation of the Kudu DDL Delegate. Propagates create and drop table statements to
- * Kudu.
- */
-public class KuduDdlDelegate extends DdlDelegate {
-
- private static final Logger LOG = LoggerFactory.getLogger(KuduDdlDelegate.class);
-
- public KuduDdlDelegate(Table msTbl) {
- setMsTbl(msTbl);
- }
-
- /**
- * Creates the Kudu table if it does not exist and returns true. If the table exists and
- * the table is not a managed table ignore and return false, otherwise throw an
- * exception.
- */
- @Override
- public void createTable()
- throws ImpalaRuntimeException {
-
- String kuduTableName = msTbl_.getParameters().get(KuduTable.KEY_TABLE_NAME);
- String kuduMasters = msTbl_.getParameters().get(KuduTable.KEY_MASTER_ADDRESSES);
-
- // Can be optional for un-managed tables
- String kuduKeyCols = msTbl_.getParameters().get(KuduTable.KEY_KEY_COLUMNS);
-
- String replication = msTbl_.getParameters().get(KuduTable.KEY_TABLET_REPLICAS);
-
- try (KuduClient client = new KuduClient.KuduClientBuilder(kuduMasters).build()) {
- // TODO should we throw if the table does not exist when its an external table?
- if (client.tableExists(kuduTableName)) {
- if (msTbl_.getTableType().equals(TableType.MANAGED_TABLE.toString())) {
- throw new ImpalaRuntimeException(String.format(
- "Table %s already exists in Kudu master %s.", kuduTableName, kuduMasters));
- }
-
- // Check if the external table matches the schema
- org.apache.kudu.client.KuduTable kuduTable = client.openTable(kuduTableName);
- if (!compareSchema(msTbl_, kuduTable)) {
- throw new ImpalaRuntimeException(String.format(
- "Table %s (%s) has a different schema in Kudu than in Hive.",
- msTbl_.getTableName(), kuduTableName));
- }
- return;
- }
-
- HashSet<String> keyColNames = parseKeyColumns(kuduKeyCols);
- List<ColumnSchema> keyColSchemas = new ArrayList<>();
-
- // Create a new Schema and map the types accordingly
- ArrayList<ColumnSchema> columns = Lists.newArrayList();
- for (FieldSchema fieldSchema: msTbl_.getSd().getCols()) {
- com.cloudera.impala.catalog.Type catalogType = com.cloudera.impala.catalog.Type
- .parseColumnType(fieldSchema.getType());
- if (catalogType == null) {
- throw new ImpalaRuntimeException(String.format(
- "Could not parse column type %s.", fieldSchema.getType()));
- }
- Type t = fromImpalaType(catalogType);
- // Create the actual column and check if the column is a key column
- ColumnSchemaBuilder csb = new ColumnSchemaBuilder(
- fieldSchema.getName(), t);
- boolean isKeyColumn = keyColNames.contains(fieldSchema.getName());
- csb.key(isKeyColumn);
- csb.nullable(!isKeyColumn);
- ColumnSchema cs = csb.build();
- columns.add(cs);
- if (isKeyColumn) keyColSchemas.add(cs);
- }
-
- Schema schema = new Schema(columns);
- CreateTableOptions cto = new CreateTableOptions();
-
- // Handle auto-partitioning of the Kudu table
- if (distributeParams_ != null) {
- for (TDistributeParam param : distributeParams_) {
- if (param.isSetBy_hash_param()) {
- Preconditions.checkState(!param.isSetBy_range_param());
- cto.addHashPartitions(param.getBy_hash_param().getColumns(),
- param.getBy_hash_param().getNum_buckets());
- } else {
- Preconditions.checkState(param.isSetBy_range_param());
- cto.setRangePartitionColumns(param.getBy_range_param().getColumns());
- for (PartialRow p : KuduUtil.parseSplits(schema, param.getBy_range_param())) {
- cto.addSplitRow(p);
- }
- }
- }
- }
-
- if (!Strings.isNullOrEmpty(replication)) {
- int r = Integer.parseInt(replication);
- if (r <= 0) {
- throw new ImpalaRuntimeException(
- "Number of tablet replicas must be greater than zero. " +
- "Given number of replicas is: " + Integer.toString(r));
- }
- cto.setNumReplicas(r);
- }
-
- client.createTable(kuduTableName, schema, cto);
- } catch (ImpalaRuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new ImpalaRuntimeException("Error creating Kudu table", e);
- }
- }
-
- @Override
- public void dropTable() throws ImpalaRuntimeException {
- // If table is an external table, do not delete the data
- if (msTbl_.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) return;
-
- String kuduTableName = msTbl_.getParameters().get(KuduTable.KEY_TABLE_NAME);
- String kuduMasters = msTbl_.getParameters().get(KuduTable.KEY_MASTER_ADDRESSES);
-
- try (KuduClient client = new KuduClient.KuduClientBuilder(kuduMasters).build()) {
- if (!client.tableExists(kuduTableName)) {
- LOG.warn("Table: %s is in inconsistent state. It does not exist in Kudu master(s)"
- + " %s, but it exists in Hive metastore. Deleting from metastore only.",
- kuduTableName, kuduMasters);
- return;
- }
- client.deleteTable(kuduTableName);
- return;
- } catch (Exception e) {
- throw new ImpalaRuntimeException("Error dropping Kudu table", e);
- }
- }
-
- public static boolean canHandle(org.apache.hadoop.hive.metastore.api.Table msTbl) {
- return KuduTable.isKuduTable(msTbl);
- }
-
- @Override
- public boolean alterTable() throws ImpalaRuntimeException {
- throw new ImpalaRuntimeException(
- "Alter table operations are not supported for Kudu tables.");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/catalog/delegates/UnsupportedOpDelegate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/delegates/UnsupportedOpDelegate.java b/fe/src/main/java/com/cloudera/impala/catalog/delegates/UnsupportedOpDelegate.java
deleted file mode 100644
index 52267be..0000000
--- a/fe/src/main/java/com/cloudera/impala/catalog/delegates/UnsupportedOpDelegate.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.catalog.delegates;
-
-import com.cloudera.impala.common.ImpalaRuntimeException;
-
-/**
- * Empty implementation for the DdlDelegate interface that does nothing.
- */
-public class UnsupportedOpDelegate extends DdlDelegate {
-
- @Override
- public void createTable() throws ImpalaRuntimeException { }
-
- @Override
- public void dropTable() throws ImpalaRuntimeException { }
-
- @Override
- public boolean alterTable() throws ImpalaRuntimeException { return true; }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/AliasGenerator.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/AliasGenerator.java b/fe/src/main/java/com/cloudera/impala/common/AliasGenerator.java
deleted file mode 100644
index 2d75d18..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/AliasGenerator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-import java.util.Set;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-
-/**
- * Abstract class representing an alias generator. It uses a prefix and a
- * monotonically increasing counter to generate new aliases. Classes extending
- * this class are responsible for initializing the prefix.
- */
-public abstract class AliasGenerator {
- private int numGeneratedAliases_ = 1;
- protected String aliasPrefix_ = null;
- protected Set<String> usedAliases_ = Sets.newHashSet();
-
- /**
- * Return the next available alias.
- */
- public String getNextAlias() {
- Preconditions.checkNotNull(aliasPrefix_);
- while (true) {
- String candidateAlias = aliasPrefix_ + Integer.toString(numGeneratedAliases_++);
- if (usedAliases_.add(candidateAlias)) return candidateAlias;
- if (numGeneratedAliases_ < 0) {
- throw new IllegalStateException("Overflow occured during alias generation.");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/AnalysisException.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/AnalysisException.java b/fe/src/main/java/com/cloudera/impala/common/AnalysisException.java
deleted file mode 100644
index 464cfa0..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/AnalysisException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-/**
- * Thrown for errors encountered during analysis of a SQL statement.
- *
- */
-public class AnalysisException extends ImpalaException {
- public AnalysisException(String msg, Throwable cause) {
- super(msg, cause);
- }
-
- public AnalysisException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/ByteUnits.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/ByteUnits.java b/fe/src/main/java/com/cloudera/impala/common/ByteUnits.java
deleted file mode 100644
index f8b43ab..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/ByteUnits.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-/**
- * Byte unit constants.
- */
-public class ByteUnits {
-
- /**
- * One kilobyte in bytes.
- */
- public final static long KILOBYTE = 1024;
-
- /**
- * One megabyte in bytes.
- */
- public final static long MEGABYTE = KILOBYTE * 1024;
-
- /**
- * One gigabyte in bytes.
- */
- public final static long GIGABYTE = MEGABYTE * 1024;
-
- /**
- * One terabyte in bytes.
- */
- public final static long TERABYTE = GIGABYTE * 1024;
-
- /**
- * One petabyte in bytes.
- */
- public final static long PETABYTE = TERABYTE * 1024;
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/ColumnAliasGenerator.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/ColumnAliasGenerator.java b/fe/src/main/java/com/cloudera/impala/common/ColumnAliasGenerator.java
deleted file mode 100644
index 51c8ece..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/ColumnAliasGenerator.java
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-import java.util.List;
-
-import com.google.common.base.Preconditions;
-
-public class ColumnAliasGenerator extends AliasGenerator {
- private static final String DEFAULT_COL_ALIAS_PREFIX = "$c$";
-
- public ColumnAliasGenerator(List<String> existingLabels, String prefix) {
- Preconditions.checkNotNull(existingLabels);
- aliasPrefix_ = prefix != null ? prefix : DEFAULT_COL_ALIAS_PREFIX;
- usedAliases_.addAll(existingLabels);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java b/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java
deleted file mode 100644
index 2239853..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java
+++ /dev/null
@@ -1,409 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.UUID;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3.S3FileSystem;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3native.NativeS3FileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.client.HdfsAdmin;
-import org.apache.hadoop.hdfs.protocol.EncryptionZone;
-import org.apache.log4j.Logger;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Common utility functions for operating on FileSystem objects.
- */
-public class FileSystemUtil {
- private static final Configuration CONF = new Configuration();
- private static final Logger LOG = Logger.getLogger(FileSystemUtil.class);
-
- /**
- * Performs a non-recursive delete of all visible (non-hidden) files in a given
- * directory. Returns the number of files deleted as part of this operation.
- */
- public static int deleteAllVisibleFiles(Path directory)
- throws IOException {
- FileSystem fs = directory.getFileSystem(CONF);
- Preconditions.checkState(fs.getFileStatus(directory).isDirectory());
- int numFilesDeleted = 0;
- for (FileStatus fStatus: fs.listStatus(directory)) {
- // Only delete files that are not hidden.
- if (fStatus.isFile() && !isHiddenFile(fStatus.getPath().getName())) {
- LOG.debug("Removing: " + fStatus.getPath());
- fs.delete(fStatus.getPath(), false);
- ++numFilesDeleted;
- }
- }
- return numFilesDeleted;
- }
-
- /**
- * Returns the total number of visible (non-hidden) files in a directory.
- */
- public static int getTotalNumVisibleFiles(Path directory) throws IOException {
- FileSystem fs = directory.getFileSystem(CONF);
- Preconditions.checkState(fs.getFileStatus(directory).isDirectory());
- int numFiles = 0;
- for (FileStatus fStatus: fs.listStatus(directory)) {
- // Only delete files that are not hidden.
- if (fStatus.isFile() && !isHiddenFile(fStatus.getPath().getName())) {
- ++numFiles;
- }
- }
- return numFiles;
- }
-
- /**
- * Returns true if path p1 and path p2 are in the same encryption zone in HDFS.
- * Returns false if they are in different encryption zones or if either of the paths
- * are not on HDFS.
- */
- private static boolean arePathsInSameHdfsEncryptionZone(FileSystem fs, Path p1,
- Path p2) throws IOException {
- // Only distributed file systems have encryption zones.
- if (!isDistributedFileSystem(p1) || !isDistributedFileSystem(p2)) return false;
- HdfsAdmin hdfsAdmin = new HdfsAdmin(fs.getUri(), CONF);
- EncryptionZone z1 = hdfsAdmin.getEncryptionZoneForPath(p1);
- EncryptionZone z2 = hdfsAdmin.getEncryptionZoneForPath(p2);
- if (z1 == null && z2 == null) return true;
- if (z1 == null || z2 == null) return false;
- return z1.equals(z2);
- }
-
- /**
- * Relocates all visible (non-hidden) files from a source directory to a destination
- * directory. Files are moved (renamed) to the new location unless the source and
- * destination directories are in different encryption zones, in which case the files
- * are copied so that they are decrypted and/or encrypted. Naming conflicts are
- * resolved by appending a UUID to the base file name. Any sub-directories within the
- * source directory are skipped. Returns the number of files relocated as part of this
- * operation.
- */
- public static int relocateAllVisibleFiles(Path sourceDir, Path destDir)
- throws IOException {
- FileSystem destFs = destDir.getFileSystem(CONF);
- FileSystem sourceFs = sourceDir.getFileSystem(CONF);
- Preconditions.checkState(destFs.isDirectory(destDir));
- Preconditions.checkState(sourceFs.isDirectory(sourceDir));
-
- // Use the same UUID to resolve all file name conflicts. This helps mitigate problems
- // that might happen if there is a conflict moving a set of files that have
- // dependent file names. For example, foo.lzo and foo.lzo_index.
- UUID uuid = UUID.randomUUID();
-
- // Enumerate all the files in the source
- int numFilesMoved = 0;
- for (FileStatus fStatus: sourceFs.listStatus(sourceDir)) {
- if (fStatus.isDirectory()) {
- LOG.debug("Skipping copy of directory: " + fStatus.getPath());
- continue;
- } else if (isHiddenFile(fStatus.getPath().getName())) {
- continue;
- }
-
- Path destFile = new Path(destDir, fStatus.getPath().getName());
- if (destFs.exists(destFile)) {
- destFile = new Path(destDir,
- appendToBaseFileName(destFile.getName(), uuid.toString()));
- }
- FileSystemUtil.relocateFile(fStatus.getPath(), destFile, false);
- ++numFilesMoved;
- }
- return numFilesMoved;
- }
-
- /**
- * Relocates the given file to a new location (either another directory or a
- * file in the same or different filesystem). The file is generally moved (renamed) to
- * the new location. However, the file is copied if the source and destination are in
- * different encryption zones so that the file can be decrypted and/or encrypted, or if
- * the source and destination are in different filesystems. If renameIfAlreadyExists is
- * true, no error will be thrown if a file with the same name already exists in the
- * destination location. Instead, a UUID will be appended to the base file name,
- * preserving the existing file extension. If renameIfAlreadyExists is false, an
- * IOException will be thrown if there is a file name conflict.
- */
- public static void relocateFile(Path sourceFile, Path dest,
- boolean renameIfAlreadyExists) throws IOException {
- FileSystem destFs = dest.getFileSystem(CONF);
- FileSystem sourceFs = sourceFile.getFileSystem(CONF);
-
- Path destFile =
- destFs.isDirectory(dest) ? new Path(dest, sourceFile.getName()) : dest;
- // If a file with the same name does not already exist in the destination location
- // then use the same file name. Otherwise, generate a unique file name.
- if (renameIfAlreadyExists && destFs.exists(destFile)) {
- Path destDir = destFs.isDirectory(dest) ? dest : dest.getParent();
- destFile = new Path(destDir,
- appendToBaseFileName(destFile.getName(), UUID.randomUUID().toString()));
- }
- boolean sameFileSystem = isPathOnFileSystem(sourceFile, destFs);
- boolean destIsDfs = isDistributedFileSystem(destFs);
-
- // If the source and the destination are on different file systems, or in different
- // encryption zones, files can't be moved from one location to the other and must be
- // copied instead.
- boolean sameEncryptionZone =
- arePathsInSameHdfsEncryptionZone(destFs, sourceFile, destFile);
- // We can do a rename if the src and dst are in the same encryption zone in the same
- // distributed filesystem.
- boolean doRename = destIsDfs && sameFileSystem && sameEncryptionZone;
- // Alternatively, we can do a rename if the src and dst are on the same
- // non-distributed filesystem.
- if (!doRename) doRename = !destIsDfs && sameFileSystem;
- if (doRename) {
- LOG.debug(String.format(
- "Moving '%s' to '%s'", sourceFile.toString(), destFile.toString()));
- // Move (rename) the file.
- destFs.rename(sourceFile, destFile);
- return;
- }
- if (destIsDfs && sameFileSystem) {
- Preconditions.checkState(!doRename);
- // We must copy rather than move if the source and dest are in different
- // encryption zones. A move would return an error from the NN because a move is a
- // metadata-only operation and the files would not be encrypted/decrypted properly
- // on the DNs.
- LOG.info(String.format(
- "Copying source '%s' to '%s' because HDFS encryption zones are different.",
- sourceFile, destFile));
- } else {
- Preconditions.checkState(!sameFileSystem);
- LOG.info(String.format("Copying '%s' to '%s' between filesystems.",
- sourceFile, destFile));
- }
- FileUtil.copy(sourceFs, sourceFile, destFs, destFile, true, true, CONF);
- }
-
- /**
- * Reads the file at path and returns the contents.
- */
- public static String readFile(Path file) throws IOException {
- FileSystem fs = file.getFileSystem(CONF);
- InputStream fileStream = fs.open(file);
- try {
- return IOUtils.toString(fileStream);
- } finally {
- IOUtils.closeQuietly(fileStream);
- }
- }
-
- /**
- * Builds a new file name based on a base file name. This is done by inserting
- * the given appendStr into the base file name, preserving the file extension (if
- * one exists).
- * For example, this could be passed a UUID string to uniquify files:
- * file1.snap -> file1_<uuid>.snap
- * file1 -> file1_<uuid>
- */
- private static String appendToBaseFileName(String baseFileName, String appendStr) {
- StringBuilder sb = new StringBuilder(baseFileName);
- // Insert the string to append, preserving the file extension.
- int extensionIdx = baseFileName.lastIndexOf('.');
- if (extensionIdx != -1) {
- sb.replace(extensionIdx, extensionIdx + 1, "_" + appendStr + ".");
- } else {
- sb.append("_" + appendStr);
- }
- return sb.toString();
- }
-
- /**
- * Returns true if the given Path contains any visible sub directories, otherwise false.
- */
- public static boolean containsVisibleSubdirectory(Path directory)
- throws FileNotFoundException, IOException {
- FileSystem fs = directory.getFileSystem(CONF);
- // Enumerate all the files in the source
- for (FileStatus fStatus: fs.listStatus(directory)) {
- String pathName = fStatus.getPath().getName();
- if (fStatus.isDirectory() && !isHiddenFile(pathName)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Makes a temporary unique directory within the given directory.
- */
- public static Path makeTmpSubdirectory(Path directory) throws IOException {
- FileSystem fs = directory.getFileSystem(CONF);
- Path tmpDir = new Path(directory, ".tmp_" + UUID.randomUUID().toString());
- fs.mkdirs(tmpDir);
- return tmpDir;
- }
-
- public static boolean isHiddenFile(String fileName) {
- // Hidden files start with '.' or '_'. The '.copying' suffix is used by some
- // filesystem utilities (e.g. hdfs put) as a temporary destination when copying
- // files. The '.tmp' suffix is Flume's default for temporary files.
- String lcFileName = fileName.toLowerCase();
- return lcFileName.startsWith(".") || lcFileName.startsWith("_") ||
- lcFileName.endsWith(".copying") || lcFileName.endsWith(".tmp");
- }
-
- /**
- * Returns true if the filesystem might override getFileBlockLocations().
- */
- public static boolean hasGetFileBlockLocations(FileSystem fs) {
- // Common case.
- if (isDistributedFileSystem(fs)) return true;
- // Blacklist FileSystems that are known to not implement getFileBlockLocations().
- return !(fs instanceof S3AFileSystem || fs instanceof NativeS3FileSystem ||
- fs instanceof S3FileSystem || fs instanceof LocalFileSystem);
- }
-
- /**
- * Returns true iff the filesystem is a S3AFileSystem.
- */
- public static boolean isS3AFileSystem(FileSystem fs) {
- return fs instanceof S3AFileSystem;
- }
-
- /**
- * Returns true iff the path is on a S3AFileSystem.
- */
- public static boolean isS3AFileSystem(Path path) throws IOException {
- return isS3AFileSystem(path.getFileSystem(CONF));
- }
-
- /**
- * Returns true iff the filesystem is an instance of LocalFileSystem.
- */
- public static boolean isLocalFileSystem(FileSystem fs) {
- return fs instanceof LocalFileSystem;
- }
-
- /**
- * Return true iff path is on a local filesystem.
- */
- public static boolean isLocalFileSystem(Path path) throws IOException {
- return isLocalFileSystem(path.getFileSystem(CONF));
- }
-
- /**
- * Returns true iff the filesystem is a DistributedFileSystem.
- */
- public static boolean isDistributedFileSystem(FileSystem fs) {
- return fs instanceof DistributedFileSystem;
- }
-
- /**
- * Return true iff path is on a DFS filesystem.
- */
- public static boolean isDistributedFileSystem(Path path) throws IOException {
- return isDistributedFileSystem(path.getFileSystem(CONF));
- }
-
- public static FileSystem getDefaultFileSystem() throws IOException {
- Path path = new Path(FileSystem.getDefaultUri(CONF));
- FileSystem fs = path.getFileSystem(CONF);
- return fs;
- }
-
- public static DistributedFileSystem getDistributedFileSystem() throws IOException {
- FileSystem fs = getDefaultFileSystem();
- Preconditions.checkState(fs instanceof DistributedFileSystem);
- return (DistributedFileSystem) fs;
- }
-
- /**
- * Fully-qualifies the given path based on the FileSystem configuration.
- */
- public static Path createFullyQualifiedPath(Path location) {
- URI defaultUri = FileSystem.getDefaultUri(CONF);
- URI locationUri = location.toUri();
- // Use the default URI only if location has no scheme or it has the same scheme as
- // the default URI. Otherwise, Path.makeQualified() will incorrectly use the
- // authority from the default URI even though the schemes don't match. See HDFS-7031.
- if (locationUri.getScheme() == null ||
- locationUri.getScheme().equalsIgnoreCase(defaultUri.getScheme())) {
- return location.makeQualified(defaultUri, location);
- }
- // Already qualified (has scheme).
- return location;
- }
-
- /**
- * Return true iff the path is on the given filesystem.
- */
- public static boolean isPathOnFileSystem(Path path, FileSystem fs) {
- try {
- // Call makeQualified() for the side-effect of FileSystem.checkPath() which will
- // throw an exception if path is not on fs.
- fs.makeQualified(path);
- return true;
- } catch (IllegalArgumentException e) {
- // Path is not on fs.
- return false;
- }
- }
-
- /**
- * Copies the source file to a destination path on the local filesystem.
- * Throws IOException on failure.
- */
- public static void copyToLocal(Path source, Path dest) throws IOException {
- FileSystem fs = source.getFileSystem(CONF);
- fs.copyToLocalFile(source, dest);
- }
-
- /**
- * Returns true if the given path is a location which supports caching (e.g. HDFS).
- */
- public static boolean isPathCacheable(Path path) {
- try {
- return isDistributedFileSystem(path);
- } catch (IOException e) {
- return false;
- }
- }
-
- /**
- * Returns the configuration.
- */
- public static Configuration getConfiguration() {
- return CONF;
- }
-
- /**
- * Returns true iff the given location is on a filesystem that Impala can write to.
- */
- public static boolean isImpalaWritableFilesystem(String location)
- throws IOException {
- Path path = new Path(location);
- return (FileSystemUtil.isDistributedFileSystem(path) ||
- FileSystemUtil.isLocalFileSystem(path) || FileSystemUtil.isS3AFileSystem(path));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/Id.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/Id.java b/fe/src/main/java/com/cloudera/impala/common/Id.java
deleted file mode 100644
index 88f626f..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/Id.java
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-
-/**
- * Integer ids that cannot accidentally be compared with ints.
- */
-public class Id<IdType extends Id<IdType>> implements Comparable<Id<IdType>> {
- static protected int INVALID_ID = -1;
- protected final int id_;
-
- public Id(int id) {
- this.id_ = id;
- }
-
- public boolean isValid() { return id_ != INVALID_ID; }
- public int asInt() { return id_; }
-
- @Override
- public int hashCode() {
- return Integer.valueOf(id_).hashCode();
- }
-
- @Override
- public String toString() {
- return Integer.toString(id_);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) return false;
- // only ids of the same subclass are comparable
- if (obj.getClass() != this.getClass()) return false;
- return ((Id)obj).id_ == id_;
- }
-
- @Override
- public int compareTo(Id<IdType> cmp) {
- return id_ - cmp.id_;
- }
-
- public ArrayList<IdType> asList() {
- ArrayList<IdType> list = new ArrayList<IdType>();
- list.add((IdType) this);
- return list;
- }
-
- public static <C extends Id> String printIds(List<C> ids) {
- ArrayList<String> l = Lists.newArrayList();
- for (C id: ids) {
- l.add(id.toString());
- }
- return "(" + Joiner.on(" ").join(l) + ")";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/IdGenerator.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/IdGenerator.java b/fe/src/main/java/com/cloudera/impala/common/IdGenerator.java
deleted file mode 100644
index cdece7d..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/IdGenerator.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-/**
- * Generator of consecutively numbered integers to be used as ids by subclasses of Id.
- * Subclasses of Id should be able to create a generator for their Id type.
- */
-public abstract class IdGenerator<IdType extends Id<IdType>> {
- protected int nextId_ = 0;
- public abstract IdType getNextId();
- public abstract IdType getMaxId();
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/ImpalaException.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/ImpalaException.java b/fe/src/main/java/com/cloudera/impala/common/ImpalaException.java
deleted file mode 100644
index 6a50bf8..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/ImpalaException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-
-/**
- * The parent class of all custom Impala exceptions.
- *
- */
-abstract public class ImpalaException extends java.lang.Exception {
- public ImpalaException(String msg, Throwable cause) {
- super(msg, cause);
- }
-
- protected ImpalaException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/common/ImpalaRuntimeException.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/ImpalaRuntimeException.java b/fe/src/main/java/com/cloudera/impala/common/ImpalaRuntimeException.java
deleted file mode 100644
index 27a8925..0000000
--- a/fe/src/main/java/com/cloudera/impala/common/ImpalaRuntimeException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.common;
-
-/**
- * Thrown for errors encountered during the execution of a SQL statement.
- *
- */
-public class ImpalaRuntimeException extends ImpalaException {
- public ImpalaRuntimeException(String msg, Throwable cause) {
- super(msg, cause);
- }
-
- public ImpalaRuntimeException(String msg) {
- super(msg);
- }
-}