You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2016/05/13 05:32:47 UTC
[7/7] tajo git commit: TAJO-2129: Apply new type implementation to
Schema and Catalog.
TAJO-2129: Apply new type implementation to Schema and Catalog.
Closes #1016
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/8d2bf407
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/8d2bf407
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/8d2bf407
Branch: refs/heads/master
Commit: 8d2bf407f5e462216eb84d193c300b46dc2817f9
Parents: 01d2fe2
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu May 12 22:29:43 2016 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu May 12 22:29:43 2016 -0700
----------------------------------------------------------------------
CHANGES | 2 +
.../apache/tajo/algebra/ColumnDefinition.java | 11 +-
.../org/apache/tajo/algebra/DataTypeExpr.java | 63 +++-
.../tajo/catalog/AbstractCatalogClient.java | 9 +-
.../main/java/org/apache/tajo/DataTypeUtil.java | 4 +-
.../apache/tajo/catalog/CatalogConstants.java | 4 -
.../org/apache/tajo/catalog/CatalogUtil.java | 260 +--------------
.../java/org/apache/tajo/catalog/Column.java | 52 +--
.../org/apache/tajo/catalog/DDLBuilder.java | 19 +-
.../org/apache/tajo/catalog/FieldConverter.java | 60 +---
.../apache/tajo/catalog/ListSchemaBuilder.java | 14 +-
.../org/apache/tajo/catalog/NestedPathUtil.java | 2 +-
.../org/apache/tajo/catalog/SchemaBuilder.java | 54 ++-
.../org/apache/tajo/catalog/SchemaLegacy.java | 90 ++---
.../org/apache/tajo/catalog/SchemaUtil.java | 2 +-
.../apache/tajo/catalog/SetSchemaBuilder.java | 32 +-
.../org/apache/tajo/catalog/TypeConverter.java | 131 ++++----
.../tajo/catalog/json/CatalogGsonHelper.java | 1 +
.../tajo/catalog/statistics/TableStats.java | 9 +-
.../src/main/proto/CatalogProtos.proto | 2 +-
.../apache/tajo/catalog/TestCatalogUtil.java | 3 +-
.../tajo/catalog/store/HiveCatalogStore.java | 26 +-
.../tajo/catalog/store/HiveCatalogUtil.java | 10 +-
.../catalog/store/TestHiveCatalogStore.java | 23 +-
.../org/apache/tajo/catalog/CatalogServer.java | 9 +-
.../dictionary/AbstractTableDescriptor.java | 12 +-
.../tajo/catalog/store/AbstractDBStore.java | 119 +++----
.../src/main/resources/schemas/derby/derby.xml | 5 +-
.../main/resources/schemas/mariadb/mariadb.xml | 5 +-
.../src/main/resources/schemas/mysql/mysql.xml | 5 +-
.../main/resources/schemas/oracle/oracle.xml | 5 +-
.../resources/schemas/postgresql/postgresql.xml | 5 +-
.../apache/tajo/catalog/CatalogTestingUtil.java | 23 +-
.../apache/tajo/catalog/MiniCatalogServer.java | 3 +-
.../org/apache/tajo/catalog/TestCatalog.java | 29 +-
.../TestCatalogAgainstCaseSensitivity.java | 37 ++-
.../tajo/catalog/TestCatalogExceptions.java | 47 +--
.../org/apache/tajo/cli/tools/TajoDump.java | 9 +-
.../cli/tsql/commands/DescTableCommand.java | 4 +-
.../apache/tajo/jdbc/TajoResultSetMetaData.java | 6 +-
.../apache/tajo/LocalTajoTestingUtility.java | 3 +-
.../java/org/apache/tajo/QueryTestCaseBase.java | 10 +-
.../org/apache/tajo/TajoTestingCluster.java | 7 +-
.../src/main/java/org/apache/tajo/Assert.java | 36 ++
.../exception/InvalidValueForCastException.java | 2 +-
.../org/apache/tajo/json/CommonGsonHelper.java | 1 +
.../java/org/apache/tajo/json/TypeAdapter.java | 42 +++
.../tajo/schema/ANSISQLIdentifierPolicy.java | 4 +-
.../main/java/org/apache/tajo/schema/Field.java | 106 ++++++
.../java/org/apache/tajo/schema/Identifier.java | 44 ++-
.../apache/tajo/schema/IdentifierPolicy.java | 8 +-
.../org/apache/tajo/schema/IdentifierUtil.java | 333 +++++++++++++++++++
.../tajo/schema/PgSQLIdentifierPolicy.java | 4 +-
.../apache/tajo/schema/QualifiedIdentifier.java | 76 ++++-
.../java/org/apache/tajo/schema/Schema.java | 142 +-------
.../org/apache/tajo/schema/package-info.java | 22 ++
.../src/main/java/org/apache/tajo/type/Any.java | 12 +-
.../main/java/org/apache/tajo/type/Array.java | 40 +--
.../main/java/org/apache/tajo/type/Blob.java | 7 +-
.../main/java/org/apache/tajo/type/Bool.java | 7 +-
.../main/java/org/apache/tajo/type/Char.java | 41 +--
.../main/java/org/apache/tajo/type/Date.java | 7 +-
.../main/java/org/apache/tajo/type/Float4.java | 7 +-
.../main/java/org/apache/tajo/type/Float8.java | 7 +-
.../main/java/org/apache/tajo/type/Inet4.java | 7 +-
.../main/java/org/apache/tajo/type/Int1.java | 7 +-
.../main/java/org/apache/tajo/type/Int2.java | 7 +-
.../main/java/org/apache/tajo/type/Int4.java | 7 +-
.../main/java/org/apache/tajo/type/Int8.java | 7 +-
.../java/org/apache/tajo/type/Interval.java | 7 +-
.../src/main/java/org/apache/tajo/type/Map.java | 41 +--
.../main/java/org/apache/tajo/type/Null.java | 7 +-
.../main/java/org/apache/tajo/type/Numeric.java | 40 +--
.../java/org/apache/tajo/type/Protobuf.java | 12 +-
.../main/java/org/apache/tajo/type/Record.java | 79 +++++
.../main/java/org/apache/tajo/type/Struct.java | 72 ----
.../main/java/org/apache/tajo/type/Text.java | 7 +-
.../main/java/org/apache/tajo/type/Time.java | 7 +-
.../java/org/apache/tajo/type/Timestamp.java | 7 +-
.../main/java/org/apache/tajo/type/Type.java | 97 +++---
.../java/org/apache/tajo/type/TypeFactory.java | 184 ++++++++++
.../apache/tajo/type/TypeParamterizedType.java | 77 +++++
.../apache/tajo/type/TypeProtobufEncoder.java | 185 +++++++++++
.../org/apache/tajo/type/TypeStringEncoder.java | 296 +++++++++++++++++
.../java/org/apache/tajo/type/TypeVisitor.java | 54 +++
.../apache/tajo/type/ValueParamterizedType.java | 79 +++++
.../main/java/org/apache/tajo/type/Varchar.java | 40 +--
.../java/org/apache/tajo/type/package-info.java | 29 ++
tajo-common/src/main/proto/DataTypes.proto | 21 ++
.../java/org/apache/tajo/schema/TestSchema.java | 26 +-
.../tajo/schema/TestTajoIdentifierPolicy.java | 4 +-
.../java/org/apache/tajo/type/TestType.java | 105 +++---
.../org/apache/tajo/type/TestTypeEncoder.java | 124 +++++++
.../tajo/type/TestTypeProtobufEncoder.java | 30 ++
.../apache/tajo/type/TestTypeStringEncoder.java | 29 ++
.../apache/tajo/cli/tools/TestDDLBuilder.java | 5 +-
.../org/apache/tajo/cli/tsql/TestTajoCli.java | 6 +-
.../tajo/cli/tsql/TestTajoCliNegatives.java | 2 +-
.../org/apache/tajo/client/TestTajoClient.java | 28 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 16 +-
.../apache/tajo/engine/eval/TestEvalTree.java | 8 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 3 +-
.../planner/TestEvalNodeToExprConverter.java | 3 +-
.../engine/planner/TestJoinOrderAlgorithm.java | 9 +-
.../engine/planner/TestLogicalOptimizer.java | 7 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 48 +--
.../tajo/engine/planner/TestPlannerUtil.java | 7 +-
.../physical/TestFullOuterHashJoinExec.java | 9 +-
.../physical/TestFullOuterMergeJoinExec.java | 11 +-
.../physical/TestLeftOuterHashJoinExec.java | 9 +-
.../planner/physical/TestPhysicalPlanner.java | 7 +-
.../physical/TestProgressExternalSortExec.java | 3 +-
.../physical/TestRightOuterHashJoinExec.java | 7 +-
.../physical/TestRightOuterMergeJoinExec.java | 11 +-
.../engine/planner/physical/TestSortExec.java | 3 +-
.../tajo/engine/query/TestAlterTable.java | 21 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 16 +-
.../tajo/engine/query/TestCreateDatabase.java | 8 +-
.../tajo/engine/query/TestCreateTable.java | 27 +-
.../tajo/engine/query/TestInsertQuery.java | 16 +-
.../query/TestJoinOnPartitionedTables.java | 10 +-
.../tajo/engine/query/TestSimpleQuery.java | 6 +-
.../tajo/engine/query/TestTablePartitions.java | 47 +--
.../tajo/master/TestExecutionBlockCursor.java | 3 +-
.../tajo/querymaster/TestTaskStatusUpdate.java | 4 +-
.../ws/rs/resources/TestTablesResource.java | 8 +-
.../create_table_various_types.sql | 3 +-
.../TestCreateTable/testComplexType1.sql | 6 +
.../positive/create_table_with_array_type.sql | 6 +
.../TestCommonConditionReduce/test51.plan | 2 +-
.../testJoinWithMultipleJoinQual2.Hash.plan | 4 +-
...nWithMultipleJoinQual2.Hash_NoBroadcast.plan | 4 +-
.../testJoinWithMultipleJoinQual2.Sort.plan | 4 +-
...nWithMultipleJoinQual2.Sort_NoBroadcast.plan | 4 +-
...tOuterJoinPredicationCaseByCase3.1.Hash.plan | 4 +-
...edicationCaseByCase3.1.Hash_NoBroadcast.plan | 4 +-
...tOuterJoinPredicationCaseByCase3.1.Sort.plan | 4 +-
...edicationCaseByCase3.1.Sort_NoBroadcast.plan | 4 +-
.../testOuterJoinAndCaseWhen1.Hash.plan | 4 +-
...tOuterJoinAndCaseWhen1.Hash_NoBroadcast.plan | 4 +-
.../testOuterJoinAndCaseWhen1.Sort.plan | 4 +-
...tOuterJoinAndCaseWhen1.Sort_NoBroadcast.plan | 4 +-
...tOuterJoinPredicationCaseByCase3.1.Hash.plan | 4 +-
...edicationCaseByCase3.1.Hash_NoBroadcast.plan | 4 +-
...tOuterJoinPredicationCaseByCase3.1.Sort.plan | 4 +-
...edicationCaseByCase3.1.Sort_NoBroadcast.plan | 4 +-
.../results/TestTPCH/testFirstJoinInQ7.plan | 16 +-
.../tajo/engine/codegen/EvalCodeGenContext.java | 4 +-
.../tajo/engine/codegen/EvalCodeGenerator.java | 8 +-
.../engine/codegen/TajoGeneratorAdapter.java | 28 +-
.../engine/codegen/VariablesPreBuilder.java | 4 +-
.../engine/planner/global/GlobalPlanner.java | 8 +-
.../tajo/master/TajoMasterClientService.java | 41 +--
.../tajo/master/exec/CreateTableExecutor.java | 7 +-
.../apache/tajo/master/exec/DDLExecutor.java | 35 +-
.../NonForwardQueryResultSystemScanner.java | 16 +-
.../apache/tajo/master/exec/QueryExecutor.java | 7 +-
.../master/exec/prehook/CreateTableHook.java | 5 +-
.../org/apache/tajo/parser/sql/SQLAnalyzer.java | 33 +-
.../java/org/apache/tajo/querymaster/Query.java | 13 +-
.../tajo/ws/rs/resources/TablesResource.java | 12 +-
.../apache/tajo/jdbc/TajoDatabaseMetaData.java | 19 +-
.../org/apache/tajo/jdbc/TestResultSet.java | 3 +-
.../tajo/jdbc/TestTajoDatabaseMetaData.java | 10 +-
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 10 +-
.../org/apache/tajo/plan/ExprAnnotator.java | 40 ++-
.../org/apache/tajo/plan/ExprNormalizer.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 109 +++---
.../org/apache/tajo/plan/NamedExprsManager.java | 5 +-
.../org/apache/tajo/plan/TypeDeterminant.java | 8 +-
.../org/apache/tajo/plan/expr/ConstEval.java | 4 +-
.../org/apache/tajo/plan/expr/EvalTreeUtil.java | 15 +-
.../org/apache/tajo/plan/expr/FieldEval.java | 60 ++--
.../apache/tajo/plan/expr/RowConstantEval.java | 4 +-
.../tajo/plan/logical/CreateDatabaseNode.java | 4 +-
.../org/apache/tajo/plan/logical/JoinSpec.java | 6 +-
.../org/apache/tajo/plan/logical/ScanNode.java | 13 +-
.../tajo/plan/nameresolver/NameResolver.java | 18 +-
.../plan/nameresolver/ResolverByLegacy.java | 4 +-
.../tajo/plan/rewrite/BaseSchemaBuildPhase.java | 22 +-
.../plan/rewrite/SelfDescSchemaBuildPhase.java | 7 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 11 +-
.../rewrite/rules/PartitionedTableRewriter.java | 3 +-
.../tajo/plan/serder/EvalNodeSerializer.java | 4 +-
.../apache/tajo/plan/serder/PlanGsonHelper.java | 1 +
.../tajo/plan/util/EvalNodeToExprConverter.java | 2 +-
.../tajo/plan/verifier/ExprsVerifier.java | 14 +-
.../plan/verifier/PreLogicalPlanVerifier.java | 14 +-
tajo-sql-parser/pom.xml | 5 +
.../org/apache/tajo/parser/sql/SQLLexer.g4 | 7 +-
.../org/apache/tajo/parser/sql/SQLParser.g4 | 19 +-
.../org/apache/tajo/storage/FileTablespace.java | 3 +-
.../storage/jdbc/JdbcMetadataProviderBase.java | 5 +-
.../storage/jdbc/SQLExpressionGenerator.java | 13 +-
.../tajo/storage/pgsql/PgSQLTablespace.java | 3 +-
195 files changed, 3135 insertions(+), 1888 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index eec145c..840ba56 100644
--- a/CHANGES
+++ b/CHANGES
@@ -14,6 +14,8 @@ Release 0.12.0 - unreleased
IMPROVEMENT
+ TAJO-2129: Apply new type implementation to Schema and Catalog. (hyunsik)
+
TAJO-2071: Supporting DATE type in Parquet format.
(Jongyoung Park via hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
index 2a829e1..aa26bfc 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
@@ -24,11 +24,6 @@ public class ColumnDefinition extends DataTypeExpr {
@Expose @SerializedName("ColumnDefName")
String columnName;
- public ColumnDefinition(String columnName, String dataType) {
- super(dataType);
- this.columnName = columnName;
- }
-
public ColumnDefinition(String columnName, DataTypeExpr dataType) {
super(dataType.getTypeName());
@@ -42,15 +37,17 @@ public class ColumnDefinition extends DataTypeExpr {
}
}
- // nested records
if (dataType.isRecordType()) {
this.recordType = dataType.recordType;
}
- // map type
if (dataType.isMapType()) {
this.mapType = dataType.mapType;
}
+
+ if (dataType.isArrayType()) {
+ this.arrayType = dataType.arrayType;
+ }
}
public String getColumnName() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
index d63532d..bd5caac 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
@@ -19,11 +19,15 @@
package org.apache.tajo.algebra;
import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
+import org.apache.tajo.Assert;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.util.TUtil;
+import static org.apache.tajo.Assert.assertNotNull;
+
public class DataTypeExpr extends Expr {
@Expose @SerializedName("DataTypeName")
String typeName;
@@ -31,6 +35,8 @@ public class DataTypeExpr extends Expr {
Integer lengthOrPrecision;
@Expose @SerializedName("Scale")
Integer scale;
+ @Expose @SerializedName("Array")
+ ArrayType arrayType; // not null if the type is ARRAY
@Expose @SerializedName("Record")
RecordType recordType; // not null if the type is RECORD
@Expose @SerializedName("Map")
@@ -38,18 +44,27 @@ public class DataTypeExpr extends Expr {
public DataTypeExpr(String typeName) {
super(OpType.DataType);
+ assertNotNull(typeName);
this.typeName = typeName;
}
+ public DataTypeExpr(ArrayType array) {
+ super(OpType.DataType);
+ assertNotNull(array);
+ this.typeName = Type.ARRAY.name();
+ this.arrayType = array;
+ }
+
public DataTypeExpr(RecordType record) {
super(OpType.DataType);
+ assertNotNull(record);
this.typeName = Type.RECORD.name();
this.recordType = record;
}
public DataTypeExpr(MapType map) {
super(OpType.DataType);
- // RECORD = 51 in DataTypes.proto
+ assertNotNull(map);
this.typeName = Type.MAP.name();
this.mapType = map;
}
@@ -59,15 +74,34 @@ public class DataTypeExpr extends Expr {
}
public boolean isPrimitiveType() {
- return !this.isRecordType() && !isMapType();
+ return !isArrayType()&& !isRecordType() && !isMapType();
+ }
+
+ public boolean isArrayType() {
+ return arrayType != null;
}
public boolean isRecordType() {
- return this.typeName.equals(Type.RECORD.name());
+ return recordType != null;
}
public boolean isMapType() {
- return this.typeName.equals(Type.MAP.name());
+ return mapType != null;
+ }
+
+ public DataTypeExpr getElementType() {
+ Preconditions.checkState(isArrayType());
+ return arrayType.type;
+ }
+
+ public DataTypeExpr getKeyType() {
+ Preconditions.checkState(isMapType());
+ return mapType.keyType;
+ }
+
+ public DataTypeExpr getValueType() {
+ Preconditions.checkState(isMapType());
+ return mapType.valueType;
}
public ColumnDefinition [] getNestedRecordTypes() {
@@ -125,6 +159,27 @@ public class DataTypeExpr extends Expr {
return dataType;
}
+ public static class ArrayType implements JsonSerializable, Cloneable {
+ @Expose
+ @SerializedName("type")
+ DataTypeExpr type;
+
+ public ArrayType(DataTypeExpr elementType) {
+ this.type = elementType;
+ }
+
+ @Override
+ public String toJson() {
+ return JsonHelper.toJson(this);
+ }
+
+ public Object clone() throws CloneNotSupportedException {
+ ArrayType newMap = (ArrayType) super.clone();
+ newMap.type = type;
+ return newMap;
+ }
+ }
+
public static class RecordType implements JsonSerializable, Cloneable {
@Expose @SerializedName("Schema")
ColumnDefinition [] schema; // not null if the type is RECORD
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index b0acbb1..2c4d41e 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@ -32,6 +32,7 @@ import org.apache.tajo.exception.*;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListResponse;
+import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.util.ProtoUtil;
import java.io.Closeable;
@@ -281,7 +282,7 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable
@Override
public TableDesc getTableDesc(String qualifiedName) throws UndefinedTableException {
- String[] splitted = CatalogUtil.splitFQTableName(qualifiedName);
+ String[] splitted = IdentifierUtil.splitFQTableName(qualifiedName);
return getTableDesc(splitted[0], splitted[1]);
}
@@ -596,7 +597,7 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable
public void dropTable(String tableName)
throws UndefinedDatabaseException, UndefinedTableException, InsufficientPrivilegeException {
- String[] splitted = CatalogUtil.splitFQTableName(tableName);
+ String[] splitted = IdentifierUtil.splitFQTableName(tableName);
final String databaseName = splitted[0];
final String simpleName = splitted[1];
@@ -617,7 +618,7 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable
@Override
public final boolean existsTable(final String databaseName, final String tableName) {
- if (CatalogUtil.isFQTableName(tableName)) {
+ if (IdentifierUtil.isFQTableName(tableName)) {
throw new IllegalArgumentException(
"tableName cannot be composed of multiple parts, but it is \"" + tableName + "\"");
}
@@ -640,7 +641,7 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable
@Override
public final boolean existsTable(final String tableName) {
- String[] splitted = CatalogUtil.splitFQTableName(tableName);
+ String[] splitted = IdentifierUtil.splitFQTableName(tableName);
return existsTable(splitted[0], splitted[1]);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java
index 9cbdbe0..69206aa 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java
@@ -87,8 +87,8 @@ public class DataTypeUtil {
*/
public static org.apache.tajo.type.Type determineType(org.apache.tajo.type.Type left,
org.apache.tajo.type.Type right) {
- TajoDataTypes.Type rhsBaseType = right.baseType();
- switch (left.baseType()) {
+ TajoDataTypes.Type rhsBaseType = right.kind();
+ switch (left.kind()) {
case INT1:
case INT2:
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
index f2acf98..6acbc34 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
@@ -19,10 +19,6 @@
package org.apache.tajo.catalog;
public class CatalogConstants {
- public final static String IDENTIFIER_DELIMITER_REGEXP = "\\.";
- public final static String IDENTIFIER_DELIMITER = ".";
- public final static String IDENTIFIER_QUOTE_STRING = "\"";
- public final static int MAX_IDENTIFIER_LENGTH = 128;
// Linux and BSD's max username length is 32. For compatibility with other systems, we should follow it.
public final static int MAX_USERNAME_LENGTH = 32;
public final static int MAX_STATEMENT_LENGTH = 128 * 1024;
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 2b20907..22dd5a4 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -18,12 +18,10 @@
package org.apache.tajo.catalog;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.DataTypeUtil;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.partition.PartitionDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
@@ -35,6 +33,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.exception.UndefinedOperatorException;
+import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
@@ -51,233 +50,6 @@ import static org.apache.tajo.common.TajoDataTypes.Type;
public class CatalogUtil {
- /**
- * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name.
- *
- * Identifier can be composed of multiple parts as follows:
- * <pre>
- * database_name.table_name.column_name
- * </pre>
- *
- * Each regular identifier part can be composed alphabet ([a-z][A-Z]), number([0-9]), and underscore([_]).
- * Also, the first letter must be an alphabet character.
- *
- * <code>normalizeIdentifier</code> normalizes each part of an identifier.
- *
- * In detail, for each part, it performs as follows:
- * <ul>
- * <li>changing a part without double quotation to be lower case letters</li>
- * <li>eliminating double quotation marks from identifier</li>
- * </ul>
- *
- * @param identifier The identifier to be normalized
- * @return The normalized identifier
- */
- public static String normalizeIdentifier(String identifier) {
- if (identifier == null || identifier.equals("")) {
- return identifier;
- }
- String [] splitted = identifier.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP);
-
- StringBuilder sb = new StringBuilder();
- boolean first = true;
- for (String part : splitted) {
- if (first) {
- first = false;
- } else {
- sb.append(CatalogConstants.IDENTIFIER_DELIMITER);
- }
- sb.append(normalizeIdentifierPart(part));
- }
- return sb.toString();
- }
-
- public static String normalizeIdentifierPart(String part) {
- return isDelimited(part) ? stripQuote(part) : part.toLowerCase();
- }
-
- /**
- * Denormalize an identifier. Denormalize means a translation from a stored identifier
- * to be a printable identifier name.
- *
- * In detail, for each part, it performs as follows:
- * <ul>
- * <li>changing a part including upper case character or non-ascii character to be lower case letters</li>
- * <li>eliminating double quotation marks from identifier</li>
- * </ul>
- *
- * @param identifier The identifier to be normalized
- * @return The denormalized identifier
- */
- public static String denormalizeIdentifier(String identifier) {
- String [] splitted = identifier.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP);
-
- StringBuilder sb = new StringBuilder();
- boolean first = true;
- for (String part : splitted) {
- if (first) {
- first = false;
- } else {
- sb.append(CatalogConstants.IDENTIFIER_DELIMITER);
- }
- sb.append(denormalizePart(part));
- }
- return sb.toString();
- }
-
- public static String denormalizePart(String identifier) {
- if (isShouldBeQuoted(identifier)) {
- return StringUtils.doubleQuote(identifier);
- } else {
- return identifier;
- }
- }
-
- public static boolean isShouldBeQuoted(String columnName) {
- for (char character : columnName.toCharArray()) {
- if (Character.isUpperCase(character)) {
- return true;
- }
-
- if (!StringUtils.isPartOfAnsiSQLIdentifier(character)) {
- return true;
- }
-
- if (RESERVED_KEYWORDS_SET.contains(columnName.toUpperCase())) {
- return true;
- }
- }
-
- return false;
- }
-
- public static String stripQuote(String str) {
- return str.substring(1, str.length() - 1);
- }
-
- public static boolean isDelimited(String identifier) {
- boolean openQuote = identifier.charAt(0) == '"';
- boolean closeQuote = identifier.charAt(identifier.length() - 1) == '"';
-
- // if at least one quote mark exists, the identifier must be grater than equal to 2 characters,
- if (openQuote ^ closeQuote && identifier.length() < 2) {
- throw new IllegalArgumentException("Invalid Identifier: " + identifier);
- }
-
- // does not allow the empty identifier (''),
- if (openQuote && closeQuote && identifier.length() == 2) {
- throw new IllegalArgumentException("zero-length delimited identifier: " + identifier);
- }
-
- // Ensure the quote open and close
- return openQuote && closeQuote;
- }
-
- /**
- * True if a given name is a simple identifier, meaning is not a dot-chained name.
- *
- * @param columnOrTableName Column or Table name to be checked
- * @return True if a given name is a simple identifier. Otherwise, it will return False.
- */
- public static boolean isSimpleIdentifier(String columnOrTableName) {
- return columnOrTableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 1;
- }
-
- public static boolean isFQColumnName(String tableName) {
- return tableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 3;
- }
-
- public static boolean isFQTableName(String tableName) {
- int lastDelimiterIdx = tableName.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER);
- return lastDelimiterIdx > -1;
- }
-
- public static String [] splitFQTableName(String qualifiedName) {
- String [] splitted = CatalogUtil.splitTableName(qualifiedName);
- if (splitted.length == 1) {
- throw new IllegalArgumentException("Table name is expected to be qualified, but was \""
- + qualifiedName + "\".");
- }
- return splitted;
- }
-
- public static String [] splitTableName(String tableName) {
- int lastDelimiterIdx = tableName.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER);
- if (lastDelimiterIdx > -1) {
- return new String [] {
- tableName.substring(0, lastDelimiterIdx),
- tableName.substring(lastDelimiterIdx + 1, tableName.length())
- };
- } else {
- return new String [] {tableName};
- }
- }
-
- public static String buildFQName(String... identifiers) {
- boolean first = true;
- StringBuilder sb = new StringBuilder();
- for(String id : identifiers) {
- if (first) {
- first = false;
- } else {
- sb.append(CatalogConstants.IDENTIFIER_DELIMITER);
- }
-
- sb.append(id);
- }
-
- return sb.toString();
- }
-
- public static Pair<String, String> separateQualifierAndName(String name) {
- Preconditions.checkArgument(isFQTableName(name), "Must be a qualified name.");
- return new Pair<>(extractQualifier(name), extractSimpleName(name));
- }
-
- /**
- * Extract a qualification name from an identifier.
- *
- * For example, consider a table identifier like 'database1.table1'.
- * In this case, this method extracts 'database1'.
- *
- * @param name The identifier to be extracted
- * @return The extracted qualifier
- */
- public static String extractQualifier(String name) {
- int lastDelimiterIdx = name.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER);
- if (lastDelimiterIdx > -1) {
- return name.substring(0, lastDelimiterIdx);
- } else {
- return TajoConstants.EMPTY_STRING;
- }
- }
-
- /**
- * Extract a simple name from an identifier.
- *
- * For example, consider a table identifier like 'database1.table1'.
- * In this case, this method extracts 'table1'.
- *
- * @param name The identifier to be extracted
- * @return The extracted simple name
- */
- public static String extractSimpleName(String name) {
- int lastDelimiterIdx = name.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER);
- if (lastDelimiterIdx > -1) {
- // plus one means skipping a delimiter.
- return name.substring(lastDelimiterIdx + 1, name.length());
- } else {
- return name;
- }
- }
-
- public static String getCanonicalTableName(String databaseName, String tableName) {
- StringBuilder sb = new StringBuilder(databaseName);
- sb.append(CatalogConstants.IDENTIFIER_DELIMITER);
- sb.append(tableName);
- return sb.toString();
- }
-
public static String getBackwardCompitableDataFormat(String dataFormat) {
return getDataFormatAsString(asDataFormat(dataFormat));
@@ -400,7 +172,7 @@ public class CatalogUtil {
}
public static String columnToDDLString(Column column) {
- StringBuilder sb = new StringBuilder(denormalizeIdentifier(column.getSimpleName()));
+ StringBuilder sb = new StringBuilder(IdentifierUtil.denormalizeIdentifier(column.getSimpleName()));
sb.append(" ").append(column.getDataType().getType());
if (column.getDataType().hasLength()) {
sb.append(" (").append(column.getDataType().getLength()).append(")");
@@ -738,32 +510,8 @@ public class CatalogUtil {
}
}
- public static final Set<String> RESERVED_KEYWORDS_SET = new HashSet<>();
-
- static final String [] RESERVED_KEYWORDS = {
- "AS", "ALL", "AND", "ANY", "ASYMMETRIC", "ASC",
- "BOTH",
- "CASE", "CAST", "CREATE", "CROSS", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP",
- "DESC", "DISTINCT",
- "END", "ELSE", "EXCEPT",
- "FALSE", "FULL", "FROM",
- "GROUP",
- "HAVING",
- "ILIKE", "IN", "INNER", "INTERSECT", "INTO", "IS",
- "JOIN",
- "LEADING", "LEFT", "LIKE", "LIMIT",
- "NATURAL", "NOT", "NULL",
- "ON", "OUTER", "OR", "ORDER",
- "RIGHT",
- "SELECT", "SOME", "SYMMETRIC",
- "TABLE", "THEN", "TRAILING", "TRUE",
- "OVER",
- "UNION", "UNIQUE", "USING",
- "WHEN", "WHERE", "WINDOW", "WITH"
- };
-
static {
- Collections.addAll(RESERVED_KEYWORDS_SET, RESERVED_KEYWORDS);
+ Collections.addAll(IdentifierUtil.RESERVED_KEYWORDS_SET, IdentifierUtil.RESERVED_KEYWORDS);
}
public static AlterTableDesc renameColumn(String tableName, String oldColumName, String newColumName,
@@ -1045,7 +793,7 @@ public class CatalogUtil {
public static String getUnifiedSimpleColumnName(Schema originalSchema, String[] columnNames) {
String[] simpleNames = new String[columnNames.length];
for (int i = 0; i < simpleNames.length; i++) {
- String[] identifiers = columnNames[i].split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP);
+ String[] identifiers = columnNames[i].split(IdentifierUtil.IDENTIFIER_DELIMITER_REGEXP);
simpleNames[i] = identifiers[identifiers.length-1];
}
Arrays.sort(simpleNames, new ColumnPosComparator(originalSchema));
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
index 641b674..ddf0c83 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
@@ -26,24 +26,27 @@ import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.json.GsonObject;
+import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.type.Type;
+import org.apache.tajo.type.TypeProtobufEncoder;
/**
* Describes a column. It is an immutable object.
*/
+@Deprecated
public class Column implements ProtoObject<ColumnProto>, GsonObject {
@Expose protected String name;
- @Expose protected TypeDesc typeDesc;
+ @Expose protected Type type;
/**
* Column Constructor
*
* @param name field name
- * @param typeDesc Type description
+ * @param type Type description
*/
- public Column(String name, TypeDesc typeDesc) {
+ public Column(String name, TypeDesc type) {
this.name = name;
- this.typeDesc = typeDesc;
+ this.type = TypeConverter.convert(type);
}
/**
@@ -70,22 +73,13 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
* @param type Type
*/
public Column(String name, Type type) {
- this(name, TypeConverter.convert(type));
- }
-
- /**
- *
- * @param name Column name
- * @param type Data Type
- * @param typeLength The length of type
- */
- public Column(String name, TajoDataTypes.Type type, int typeLength) {
- this(name, CatalogUtil.newDataTypeWithLen(type, typeLength));
+ this.name = name;
+ this.type = type;
}
public Column(ColumnProto proto) {
name = proto.getName();
- typeDesc = new TypeDesc(proto.getDataType());
+ type = TypeProtobufEncoder.decode(proto.getType());
}
/**
@@ -93,7 +87,7 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
* @return True if a column includes a table name. Otherwise, it returns False.
*/
public boolean hasQualifier() {
- return name.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length > 1;
+ return name.split(IdentifierUtil.IDENTIFIER_DELIMITER_REGEXP).length > 1;
}
/**
@@ -109,7 +103,7 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
* @return The qualifier
*/
public String getQualifier() {
- return CatalogUtil.extractQualifier(name);
+ return IdentifierUtil.extractQualifier(name);
}
/**
@@ -117,7 +111,7 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
* @return The simple name without qualifications
*/
public String getSimpleName() {
- return CatalogUtil.extractSimpleName(name);
+ return IdentifierUtil.extractSimpleName(name);
}
/**
@@ -126,7 +120,14 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
* @return TypeDesc
*/
public TypeDesc getTypeDesc() {
- return this.typeDesc;
+ return TypeConverter.convert(this.type);
+ }
+
+ /**
+ * @return Type which includes domain type and scale.
+ */
+ public Type getType() {
+ return this.type;
}
/**
@@ -134,21 +135,20 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
* @return DataType which includes domain type and scale.
*/
public DataType getDataType() {
- return this.typeDesc.dataType;
+ return TypeConverter.convert(this.type).getDataType();
}
@Override
public boolean equals(Object o) {
if (o instanceof Column) {
Column another = (Column)o;
- return name.equals(another.name) && typeDesc.equals(another.typeDesc);
+ return name.equals(another.name) && type.equals(another.type);
}
return false;
}
public int hashCode() {
- return Objects.hashCode(name, typeDesc);
-
+ return Objects.hashCode(name, type);
}
/**
@@ -160,13 +160,13 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
ColumnProto.Builder builder = ColumnProto.newBuilder();
builder
.setName(this.name)
- .setDataType(this.typeDesc.getDataType());
+ .setType(this.type.getProto());
return builder.build();
}
public String toString() {
StringBuilder sb = new StringBuilder(getQualifiedName());
- sb.append(" (").append(typeDesc.toString()).append(")");
+ sb.append(" (").append(type.toString()).append(")");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index e2250eb..387c669 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -21,6 +21,7 @@ package org.apache.tajo.catalog;
import com.google.common.base.Function;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
+import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.StringUtils;
@@ -36,11 +37,11 @@ public class DDLBuilder {
StringBuilder sb = new StringBuilder();
sb.append("--\n")
- .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;")
+ .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;")
.append(" Storage: ").append(desc.getMeta().getDataFormat());
sb.append("\n-- Path: ").append(desc.getUri());
sb.append("\n--\n");
- sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName()));
+ sb.append("CREATE EXTERNAL TABLE ").append(IdentifierUtil.denormalizeIdentifier(desc.getName()));
buildSchema(sb, desc.getSchema());
buildUsingClause(sb, desc.getMeta());
buildWithClause(sb, desc.getMeta());
@@ -59,10 +60,10 @@ public class DDLBuilder {
StringBuilder sb = new StringBuilder();
sb.append("--\n")
- .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;")
+ .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;")
.append(" Storage: ").append(desc.getMeta().getDataFormat());
sb.append("\n--\n");
- sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName()));
+ sb.append("CREATE TABLE ").append(IdentifierUtil.denormalizeIdentifier(desc.getName()));
buildSchema(sb, desc.getSchema());
buildUsingClause(sb, desc.getMeta());
buildWithClause(sb, desc.getMeta());
@@ -79,11 +80,11 @@ public class DDLBuilder {
StringBuilder sb = new StringBuilder();
sb.append("--\n")
- .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: INDEX;")
+ .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: INDEX;")
.append(" Index Method: ").append(desc.getIndexMethod());
sb.append("\n--\n");
- sb.append("CREATE INDEX ").append(CatalogUtil.denormalizeIdentifier(desc.getName()));
- sb.append(" on ").append(CatalogUtil.denormalizeIdentifier(desc.getTableName())).append(" ( ");
+ sb.append("CREATE INDEX ").append(IdentifierUtil.denormalizeIdentifier(desc.getName()));
+ sb.append(" on ").append(IdentifierUtil.denormalizeIdentifier(desc.getTableName())).append(" ( ");
for (SortSpec sortSpec : desc.getKeySortSpecs()) {
sb.append(sortSpec.getSortKey().getQualifiedName()).append(" ");
@@ -108,7 +109,7 @@ public class DDLBuilder {
sb.append(", ");
}
- sb.append(CatalogUtil.denormalizeIdentifier(column.getSimpleName())).append(" ");
+ sb.append(IdentifierUtil.denormalizeIdentifier(column.getSimpleName())).append(" ");
TypeDesc typeDesc = column.getTypeDesc();
sb.append(typeDesc);
}
@@ -177,7 +178,7 @@ public class DDLBuilder {
*/
public static String buildDDLForAddPartition(TableDesc table, PartitionDescProto partition) {
StringBuilder sb = new StringBuilder();
- sb.append("ALTER TABLE ").append(CatalogUtil.denormalizeIdentifier(table.getName()))
+ sb.append("ALTER TABLE ").append(IdentifierUtil.denormalizeIdentifier(table.getName()))
.append(" ADD IF NOT EXISTS PARTITION (");
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java
index 616b44e..1b34429 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java
@@ -21,16 +21,12 @@ package org.apache.tajo.catalog;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
-import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.exception.NotImplementedException;
import org.apache.tajo.exception.TajoRuntimeException;
+import org.apache.tajo.schema.Field;
import org.apache.tajo.schema.Identifier;
-import org.apache.tajo.schema.IdentifierPolicy;
+import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.schema.QualifiedIdentifier;
-import org.apache.tajo.schema.Schema;
-import org.apache.tajo.schema.Schema.NamedPrimitiveType;
-import org.apache.tajo.schema.Schema.NamedStructType;
-import org.apache.tajo.type.*;
import javax.annotation.Nullable;
import java.util.Collection;
@@ -42,58 +38,20 @@ public class FieldConverter {
final Collection<Identifier> identifiers = Collections2.transform(elems, new Function<String, Identifier>() {
@Override
public Identifier apply(@Nullable String input) {
- boolean needQuote = CatalogUtil.isShouldBeQuoted(input);
- return Identifier._(input, needQuote);
+ return Identifier._(input, IdentifierUtil.isShouldBeQuoted(input));
}
});
return QualifiedIdentifier.$(identifiers);
}
- public static TypeDesc convert(Schema.NamedType src) {
- if (src instanceof NamedStructType) {
- NamedStructType structType = (NamedStructType) src;
-
- ImmutableList.Builder<Column> fields = ImmutableList.builder();
- for (Schema.NamedType t: structType.fields()) {
- fields.add(new Column(t.name().raw(IdentifierPolicy.DefaultPolicy()), convert(t)));
- }
-
- return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build());
- } else {
- final NamedPrimitiveType namedType = (NamedPrimitiveType) src;
- final Type type = namedType.type();
- if (type instanceof Char) {
- Char charType = (Char) type;
- return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length()));
- } else if (type instanceof Varchar) {
- Varchar varcharType = (Varchar) type;
- return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length()));
- } else if (type instanceof Numeric) {
- Numeric numericType = (Numeric) type;
- return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision()));
- } else if (type instanceof Protobuf) {
- Protobuf protobuf = (Protobuf) type;
- return new TypeDesc(CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName()));
- } else {
- return new TypeDesc(TypeConverter.convert(namedType.type()));
- }
+ public static Field convert(Column column) {
+ if (column.type.isStruct() && column.getTypeDesc().getNestedSchema() == null) {
+ throw new TajoRuntimeException(new NotImplementedException("record type projection"));
}
+ return new Field(toQualifiedIdentifier(column.getQualifiedName()), column.type);
}
- public static Schema.NamedType convert(Column column) {
- if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) {
-
- if (column.getTypeDesc().getNestedSchema() == null) {
- throw new TajoRuntimeException(new NotImplementedException("record type projection"));
- }
-
- return new NamedStructType(toQualifiedIdentifier(column.getQualifiedName()),
- TypeConverter.convert(column.getTypeDesc()));
-
- } else {
- return new NamedPrimitiveType(toQualifiedIdentifier(column.getQualifiedName()),
- TypeConverter.convert(column.getDataType())
- );
- }
+ public static Column convert(Field field) {
+ return new Column(field.name().interned(), field.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java
index e0c9d9e..8f77489 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java
@@ -20,30 +20,30 @@ package org.apache.tajo.catalog;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
-import org.apache.tajo.schema.Schema.NamedType;
+import org.apache.tajo.schema.Field;
import java.util.Iterator;
public class ListSchemaBuilder implements SchemaBuilder.SchemaCollector {
- private final ImmutableList.Builder<NamedType> fields = new ImmutableList.Builder();
+ private final ImmutableList.Builder<Field> fields = new ImmutableList.Builder();
@Override
- public void add(NamedType namedType) {
- fields.add(namedType);
+ public void add(Field field) {
+ fields.add(field);
}
@Override
- public void addAll(Iterator<NamedType> fields) {
+ public void addAll(Iterator<Field> fields) {
this.fields.addAll(fields);
}
@Override
- public void addAll(Iterable<NamedType> fields) {
+ public void addAll(Iterable<Field> fields) {
this.fields.addAll(fields);
}
@Override
- public ImmutableCollection<NamedType> build() {
+ public ImmutableCollection<Field> build() {
return fields.build();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
index bf874b9..afe7e31 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
@@ -73,7 +73,7 @@ public class NestedPathUtil {
Column found = null;
if (currentColumn.getDataType().getType() == Type.RECORD) {
- found = currentColumn.typeDesc.nestedRecordSchema.getColumn(paths[depth]);
+ found = TypeConverter.convert(currentColumn.type).nestedRecordSchema.getColumn(paths[depth]);
}
if (found != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java
index 35e38b5..582b5ee 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java
@@ -22,18 +22,14 @@ import com.google.common.base.Function;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.schema.Field;
import org.apache.tajo.schema.QualifiedIdentifier;
-import org.apache.tajo.schema.Schema.NamedPrimitiveType;
-import org.apache.tajo.schema.Schema.NamedStructType;
-import org.apache.tajo.schema.Schema.NamedType;
import org.apache.tajo.type.Type;
import javax.annotation.Nullable;
-import java.util.Collection;
import java.util.Iterator;
import static org.apache.tajo.catalog.FieldConverter.toQualifiedIdentifier;
-import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy;
/**
* Builder for Schema
@@ -42,10 +38,10 @@ public class SchemaBuilder {
private final SchemaCollector fields;
public interface SchemaCollector {
- void add(NamedType field);
- void addAll(Iterator<NamedType> fields);
- void addAll(Iterable<NamedType> fields);
- ImmutableCollection<NamedType> build();
+ void add(Field field);
+ void addAll(Iterator<Field> fields);
+ void addAll(Iterable<Field> fields);
+ ImmutableCollection<Field> build();
}
public static SchemaLegacy empty() {
@@ -64,28 +60,19 @@ public class SchemaBuilder {
this.fields = collector;
}
- public SchemaBuilder add(NamedType namedType) {
- fields.add(namedType);
+ public SchemaBuilder add(Field field) {
+ fields.add(field);
return this;
}
public SchemaBuilder add(QualifiedIdentifier id, Type type) {
- add(new NamedPrimitiveType(id, type));
- return this;
- }
-
- public SchemaBuilder addStruct(QualifiedIdentifier id, Collection<NamedType> fields) {
- add(new NamedStructType(id, fields));
+ add(new Field(id, type));
return this;
}
@Deprecated
public SchemaBuilder add(String name, TypeDesc legacyType) {
- if (legacyType.getDataType().getType() == TajoDataTypes.Type.RECORD) {
- addStruct(toQualifiedIdentifier(name), TypeConverter.convert(legacyType));
- } else {
- add(toQualifiedIdentifier(name), TypeConverter.convert(legacyType.getDataType()));
- }
+ add(toQualifiedIdentifier(name), TypeConverter.convert(legacyType));
return this;
}
@@ -109,9 +96,9 @@ public class SchemaBuilder {
@Deprecated
public SchemaBuilder addAll(Iterable<Column> columns) {
- return addAll2(columns, new Function<Column, NamedType>() {
+ return addAll2(columns, new Function<Column, Field>() {
@Override
- public NamedType apply(@Nullable Column input) {
+ public Field apply(@Nullable Column input) {
return FieldConverter.convert(input);
}
});
@@ -119,9 +106,9 @@ public class SchemaBuilder {
@Deprecated
public SchemaBuilder addAll(Column [] columns) {
- return addAll2(columns, new Function<Column, NamedType>() {
+ return addAll2(columns, new Function<Column, Field>() {
@Override
- public NamedType apply(@Nullable Column input) {
+ public Field apply(@Nullable Column input) {
return FieldConverter.convert(input);
}
});
@@ -152,21 +139,26 @@ public class SchemaBuilder {
return this;
}
- public <T> SchemaBuilder addAll2(T [] fields, Function<T, NamedType> fn) {
+ public SchemaBuilder addAll2(Iterable<Field> fields) {
+ this.fields.addAll(fields);
+ return this;
+ }
+
+ public <T> SchemaBuilder addAll2(T [] fields, Function<T, Field> fn) {
for (T t : fields) {
add(fn.apply(t));
}
return this;
}
- public <T> SchemaBuilder addAll2(Iterable<T> fields, Function<T, NamedType> fn) {
+ public <T> SchemaBuilder addAll2(Iterable<T> fields, Function<T, Field> fn) {
for (T t : fields) {
add(fn.apply(t));
}
return this;
}
- public <T> SchemaBuilder addAll2(Iterator<T> fields, Function<T, NamedType> fn) {
+ public <T> SchemaBuilder addAll2(Iterator<T> fields, Function<T, Field> fn) {
while(fields.hasNext()) {
T t = fields.next();
add(fn.apply(t));
@@ -177,8 +169,8 @@ public class SchemaBuilder {
@Deprecated
public SchemaLegacy build() {
ImmutableList.Builder<Column> columns = new ImmutableList.Builder();
- for (NamedType namedType : fields.build()) {
- columns.add(new Column(namedType.name().raw(DefaultPolicy()), FieldConverter.convert(namedType)));
+ for (Field field : fields.build()) {
+ columns.add(new Column(field.name().interned(), field.type()));
}
return new SchemaLegacy(columns.build());
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java
index 2a16fa6..db1e566 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java
@@ -18,23 +18,27 @@
package org.apache.tajo.catalog;
+import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.SchemaUtil.ColumnVisitor;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.common.TajoDataTypes.DataType;
-import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.exception.DuplicateColumnException;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.json.GsonObject;
import org.apache.tajo.util.StringUtils;
+import javax.annotation.Nullable;
import java.util.*;
+import static com.google.common.collect.Collections2.transform;
+
+@Deprecated
public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable, GsonObject {
@Expose protected List<Column> fields = null;
@@ -55,47 +59,18 @@ public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable
public SchemaLegacy(SchemaProto proto) {
init();
- List<Column> toBeAdded = new ArrayList<>();
- for (int i = 0; i < proto.getFieldsCount(); i++) {
- deserializeColumn(toBeAdded, proto.getFieldsList(), i);
- }
+ Collection<Column> toBeAdded = transform(proto.getFieldsList(), new Function<ColumnProto, Column>() {
+ @Override
+ public Column apply(@Nullable ColumnProto proto) {
+ return new Column(proto);
+ }
+ });
for (Column c : toBeAdded) {
addColumn(c);
}
}
- /**
- * This method transforms a list of ColumnProtos into a schema tree.
- * It assumes that <code>protos</code> contains a list of ColumnProtos in the depth-first order.
- *
- * @param tobeAdded
- * @param protos
- * @param serializedColumnIndex
- */
- private static void deserializeColumn(List<Column> tobeAdded, List<ColumnProto> protos, int serializedColumnIndex) {
- ColumnProto columnProto = protos.get(serializedColumnIndex);
- if (columnProto.getDataType().getType() == Type.RECORD) {
-
- // Get the number of child fields
- int childNum = columnProto.getDataType().getNumNestedFields();
- // where is start index of nested fields?
- int childStartIndex = tobeAdded.size() - childNum;
- // Extract nested fields
- List<Column> nestedColumns = new ArrayList<>(tobeAdded.subList(childStartIndex, childStartIndex + childNum));
-
- // Remove nested fields from the the current level
- for (int i = 0; i < childNum; i++) {
- tobeAdded.remove(tobeAdded.size() - 1);
- }
-
- // Add the nested fields to the list as a single record column
- tobeAdded.add(new Column(columnProto.getName(), new TypeDesc(new SchemaLegacy(nestedColumns))));
- } else {
- tobeAdded.add(new Column(protos.get(serializedColumnIndex)));
- }
- }
-
public SchemaLegacy(Schema schema) {
new SchemaLegacy(schema.getRootColumns());
}
@@ -139,7 +114,7 @@ public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable
Column newColumn;
for (Column c : columns) {
- newColumn = new Column(qualifier + "." + c.getSimpleName(), c.typeDesc);
+ newColumn = new Column(qualifier + "." + c.getSimpleName(), c.type);
addColumn(newColumn);
}
}
@@ -200,7 +175,7 @@ public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable
Column columnPath = new Column(
column.getQualifiedName() + NestedPathUtil.makePath(paths, 1),
- actualColumn.typeDesc);
+ actualColumn.type);
return columnPath;
} else {
@@ -424,13 +399,13 @@ public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable
return false;
}
- private SchemaLegacy addColumn(String name, TypeDesc typeDesc) {
+ private SchemaLegacy addColumn(String name, org.apache.tajo.type.Type type) {
String normalized = name;
if(fieldsByQualifiedName.containsKey(normalized)) {
throw new TajoRuntimeException(new DuplicateColumnException(normalized));
}
- Column newCol = new Column(normalized, typeDesc);
+ Column newCol = new Column(normalized, type);
fields.add(newCol);
fieldsByQualifiedName.put(newCol.getQualifiedName(), fields.size() - 1);
List<Integer> inputList = new ArrayList<>();
@@ -441,7 +416,7 @@ public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable
}
private synchronized void addColumn(Column column) {
- addColumn(column.getQualifiedName(), column.typeDesc);
+ addColumn(column.getQualifiedName(), column.type);
}
@Override
@@ -472,34 +447,15 @@ public class SchemaLegacy implements Schema, ProtoObject<SchemaProto>, Cloneable
@Override
public SchemaProto getProto() {
SchemaProto.Builder builder = SchemaProto.newBuilder();
- SchemaProtoBuilder recursiveBuilder = new SchemaProtoBuilder(builder);
- SchemaUtil.visitSchema(this, recursiveBuilder);
+ builder.addAllFields(Iterables.transform(getRootColumns(), new Function<Column, ColumnProto>() {
+ @Override
+ public ColumnProto apply(@Nullable Column column) {
+ return column.getProto();
+ }
+ }));
return builder.build();
}
- private static class SchemaProtoBuilder implements ColumnVisitor {
- private SchemaProto.Builder builder;
- public SchemaProtoBuilder(SchemaProto.Builder builder) {
- this.builder = builder;
- }
-
- @Override
- public void visit(int depth, List<String> path, Column column) {
-
- if (column.getDataType().getType() == Type.RECORD) {
- DataType.Builder updatedType = DataType.newBuilder(column.getDataType());
- updatedType.setNumNestedFields(column.typeDesc.nestedRecordSchema.size());
-
- ColumnProto.Builder updatedColumn = ColumnProto.newBuilder(column.getProto());
- updatedColumn.setDataType(updatedType);
-
- builder.addFields(updatedColumn.build());
- } else {
- builder.addFields(column.getProto());
- }
- }
- }
-
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
index c0b60a3..fc6d353 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
@@ -185,7 +185,7 @@ public class SchemaUtil {
Column column) {
if (column.getDataType().getType() == Type.RECORD) {
- for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getRootColumns()) {
+ for (Column nestedColumn : TypeConverter.convert(column.type).nestedRecordSchema.getRootColumns()) {
List<String> newPath = new ArrayList<>(path);
newPath.add(column.getQualifiedName());
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java
index b66aac2..4a4099e 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java
@@ -20,47 +20,41 @@ package org.apache.tajo.catalog;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
-import org.apache.commons.collections.collection.UnmodifiableCollection;
-import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.schema.QualifiedIdentifier;
-import org.apache.tajo.schema.Schema.NamedPrimitiveType;
-import org.apache.tajo.schema.Schema.NamedStructType;
-import org.apache.tajo.schema.Schema.NamedType;
-import org.apache.tajo.type.Type;
+import org.apache.tajo.schema.Field;
-import java.util.*;
-
-import static org.apache.tajo.catalog.FieldConverter.toQualifiedIdentifier;
-import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
public class SetSchemaBuilder implements SchemaBuilder.SchemaCollector {
private final Set<QualifiedIdentifier> nameSet = new HashSet<>();
- private final ImmutableList.Builder<NamedType> fields = new ImmutableList.Builder();
+ private final ImmutableList.Builder<Field> fields = new ImmutableList.Builder();
@Override
- public void add(NamedType namedType) {
- if (!nameSet.contains(namedType.name())) {
- fields.add(namedType);
- nameSet.add(namedType.name());
+ public void add(Field field) {
+ if (!nameSet.contains(field.name())) {
+ fields.add(field);
+ nameSet.add(field.name());
}
}
@Override
- public void addAll(Iterator<NamedType> fields) {
+ public void addAll(Iterator<Field> fields) {
while (fields.hasNext()) {
add(fields.next());
}
}
@Override
- public void addAll(Iterable<NamedType> fields) {
- for (NamedType n : fields) {
+ public void addAll(Iterable<Field> fields) {
+ for (Field n : fields) {
add(n);
}
}
@Override
- public ImmutableCollection<NamedType> build() {
+ public ImmutableCollection<Field> build() {
return fields.build();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java
index f4164d4..387cf00 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java
@@ -20,66 +20,25 @@ package org.apache.tajo.catalog;
import com.google.common.collect.ImmutableList;
import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.exception.NotImplementedException;
-import org.apache.tajo.exception.TajoRuntimeException;
-import org.apache.tajo.exception.UnsupportedException;
-import org.apache.tajo.schema.Schema;
+import org.apache.tajo.schema.Field;
import org.apache.tajo.type.*;
-import java.util.Collection;
-
+import static org.apache.tajo.catalog.CatalogUtil.newDataTypeWithLen;
+import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType;
+import static org.apache.tajo.common.TajoDataTypes.Type.*;
import static org.apache.tajo.type.Type.*;
public class TypeConverter {
- public static Collection<Schema.NamedType> convert(TypeDesc type) {
- ImmutableList.Builder<Schema.NamedType> fields = ImmutableList.builder();
- for (Column c : type.getNestedSchema().getRootColumns()) {
- fields.add(FieldConverter.convert(c));
- }
- return fields.build();
- }
-
- public static Type convert(TajoDataTypes.Type legacyBaseType) {
- switch (legacyBaseType) {
- case BOOLEAN:
- return Bool;
- case INT1:
- return Int1;
- case INT2:
- return Int2;
- case INT4:
- return Int4;
- case INT8:
- return Int8;
- case FLOAT4:
- return Float4;
- case FLOAT8:
- return Float8;
- case DATE:
- return Date;
- case TIME:
- return Time;
- case TIMESTAMP:
- return Timestamp;
- case INTERVAL:
- return Interval;
- case CHAR:
- return Char(1); // default len = 1
- case TEXT:
- return Text;
- case BLOB:
- return Blob;
- case INET4:
- return Inet4;
- case RECORD:
- throw new TajoRuntimeException(new NotImplementedException("record projection"));
- case NULL_TYPE:
- return Null;
- case ANY:
- return Any;
- default:
- throw new TajoRuntimeException(new UnsupportedException(legacyBaseType.name()));
+ public static Type convert(TypeDesc type) {
+ if (type.getDataType().getType() == TajoDataTypes.Type.RECORD) {
+ ImmutableList.Builder<Field> fields = ImmutableList.builder();
+ for (Column c : type.getNestedSchema().getRootColumns()) {
+ fields.add(FieldConverter.convert(c));
+ }
+ return Record(fields.build());
+ } else {
+ return convert(type.dataType);
}
}
@@ -96,26 +55,58 @@ public class TypeConverter {
case PROTOBUF:
return new Protobuf(legacyType.getCode());
default:
- return convert(legacyType.getType());
+ return TypeFactory.create(legacyType.getType());
}
}
- public static TajoDataTypes.DataType convert(Type type) {
- switch (type.baseType()) {
- case CHAR:
- Char charType = (Char) type;
- return CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length());
- case VARCHAR:
- Varchar varcharType = (Varchar) type;
- return CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length());
- case PROTOBUF:
- Protobuf protobuf = (Protobuf) type;
- return CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName());
- case NUMERIC:
- Numeric numericType = (Numeric) type;
- return CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision());
+ public static TypeDesc convert(Field src) {
+ return convert(src.type());
+ }
+
+ public static TypeDesc convert(Type type) {
+ switch (type.kind()) {
+ case CHAR:
+ Char charType = (Char) type;
+ return new TypeDesc(newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length()));
+ case VARCHAR:
+ Varchar varcharType = (Varchar) type;
+ return new TypeDesc(newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length()));
+ case NUMERIC:
+ Numeric numericType = (Numeric) type;
+ return new TypeDesc(newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision()));
+ case PROTOBUF:
+ Protobuf protobuf = (Protobuf) type;
+ return new TypeDesc(CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName()));
+ case RECORD:
+ Record record = (Record) type;
+ ImmutableList.Builder<Column> fields = ImmutableList.builder();
+ for (Field t: record.fields()) {
+ fields.add(new Column(t.name().interned(), convert(t)));
+ }
+ return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build());
+
+ case ARRAY:
+ Array array = (Array) type;
+ Type elemType = array.elementType();
+ switch (elemType.kind()) {
+ case INT1:
+ return new TypeDesc(newSimpleDataType(INT1_ARRAY));
+ case INT2:
+ return new TypeDesc(newSimpleDataType(INT2_ARRAY));
+ case INT4:
+ return new TypeDesc(newSimpleDataType(INT4_ARRAY));
+ case INT8:
+ return new TypeDesc(newSimpleDataType(INT8_ARRAY));
+ case FLOAT4:
+ return new TypeDesc(newSimpleDataType(FLOAT4_ARRAY));
+ case FLOAT8:
+ return new TypeDesc(newSimpleDataType(FLOAT8_ARRAY));
default:
- return CatalogUtil.newSimpleDataType(type.baseType());
+ return new TypeDesc(newSimpleDataType(type.kind()));
+ }
+
+ default:
+ return new TypeDesc(newSimpleDataType(type.kind()));
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
index c145ecd..27c2309 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
@@ -48,6 +48,7 @@ public class CatalogGsonHelper {
adapters.put(Datum.class, new DatumAdapter());
adapters.put(DataType.class, new DataTypeAdapter());
adapters.put(Schema.class, new SchemaAdapter());
+ adapters.put(org.apache.tajo.type.Type.class, new TypeAdapter());
return adapters;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
index c66500d..47acb50 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
@@ -26,6 +26,7 @@ import com.google.gson.Gson;
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.common.TajoDataTypes.TypeProto;
import org.apache.tajo.json.GsonObject;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
@@ -35,6 +36,8 @@ import org.apache.tajo.util.TUtil;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.tajo.common.TajoDataTypes.Type.PROTOBUF;
+
public class TableStats implements ProtoObject<TableStatsProto>, Cloneable, GsonObject {
@Expose private Long numRows = null; // required
@Expose private Long numBytes = null; // required
@@ -85,13 +88,17 @@ public class TableStats implements ProtoObject<TableStatsProto>, Cloneable, Gson
this.columnStatses = new ArrayList<>();
for (CatalogProtos.ColumnStatsProto colProto : proto.getColStatList()) {
- if (colProto.getColumn().getDataType().getType() == TajoDataTypes.Type.PROTOBUF) {
+ if (peekType(colProto.getColumn().getType()) == PROTOBUF) {
continue;
}
columnStatses.add(new ColumnStats(colProto));
}
}
+ private static TajoDataTypes.Type peekType(TypeProto proto) {
+ return proto.getElements(proto.getElementsCount() - 1).getKind();
+ }
+
public Long getNumRows() {
return this.numRows;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index d064c62..e79bc75 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -66,7 +66,7 @@ enum AlterTableType {
message ColumnProto {
required string name = 1;
optional int32 tid = 2;
- required DataType data_type = 3;
+ required TypeProto type = 3;
}
message SchemaProto {
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java
index 0a7fd0a..c09909a 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java
@@ -20,6 +20,7 @@ package org.apache.tajo.catalog;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.function.FunctionUtil;
+import org.apache.tajo.schema.IdentifierUtil;
import org.junit.Test;
import java.util.Arrays;
@@ -51,7 +52,7 @@ public class TestCatalogUtil {
@Test
public final void testNormalizeIdentifier() {
for (int i = 0; i < sources.length; i++) {
- assertEquals(normalized[i], CatalogUtil.normalizeIdentifier(sources[i]));
+ assertEquals(normalized[i], IdentifierUtil.normalizeIdentifier(sources[i]));
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index 6aa0585..b855c77 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -57,7 +57,9 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.*;
import org.apache.tajo.plan.expr.AlgebraicUtil;
import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor;
+import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.type.TypeProtobufEncoder;
import org.apache.tajo.util.KeyValueSet;
import org.apache.thrift.TException;
@@ -165,8 +167,8 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
}
if (!isPartitionKey) {
- String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName +
- CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName();
+ String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName +
+ IdentifierUtil.IDENTIFIER_DELIMITER + eachField.getName();
TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(eachField.getType());
schemaBuilder.add(fieldName, dataType);
}
@@ -246,8 +248,8 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
for (int i = 0; i < partitionKeys.size(); i++) {
FieldSchema fieldSchema = partitionKeys.get(i);
TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType());
- String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName +
- CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName();
+ String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName +
+ IdentifierUtil.IDENTIFIER_DELIMITER + fieldSchema.getName();
expressionSchema.add(new Column(fieldName, dataType));
if (i > 0) {
sb.append(",");
@@ -421,7 +423,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
TableDesc tableDesc = new TableDesc(tableDescProto);
- String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName());
+ String[] splitted = IdentifierUtil.splitFQTableName(tableDesc.getName());
String databaseName = splitted[0];
String tableName = splitted[1];
@@ -460,7 +462,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
for (Column eachField : columns) {
cols.add(new FieldSchema(eachField.getSimpleName(),
- HiveCatalogUtil.getHiveFieldType(eachField.getDataType()), ""));
+ HiveCatalogUtil.getHiveFieldType(eachField.getType()), ""));
}
sd.setCols(cols);
@@ -469,7 +471,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
List<FieldSchema> partitionKeys = new ArrayList<>();
for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(),
- HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), ""));
+ HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getType()), ""));
}
table.setPartitionKeys(partitionKeys);
}
@@ -605,7 +607,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
throws DuplicateTableException, DuplicateColumnException, DuplicatePartitionException,
UndefinedPartitionException {
- final String[] split = CatalogUtil.splitFQTableName(alterTableDescProto.getTableName());
+ final String[] split = IdentifierUtil.splitFQTableName(alterTableDescProto.getTableName());
if (split.length == 1) {
throw new IllegalArgumentException("alterTable() requires a qualified table name, but it is \""
@@ -713,7 +715,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
Table table = client.getHiveClient().getTable(databaseName, tableName);
List<FieldSchema> columns = table.getSd().getCols();
columns.add(new FieldSchema(columnProto.getName(),
- HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), ""));
+ HiveCatalogUtil.getHiveFieldType(TypeProtobufEncoder.decode(columnProto.getType())), ""));
client.getHiveClient().alter_table(databaseName, tableName, table);
@@ -812,8 +814,8 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
for (int i = 0; i < partitionKeys.size(); i++) {
FieldSchema fieldSchema = partitionKeys.get(i);
TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType());
- String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName +
- CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName();
+ String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName +
+ IdentifierUtil.IDENTIFIER_DELIMITER + fieldSchema.getName();
expressionSchema.add(new Column(fieldName, dataType));
if (i > 0) {
sb.append(",");
@@ -1041,7 +1043,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
if (i > 0) {
partitionName.append(File.separator);
}
- partitionName.append(CatalogUtil.extractSimpleName(parititonColumns.get(i).getName()));
+ partitionName.append(IdentifierUtil.extractSimpleName(parititonColumns.get(i).getName()));
partitionName.append("=");
partitionName.append(hivePartition.getValues().get(i));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/8d2bf407/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java
index 87b391e..faefd28 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java
@@ -39,6 +39,8 @@ import org.apache.tajo.exception.LMDNoMatchedDatatypeException;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.exception.UnknownDataFormatException;
import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.type.Type;
+import org.apache.tajo.type.TypeStringEncoder;
import org.apache.thrift.TException;
public class HiveCatalogUtil {
@@ -82,10 +84,10 @@ public class HiveCatalogUtil {
}
}
- public static String getHiveFieldType(TajoDataTypes.DataType dataType) throws LMDNoMatchedDatatypeException {
- Preconditions.checkNotNull(dataType);
+ public static String getHiveFieldType(Type type) throws LMDNoMatchedDatatypeException {
+ Preconditions.checkNotNull(type);
- switch (dataType.getType()) {
+ switch (type.kind()) {
case CHAR: return serdeConstants.CHAR_TYPE_NAME;
case BOOLEAN: return serdeConstants.BOOLEAN_TYPE_NAME;
case INT1: return serdeConstants.TINYINT_TYPE_NAME;
@@ -104,7 +106,7 @@ public class HiveCatalogUtil {
case DATE: return serdeConstants.DATE_TYPE_NAME;
case TIMESTAMP: return serdeConstants.TIMESTAMP_TYPE_NAME;
default:
- throw new LMDNoMatchedDatatypeException(dataType.getType().name());
+ throw new LMDNoMatchedDatatypeException(TypeStringEncoder.encode(type));
}
}