You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/03/13 04:24:25 UTC
[1/3] tajo git commit: TAJO-1395: Remove deprecated sql files for
Oracle and PostgreSQL
Repository: tajo
Updated Branches:
refs/heads/index_support 292cd9e41 -> 5bb4108c9
TAJO-1395: Remove deprecated sql files for Oracle and PostgreSQL
Closes #415
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/6defbda9
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/6defbda9
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/6defbda9
Branch: refs/heads/index_support
Commit: 6defbda966912ee2dd42197c2a82e94439b25bd8
Parents: 4a9da73
Author: Jihun Kang <ji...@apache.org>
Authored: Thu Mar 12 23:27:58 2015 +0900
Committer: Jihun Kang <ji...@apache.org>
Committed: Thu Mar 12 23:27:58 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../src/main/resources/schemas/oracle/columns.sql | 9 ---------
.../src/main/resources/schemas/oracle/databases.sql | 6 ------
.../src/main/resources/schemas/oracle/indexes.sql | 14 --------------
.../resources/schemas/oracle/partition_methods.sql | 7 -------
.../src/main/resources/schemas/oracle/partitions.sql | 10 ----------
.../src/main/resources/schemas/oracle/stats.sql | 6 ------
.../resources/schemas/oracle/table_properties.sql | 7 -------
.../src/main/resources/schemas/oracle/tables.sql | 9 ---------
.../src/main/resources/schemas/oracle/tablespaces.sql | 6 ------
.../src/main/resources/schemas/postgresql/columns.sql | 9 ---------
.../main/resources/schemas/postgresql/databases.sql | 6 ------
.../schemas/postgresql/partition_methods.sql | 7 -------
.../main/resources/schemas/postgresql/partitions.sql | 10 ----------
.../resources/schemas/postgresql/table_properties.sql | 7 -------
.../src/main/resources/schemas/postgresql/tables.sql | 9 ---------
.../main/resources/schemas/postgresql/tablespaces.sql | 6 ------
17 files changed, 2 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 39f0fc4..6854b4f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -9,6 +9,8 @@ Release 0.11.0 - unreleased
IMPROVEMENT
+ TAJO-1395: Remove deprecated sql files for Oracle and PostgreSQL. (jihun)
+
TAJO-1394: Support reconnect on tsql.
(Contributed by navis, Committed by hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/columns.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/columns.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/columns.sql
deleted file mode 100644
index 71943c0..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/columns.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-CREATE TABLE COLUMNS (
- TID INT NOT NULL,
- COLUMN_NAME VARCHAR2(255) NOT NULL,
- ORDINAL_POSITION INT NOT NULL,
- DATA_TYPE CHAR(16),
- TYPE_LENGTH INTEGER,
- CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, COLUMN_NAME),
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/databases.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/databases.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/databases.sql
deleted file mode 100644
index 33d82d8..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/databases.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-CREATE TABLE DATABASES_ (
- DB_ID NUMBER(10) NOT NULL PRIMARY KEY,
- DB_NAME VARCHAR2(128) NOT NULL UNIQUE,
- SPACE_ID INT NOT NULL,
- FOREIGN KEY (SPACE_ID) REFERENCES TABLESPACES (SPACE_ID)
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql
deleted file mode 100644
index ae31d6f..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql
+++ /dev/null
@@ -1,14 +0,0 @@
-CREATE TABLE INDEXES (
- DB_ID INT NOT NULL,
- TID INT NOT NULL,
- INDEX_NAME VARCHAR2(128) NOT NULL,
- COLUMN_NAME VARCHAR2(128) NOT NULL,
- DATA_TYPE VARCHAR2(128) NOT NULL,
- INDEX_TYPE CHAR(32) NOT NULL,
- IS_UNIQUE CHAR NOT NULL,
- IS_CLUSTERED CHAR NOT NULL,
- IS_ASCENDING CHAR NOT NULL,
- CONSTRAINT INDEXES_PKEY PRIMARY KEY (DB_ID, INDEX_NAME),
- FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/partition_methods.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/partition_methods.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/partition_methods.sql
deleted file mode 100644
index 7d37a6e..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/partition_methods.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-CREATE TABLE PARTITION_METHODS (
- TID INT NOT NULL PRIMARY KEY,
- PARTITION_TYPE VARCHAR2(10) NOT NULL,
- EXPRESSION VARCHAR2(1024) NOT NULL,
- EXPRESSION_SCHEMA RAW(1024) NOT NULL,
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
-)
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/partitions.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/partitions.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/partitions.sql
deleted file mode 100644
index 2fee267..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/partitions.sql
+++ /dev/null
@@ -1,10 +0,0 @@
-CREATE TABLE PARTITIONS (
- PID INT NOT NULL PRIMARY KEY,
- TID INT NOT NULL,
- PARTITION_NAME VARCHAR2(128),
- ORDINAL_POSITION INT NOT NULL,
- PARTITION_VALUE VARCHAR2(1024),
- PATH VARCHAR2(4000),
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
- CONSTRAINT C_PARTITION_UNIQUE UNIQUE (TID, PARTITION_NAME)
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/stats.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/stats.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/stats.sql
deleted file mode 100644
index d7091b4..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/stats.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-CREATE TABLE STATS (
- TID INT NOT NULL PRIMARY KEY,
- NUM_ROWS NUMBER(38),
- NUM_BYTES NUMBER(38),
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/table_properties.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/table_properties.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/table_properties.sql
deleted file mode 100644
index 130402c..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/table_properties.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-CREATE TABLE OPTIONS (
- TID INT NOT NULL,
- KEY_ VARCHAR2(255) NOT NULL,
- VALUE_ VARCHAR2(255) NOT NULL,
- CONSTRAINT OPTIONS_PKEY PRIMARY KEY (TID, KEY_),
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/tables.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/tables.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/tables.sql
deleted file mode 100644
index 36f8067..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/tables.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-CREATE TABLE TABLES (
- TID NUMBER(10) NOT NULL PRIMARY KEY,
- DB_ID INT NOT NULL,
- TABLE_NAME VARCHAR2(128) NOT NULL,
- TABLE_TYPE VARCHAR2(128) NOT NULL,
- PATH VARCHAR2(4000),
- STORE_TYPE CHAR(16),
- FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID)
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/tablespaces.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/tablespaces.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/tablespaces.sql
deleted file mode 100644
index 461e77c..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/tablespaces.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-CREATE TABLE TABLESPACES (
- SPACE_ID NUMBER(10) NOT NULL PRIMARY KEY,
- SPACE_NAME VARCHAR2(128) NOT NULL UNIQUE,
- SPACE_HANDLER VARCHAR2(1024) DEFAULT 'HDFS',
- SPACE_URI VARCHAR2(4000) NOT NULL
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/columns.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/columns.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/columns.sql
deleted file mode 100644
index 425be96..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/columns.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-CREATE TABLE COLUMNS (
- TID INT NOT NULL,
- COLUMN_NAME VARCHAR(255) NOT NULL,
- ORDINAL_POSITION INT NOT NULL,
- DATA_TYPE CHAR(16),
- TYPE_LENGTH INTEGER,
- CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, COLUMN_NAME),
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/databases.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/databases.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/databases.sql
deleted file mode 100644
index 8232884..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/databases.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-CREATE TABLE DATABASES_ (
- DB_ID SERIAL NOT NULL PRIMARY KEY,
- DB_NAME VARCHAR(128) NOT NULL UNIQUE,
- SPACE_ID INT NOT NULL,
- FOREIGN KEY (SPACE_ID) REFERENCES TABLESPACES (SPACE_ID)
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/partition_methods.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/partition_methods.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/partition_methods.sql
deleted file mode 100644
index 55289ae..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/partition_methods.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-CREATE TABLE PARTITION_METHODS (
- TID INT NOT NULL PRIMARY KEY,
- PARTITION_TYPE VARCHAR(10) NOT NULL,
- EXPRESSION VARCHAR(1024) NOT NULL,
- EXPRESSION_SCHEMA BYTEA NOT NULL,
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/partitions.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/partitions.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/partitions.sql
deleted file mode 100644
index d46a59a..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/partitions.sql
+++ /dev/null
@@ -1,10 +0,0 @@
-CREATE TABLE PARTITIONS (
- PID INT NOT NULL PRIMARY KEY,
- TID INT NOT NULL,
- PARTITION_NAME VARCHAR(128),
- ORDINAL_POSITION INT NOT NULL,
- PARTITION_VALUE VARCHAR(1024),
- PATH VARCHAR(4096),
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
- CONSTRAINT C_PARTITION_UNIQUE UNIQUE (TID, PARTITION_NAME)
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/table_properties.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/table_properties.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/table_properties.sql
deleted file mode 100644
index 57aa316..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/table_properties.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-CREATE TABLE OPTIONS (
- TID INT NOT NULL,
- KEY_ VARCHAR(255) NOT NULL,
- VALUE_ VARCHAR(255) NOT NULL,
- CONSTRAINT OPTIONS_PKEY PRIMARY KEY (TID, KEY_),
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/tables.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/tables.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/tables.sql
deleted file mode 100644
index d87264a..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/tables.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-CREATE TABLE TABLES (
- TID SERIAL NOT NULL PRIMARY KEY,
- DB_ID INT NOT NULL,
- TABLE_NAME VARCHAR(128) NOT NULL,
- TABLE_TYPE VARCHAR(128) NOT NULL,
- PATH VARCHAR(4096),
- STORE_TYPE CHAR(16),
- FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID)
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/6defbda9/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/tablespaces.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/tablespaces.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/tablespaces.sql
deleted file mode 100644
index 65a366d..0000000
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/tablespaces.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-CREATE TABLE TABLESPACES (
- SPACE_ID SERIAL NOT NULL PRIMARY KEY,
- SPACE_NAME VARCHAR(128) NOT NULL UNIQUE,
- SPACE_HANDLER VARCHAR (1024) DEFAULT 'HDFS',
- SPACE_URI VARCHAR (4096) NOT NULL
-)
\ No newline at end of file
[2/3] tajo git commit: TAJO-1329: Improve Schema class to support
nested struct support.
Posted by ji...@apache.org.
TAJO-1329: Improve Schema class to support nested struct support.
Closes #389
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/f48d4bd0
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/f48d4bd0
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/f48d4bd0
Branch: refs/heads/index_support
Commit: f48d4bd04ec46b6678773209a3397555eb2ce24d
Parents: 6defbda
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu Mar 12 19:21:39 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu Mar 12 19:21:39 2015 -0700
----------------------------------------------------------------------
CHANGES | 3 +
.../apache/tajo/algebra/ColumnDefinition.java | 2 +-
.../org/apache/tajo/algebra/DataTypeExpr.java | 18 ++-
.../org/apache/tajo/catalog/CatalogUtil.java | 28 ++--
.../java/org/apache/tajo/catalog/Column.java | 45 ++++--
.../org/apache/tajo/catalog/DDLBuilder.java | 9 +-
.../java/org/apache/tajo/catalog/Schema.java | 156 +++++++++++++------
.../org/apache/tajo/catalog/SchemaUtil.java | 42 +++++
.../java/org/apache/tajo/catalog/TypeDesc.java | 75 +++++++++
.../org/apache/tajo/catalog/TestSchema.java | 137 ++++++++++++++++
.../dictionary/ColumnsTableDescriptor.java | 7 +-
.../tajo/catalog/store/AbstractDBStore.java | 64 +++++---
.../apache/tajo/catalog/store/MariaDBStore.java | 5 +-
.../apache/tajo/catalog/store/MySQLStore.java | 11 +-
.../apache/tajo/catalog/store/OracleStore.java | 2 +-
.../tajo/catalog/store/PostgreSQLStore.java | 4 +-
.../src/main/resources/schemas/derby/derby.xml | 12 +-
.../main/resources/schemas/mariadb/columns.sql | 3 +-
.../main/resources/schemas/mysql/columns.sql | 3 +-
.../main/resources/schemas/oracle/oracle.xml | 10 +-
.../resources/schemas/postgresql/postgresql.xml | 10 +-
.../org/apache/tajo/catalog/TestCatalog.java | 101 ++++++++++++
.../cli/tsql/commands/DescTableCommand.java | 5 +-
tajo-common/src/main/proto/DataTypes.proto | 10 ++
tajo-core/pom.xml | 56 +++++++
.../org/apache/tajo/cli/tools/TestTajoDump.java | 21 +++
.../org/apache/tajo/cli/tsql/TestTajoCli.java | 40 +++--
.../tajo/engine/query/TestCreateTable.java | 24 +++
.../TestCreateTable/testNestedRecord1.sql | 1 +
.../TestCreateTable/testNestedRecord2.sql | 1 +
.../TestSelectQuery/testExplainSelect.result | 4 +-
.../testDescTableForNestedSchema.result | 29 ++++
.../results/TestTajoDump/testDump2.result | 16 ++
.../org/apache/tajo/plan/ExprAnnotator.java | 5 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 18 ++-
.../org/apache/tajo/plan/TypeDeterminant.java | 4 +-
36 files changed, 828 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 6854b4f..9bec78c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -59,6 +59,9 @@ Release 0.11.0 - unreleased
SUB TASKS
+ TAJO-1329: Improve Schema class to support nested struct support.
+ (hyunsik)
+
TAJO-1353: Nested record support in CREATE TABLE statement. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
index 80ecac4..f8ea0f1 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
@@ -44,7 +44,7 @@ public class ColumnDefinition extends DataTypeExpr {
// nested records
if (dataType.isNestedRecordType()) {
- this.nestedRecord = dataType.nestedRecord;
+ this.nestedRecordTypes = dataType.nestedRecordTypes;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
index 28f9a5e..b280397 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
@@ -21,7 +21,6 @@ package org.apache.tajo.algebra;
import com.google.common.base.Objects;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
-import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.util.TUtil;
@@ -33,19 +32,18 @@ public class DataTypeExpr extends Expr {
@Expose @SerializedName("Scale")
Integer scale;
@Expose @SerializedName("Record")
- DataTypeExpr [] nestedRecord;
+ ColumnDefinition [] nestedRecordTypes; // not null if the type is RECORD
public DataTypeExpr(String typeName) {
super(OpType.DataType);
this.typeName = typeName;
}
- public DataTypeExpr(DataTypeExpr [] nestedRecordTypes) {
+ public DataTypeExpr(ColumnDefinition [] nestedRecordTypes) {
super(OpType.DataType);
- // Please refer to DataTypes.proto. 'STRUCT' must be equivalent to Enum type in DataTypes.proto.
- // STRUCT = 51;
+ // RECORD = 51 in DataTypes.proto
this.typeName = Type.RECORD.name();
- this.nestedRecord = nestedRecordTypes;
+ this.nestedRecordTypes = nestedRecordTypes;
}
public String getTypeName() {
@@ -56,6 +54,10 @@ public class DataTypeExpr extends Expr {
return this.typeName.equals(Type.RECORD.name());
}
+ public ColumnDefinition [] getNestedRecordTypes() {
+ return nestedRecordTypes;
+ }
+
public boolean hasLengthOrPrecision() {
return lengthOrPrecision != null;
}
@@ -91,7 +93,7 @@ public class DataTypeExpr extends Expr {
return typeName.equals(another.typeName) &&
TUtil.checkEquals(lengthOrPrecision, another.lengthOrPrecision) &&
TUtil.checkEquals(scale, another.scale) &&
- TUtil.checkEquals(nestedRecord, another.nestedRecord);
+ TUtil.checkEquals(nestedRecordTypes, another.nestedRecordTypes);
}
@Override
@@ -100,7 +102,7 @@ public class DataTypeExpr extends Expr {
dataType.typeName = typeName;
dataType.lengthOrPrecision = lengthOrPrecision;
dataType.scale = scale;
- dataType.nestedRecord = nestedRecord;
+ dataType.nestedRecordTypes = nestedRecordTypes;
return dataType;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 45609d0..afcff2d 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -318,8 +318,7 @@ public class CatalogUtil {
}
/**
- * This method transforms the unqualified names of a given schema into
- * the qualified names.
+ * This method transforms the unqualified names of a schema to the qualified names.
*
* @param tableName a table name to be prefixed
* @param schema a schema to be transformed
@@ -327,15 +326,9 @@ public class CatalogUtil {
* @return
*/
public static SchemaProto getQualfiedSchema(String tableName, SchemaProto schema) {
- SchemaProto.Builder revisedSchema = SchemaProto.newBuilder(schema);
- revisedSchema.clearFields();
- for (ColumnProto col : schema.getFieldsList()) {
- ColumnProto.Builder builder = ColumnProto.newBuilder(col);
- builder.setName(tableName + CatalogConstants.IDENTIFIER_DELIMITER + extractSimpleName(col.getName()));
- revisedSchema.addFields(builder.build());
- }
-
- return revisedSchema.build();
+ Schema restored = new Schema(schema);
+ restored.setQualifier(tableName);
+ return restored.getProto();
}
public static DataType newDataType(Type type, String code) {
@@ -354,6 +347,19 @@ public class CatalogUtil {
return DataType.newBuilder().setType(type).build();
}
+ /**
+ * Create a record type
+ *
+ * @param nestedFieldNum The number of nested fields
+ * @return RECORD DataType
+ */
+ public static DataType newRecordType(int nestedFieldNum) {
+ DataType.Builder builder = DataType.newBuilder();
+ builder.setType(Type.RECORD);
+ builder.setNumNestedFields(nestedFieldNum);
+ return builder.build();
+ }
+
public static DataType [] newSimpleDataTypeArray(Type... types) {
DataType [] dataTypes = new DataType[types.length];
for (int i = 0; i < types.length; i++) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
index aceb6f1..12edaa7 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
@@ -32,7 +32,18 @@ import org.apache.tajo.json.GsonObject;
*/
public class Column implements ProtoObject<ColumnProto>, GsonObject {
@Expose protected String name;
- @Expose protected DataType dataType;
+ @Expose protected TypeDesc typeDesc;
+
+ /**
+ * Column Constructor
+ *
+ * @param name field name
+ * @param typeDesc Type description
+ */
+ public Column(String name, TypeDesc typeDesc) {
+ this.name = name;
+ this.typeDesc = typeDesc;
+ }
/**
*
@@ -40,8 +51,7 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
* @param dataType Data Type with length
*/
public Column(String name, DataType dataType) {
- this.name = name;
- this.dataType = dataType;
+ this(name, new TypeDesc(dataType));
}
/**
@@ -65,7 +75,7 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
public Column(ColumnProto proto) {
name = proto.getName();
- dataType = proto.getDataType();
+ typeDesc = new TypeDesc(proto.getDataType());
}
/**
@@ -101,24 +111,33 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
}
/**
+ * Return type description
+ *
+ * @return TypeDesc
+ */
+ public TypeDesc getTypeDesc() {
+ return this.typeDesc;
+ }
+
+ /**
*
* @return DataType which includes domain type and scale.
*/
public DataType getDataType() {
- return this.dataType;
+ return this.typeDesc.dataType;
}
@Override
public boolean equals(Object o) {
if (o instanceof Column) {
Column another = (Column)o;
- return name.equals(another.name) && dataType.equals(another.dataType);
+ return name.equals(another.name) && typeDesc.equals(another.typeDesc);
}
return false;
}
public int hashCode() {
- return Objects.hashCode(name, dataType);
+ return Objects.hashCode(name, typeDesc);
}
@@ -128,16 +147,16 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
*/
@Override
public ColumnProto getProto() {
- return ColumnProto.newBuilder().setName(this.name).setDataType(this.dataType).build();
+ ColumnProto.Builder builder = ColumnProto.newBuilder();
+ builder
+ .setName(this.name)
+ .setDataType(this.typeDesc.getDataType());
+ return builder.build();
}
public String toString() {
StringBuilder sb = new StringBuilder(getQualifiedName());
- sb.append(" (").append(getDataType().getType());
- if (getDataType().getLength() > 0) {
- sb.append("(" + getDataType().getLength() + ")");
- }
- sb.append(")");
+ sb.append(" (").append(typeDesc.toString()).append(")");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index 978092a..1a59e88 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -69,7 +69,7 @@ public class DDLBuilder {
return sb.toString();
}
- private static void buildSchema(StringBuilder sb, Schema schema) {
+ public static void buildSchema(StringBuilder sb, Schema schema) {
boolean first = true;
sb.append(" (");
@@ -81,11 +81,8 @@ public class DDLBuilder {
}
sb.append(CatalogUtil.denormalizeIdentifier(column.getSimpleName())).append(" ");
- TajoDataTypes.DataType dataType = column.getDataType();
- sb.append(dataType.getType().name());
- if (column.getDataType().hasLength() && column.getDataType().getLength() > 0) {
- sb.append(" (").append(column.getDataType().getLength()).append(")");
- }
+ TypeDesc typeDesc = column.getTypeDesc();
+ sb.append(typeDesc);
}
sb.append(")");
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index 71c1b01..ed2cd2c 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
import com.google.gson.annotations.Expose;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.catalog.SchemaUtil.ColumnVisitor;
import org.apache.tajo.catalog.exception.AlreadyExistsFieldException;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
@@ -45,36 +46,77 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
public Schema() {
init();
}
-
+
+ /**
+ * This Schema constructor restores a serialized schema into in-memory Schema structure.
+ * A serialized schema is an ordered list in depth-first order over a nested schema.
+ * This constructor transforms the list into a tree-like structure.
+ *
+ * @param proto
+ */
public Schema(SchemaProto proto) {
- this.fields = new ArrayList<Column>();
- this.fieldsByQualifiedName = new HashMap<String, Integer>();
- this.fieldsByName = new HashMap<String, List<Integer>>();
- for(ColumnProto colProto : proto.getFieldsList()) {
- Column tobeAdded = new Column(colProto);
- fields.add(tobeAdded);
- if (tobeAdded.hasQualifier()) {
- fieldsByQualifiedName.put(tobeAdded.getQualifier() + "." + tobeAdded.getSimpleName(), fields.size() - 1);
- } else {
- fieldsByQualifiedName.put(tobeAdded.getSimpleName(), fields.size() - 1);
- }
- if (fieldsByName.containsKey(tobeAdded.getSimpleName())) {
- fieldsByName.get(tobeAdded.getSimpleName()).add(fields.size() - 1);
- } else {
- fieldsByName.put(tobeAdded.getSimpleName(), TUtil.newList(fields.size() - 1));
+ init();
+
+ List<Column> toBeAdded = TUtil.newList();
+ for (int i = 0; i < proto.getFieldsCount(); i++) {
+ deserializeColumn(toBeAdded, proto.getFieldsList(), i);
+ }
+
+ for (Column c : toBeAdded) {
+ addColumn(c);
+ }
+ }
+
+ /**
+ * This method transforms a list of ColumnProtos into a schema tree.
+ * It assumes that <code>protos</code> contains a list of ColumnProtos in the depth-first order.
+ *
+ * @param tobeAdded
+ * @param protos
+ * @param serializedColumnIndex
+ */
+ private static void deserializeColumn(List<Column> tobeAdded, List<ColumnProto> protos, int serializedColumnIndex) {
+ ColumnProto columnProto = protos.get(serializedColumnIndex);
+ if (columnProto.getDataType().getType() == Type.RECORD) {
+
+ // Get the number of child fields
+ int childNum = columnProto.getDataType().getNumNestedFields();
+ // where is start index of nested fields?
+ int childStartIndex = tobeAdded.size() - childNum;
+ // Extract nested fields
+ List<Column> nestedColumns = TUtil.newList(tobeAdded.subList(childStartIndex, childStartIndex + childNum));
+
+ // Remove nested fields from the the current level
+ for (int i = 0; i < childNum; i++) {
+ tobeAdded.remove(tobeAdded.size() - 1);
}
+
+ // Add the nested fields to the list as a single record column
+ tobeAdded.add(new Column(columnProto.getName(), new TypeDesc(new Schema(nestedColumns))));
+ } else {
+ tobeAdded.add(new Column(protos.get(serializedColumnIndex)));
}
}
public Schema(Schema schema) {
this();
+
this.fields.addAll(schema.fields);
this.fieldsByQualifiedName.putAll(schema.fieldsByQualifiedName);
this.fieldsByName.putAll(schema.fieldsByName);
}
- public Schema(Column [] columns) {
+ public Schema(Column [] columns) {
init();
+
+ for(Column c : columns) {
+ addColumn(c);
+ }
+ }
+
+ public Schema(Iterable<Column> columns) {
+ init();
+
for(Column c : columns) {
addColumn(c);
}
@@ -93,21 +135,15 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
* @param qualifier The qualifier
*/
public void setQualifier(String qualifier) {
- Schema copy = null;
- try {
- copy = (Schema) clone();
- } catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
+ List<Column> columns = getColumns();
fields.clear();
fieldsByQualifiedName.clear();
fieldsByName.clear();
Column newColumn;
- for (int i = 0; i < copy.size(); i++) {
- Column column = copy.getColumn(i);
- newColumn = new Column(qualifier + "." + column.getSimpleName(), column.getDataType());
+ for (Column c : columns) {
+ newColumn = new Column(qualifier + "." + c.getSimpleName(), c.typeDesc);
addColumn(newColumn);
}
}
@@ -275,6 +311,21 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
return fields.containsAll(columns);
}
+ public synchronized Schema addColumn(String name, TypeDesc typeDesc) {
+ String normalized = name;
+ if(fieldsByQualifiedName.containsKey(normalized)) {
+ LOG.error("Already exists column " + normalized);
+ throw new AlreadyExistsFieldException(normalized);
+ }
+
+ Column newCol = new Column(normalized, typeDesc);
+ fields.add(newCol);
+ fieldsByQualifiedName.put(newCol.getQualifiedName(), fields.size() - 1);
+ fieldsByName.put(newCol.getSimpleName(), TUtil.newList(fields.size() - 1));
+
+ return this;
+ }
+
public synchronized Schema addColumn(String name, Type type) {
if (type == Type.CHAR) {
return addColumn(name, CatalogUtil.newDataTypeWithLen(type, 1));
@@ -287,22 +338,13 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public synchronized Schema addColumn(String name, DataType dataType) {
- String normalized = name;
- if(fieldsByQualifiedName.containsKey(normalized)) {
- LOG.error("Already exists column " + normalized);
- throw new AlreadyExistsFieldException(normalized);
- }
-
- Column newCol = new Column(normalized, dataType);
- fields.add(newCol);
- fieldsByQualifiedName.put(newCol.getQualifiedName(), fields.size() - 1);
- fieldsByName.put(newCol.getSimpleName(), TUtil.newList(fields.size() - 1));
-
+ addColumn(name, new TypeDesc(dataType));
+
return this;
}
public synchronized void addColumn(Column column) {
- addColumn(column.getQualifiedName(), column.getDataType());
+ addColumn(column.getQualifiedName(), column.typeDesc);
}
public synchronized void addColumns(Schema schema) {
@@ -327,10 +369,9 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
@Override
public Object clone() throws CloneNotSupportedException {
- Schema schema = null;
-
- schema = (Schema) super.clone();
+ Schema schema = (Schema) super.clone();
schema.init();
+
for(Column column: this.fields) {
schema.addColumn(column);
}
@@ -340,15 +381,34 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
@Override
public SchemaProto getProto() {
SchemaProto.Builder builder = SchemaProto.newBuilder();
- builder.clearFields();
- if (this.fields != null) {
- for(Column col : fields) {
- builder.addFields(col.getProto());
- }
- }
+ SchemaProtoBuilder recursiveBuilder = new SchemaProtoBuilder(builder);
+ SchemaUtil.visitSchema(this, recursiveBuilder);
return builder.build();
}
+ private static class SchemaProtoBuilder implements ColumnVisitor {
+ private SchemaProto.Builder builder;
+ public SchemaProtoBuilder(SchemaProto.Builder builder) {
+ this.builder = builder;
+ }
+
+ @Override
+ public void visit(int depth, Column column) {
+
+ if (column.getDataType().getType() == Type.RECORD) {
+ DataType.Builder updatedType = DataType.newBuilder(column.getDataType());
+ updatedType.setNumNestedFields(column.typeDesc.nestedRecordSchema.size());
+
+ ColumnProto.Builder updatedColumn = ColumnProto.newBuilder(column.getProto());
+ updatedColumn.setDataType(updatedType);
+
+ builder.addFields(updatedColumn.build());
+ } else {
+ builder.addFields(column.getProto());
+ }
+ }
+ }
+
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{(").append(size()).append(") ");
@@ -356,7 +416,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
for(Column col : fields) {
sb.append(col);
if (i < fields.size() - 1) {
- sb.append(",");
+ sb.append(", ");
}
i++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
index 23ebe1b..f2bb71c 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
@@ -108,4 +108,46 @@ public class SchemaUtil {
}
return names;
}
+
+ /**
+ * Column visitor interface
+ */
+ public static interface ColumnVisitor {
+ public void visit(int depth, Column column);
+ }
+
+ /**
+ * It allows a column visitor to traverse all columns in a schema in a depth-first order.
+ * @param schema
+ * @param function
+ */
+ public static void visitSchema(Schema schema, ColumnVisitor function) {
+ for(Column col : schema.getColumns()) {
+ visitInDepthFirstOrder(0, function, col);
+ }
+ }
+
+ /**
+ * A recursive function to traverse all columns in a schema in a depth-first order.
+ *
+ * @param depth Nested depth. 0 is root column.
+ * @param function Visitor
+ * @param column Current visiting column
+ */
+ private static void visitInDepthFirstOrder(int depth, ColumnVisitor function, Column column) {
+ if (column.getDataType().getType() == Type.RECORD) {
+ for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getColumns()) {
+ visitInDepthFirstOrder(depth + 1, function, nestedColumn);
+ }
+ function.visit(depth, column);
+ } else {
+ function.visit(depth, column);
+ }
+ }
+
+ public static String toDisplayString(Schema schema) {
+ StringBuilder sb = new StringBuilder();
+ DDLBuilder.buildSchema(sb, schema);
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeDesc.java
new file mode 100644
index 0000000..3bd0f00
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeDesc.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.catalog;
+
+import com.google.common.base.Objects;
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.common.TajoDataTypes.DataType;
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.util.TUtil;
+
+/**
+ * Type Description for a column
+ */
+public class TypeDesc {
+ @Expose protected DataType dataType;
+ @Expose protected Schema nestedRecordSchema; // NULL unless type is RECORD.
+
+ public TypeDesc(DataType dataType) {
+ this.dataType = dataType;
+ }
+
+ public TypeDesc(Schema recordSchema) {
+ this.dataType = CatalogUtil.newSimpleDataType(Type.RECORD);
+ this.nestedRecordSchema = recordSchema;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj instanceof TypeDesc) {
+ TypeDesc other = (TypeDesc) obj;
+ return this.dataType.equals(other.dataType) &&
+ TUtil.checkEquals(nestedRecordSchema, other.nestedRecordSchema);
+
+ } else {
+ return false;
+ }
+ }
+
+ public int hashCode() {
+ return Objects.hashCode(dataType.hashCode(), nestedRecordSchema);
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ if (dataType.getType() == Type.RECORD) {
+ sb.append("RECORD").append(SchemaUtil.toDisplayString(nestedRecordSchema)).append("");
+ } else {
+ sb.append(dataType.getType().name());
+ if (dataType.getLength() > 0) {
+ sb.append("(" + dataType.getLength() + ")");
+ }
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java
index a61b422..edd0f3e 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java
@@ -34,6 +34,87 @@ public class TestSchema {
Column col2;
Column col3;
+ public static final Schema nestedSchema1;
+ public static final Schema nestedSchema2;
+ public static final Schema nestedSchema3;
+
+ static {
+ // simple nested schema
+ nestedSchema1 = new Schema();
+ nestedSchema1.addColumn("s1", Type.INT8);
+
+ Schema nestedRecordSchema = new Schema();
+ nestedRecordSchema.addColumn("s2", Type.FLOAT4);
+ nestedRecordSchema.addColumn("s3", Type.TEXT);
+
+ Column nestedField = new Column("s4", new TypeDesc(nestedRecordSchema));
+ nestedSchema1.addColumn(nestedField);
+
+ nestedSchema1.addColumn("s5", Type.FLOAT8);
+
+ // two level nested schema
+ //
+ // s1
+ // |- s2
+ // |- s4
+ // |- s3
+ // |- s4
+ // |- s5
+ // |- s8
+ // |- s6
+ // |- s7
+ nestedSchema2 = new Schema();
+ nestedSchema2.addColumn("s1", Type.INT8);
+
+ Schema nestedRecordSchema1 = new Schema();
+ nestedRecordSchema1.addColumn("s2", Type.FLOAT4);
+ nestedRecordSchema1.addColumn("s3", Type.TEXT);
+
+ Column nestedField1 = new Column("s4", new TypeDesc(nestedRecordSchema1));
+ nestedSchema2.addColumn(nestedField1);
+
+ nestedSchema2.addColumn("s5", Type.FLOAT8);
+
+ Schema nestedRecordSchema2 = new Schema();
+ nestedRecordSchema2.addColumn("s6", Type.FLOAT4);
+ nestedRecordSchema2.addColumn("s7", Type.TEXT);
+
+ Column nestedField2 = new Column("s8", new TypeDesc(nestedRecordSchema2));
+ nestedSchema2.addColumn(nestedField2);
+
+
+ // three level nested schema
+ //
+ // s1
+ // |- s2
+ // |- s3
+ // |- s4
+ // |- s7
+ // |- s5
+ // |- s6
+ // |- s8
+ // |- s9
+
+ nestedSchema3 = new Schema();
+ nestedSchema3.addColumn("s1", Type.INT8);
+
+ nestedSchema3.addColumn("s2", Type.INT8);
+
+ Schema s5 = new Schema();
+ s5.addColumn("s6", Type.INT8);
+
+ Schema s7 = new Schema();
+ s7.addColumn("s5", new TypeDesc(s5));
+
+ Schema s3 = new Schema();
+ s3.addColumn("s4", Type.INT8);
+ s3.addColumn("s7", new TypeDesc(s7));
+ s3.addColumn("s8", Type.INT8);
+
+ nestedSchema3.addColumn("s3", new TypeDesc(s3));
+ nestedSchema3.addColumn("s9", Type.INT8);
+ }
+
@Before
public void setUp() throws Exception {
schema = new Schema();
@@ -167,6 +248,62 @@ public class TestSchema {
assertEquals(schema3.getColumn(0), schema3.getColumn("tb2.col1"));
assertEquals(schema3.getColumn(1), schema3.getColumn("col2"));
assertEquals(schema3.getColumn(1), schema3.getColumn("tb2.col2"));
+ }
+
+ @Test
+ public void testNestedRecord1() {
+ verifySchema(nestedSchema1);
+ }
+
+ @Test
+ public void testNestedRecord2() {
+ verifySchema(nestedSchema2);
+ }
+
+ @Test
+ public void testNestedRecord3() {
+ verifySchema(nestedSchema3);
+ }
+
+ @Test
+ public void testNestedRecord4() {
+ Schema root = new Schema();
+
+ Schema nf2DotNf1 = new Schema();
+ nf2DotNf1.addColumn("f1", Type.INT8);
+ nf2DotNf1.addColumn("f2", Type.INT8);
+
+ Schema nf2DotNf2 = new Schema();
+ nf2DotNf2.addColumn("f1", Type.INT8);
+ nf2DotNf2.addColumn("f2", Type.INT8);
+
+ Schema nf2 = new Schema();
+ nf2.addColumn("f1", Type.INT8);
+ nf2.addColumn("nf1", new TypeDesc(nf2DotNf1));
+ nf2.addColumn("nf2", new TypeDesc(nf2DotNf2));
+ nf2.addColumn("f2", Type.INT8);
+
+ root.addColumn("f1", Type.INT8);
+ root.addColumn("nf1", Type.INT8);
+ root.addColumn("nf2", new TypeDesc(nf2));
+ root.addColumn("f2", Type.INT8);
+
+ verifySchema(root);
+ }
+
+ public static void verifySchema(Schema s1) {
+ assertEquals(s1, s1);
+
+ SchemaProto proto = s1.getProto();
+ assertEquals("Proto (de)serialized schema is different from the original: ", s1, new Schema(proto));
+
+ Schema cloned = null;
+ try {
+ cloned = (Schema) s1.clone();
+ } catch (CloneNotSupportedException e) {
+ fail("Clone is failed");
+ }
+ assertEquals("Cloned schema is different from the original one:", s1, cloned);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java
index 85b8f20..9ab65ec 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ColumnsTableDescriptor.java
@@ -24,9 +24,10 @@ class ColumnsTableDescriptor extends AbstractTableDescriptor {
private static final String TABLENAME = "columns";
private final ColumnDescriptor[] columns = new ColumnDescriptor[] {
- new ColumnDescriptor("tid", Type.INT4, 0),
- new ColumnDescriptor("column_name", Type.TEXT, 0),
- new ColumnDescriptor("ordinal_position", Type.INT4, 0),
+ new ColumnDescriptor("tid", Type.INT4, 0), // just key for DBMS
+ new ColumnDescriptor("column_name", Type.TEXT, 0), // column name
+ new ColumnDescriptor("ordinal_position", Type.INT4, 0), // the ordinal position in a schema
+ new ColumnDescriptor("nested_field_num", Type.INT4, 0), // the number of child nested fields
new ColumnDescriptor("data_type", Type.TEXT, 0),
new ColumnDescriptor("type_length", Type.INT4, 0)
};
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index 5c78f82..be6bf1c 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -33,6 +33,7 @@ import org.apache.tajo.catalog.FunctionDesc;
import org.apache.tajo.catalog.exception.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.*;
+import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.exception.InternalException;
import org.apache.tajo.exception.UnimplementedException;
@@ -800,7 +801,10 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt.close();
String colSql =
- "INSERT INTO " + TB_COLUMNS + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) ";
+ "INSERT INTO " + TB_COLUMNS +
+ // 1 2 3 4 5 6
+ " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH)" +
+ " VALUES(?, ?, ?, ?, ?, ?) ";
if (LOG.isDebugEnabled()) {
LOG.debug(colSql);
@@ -809,11 +813,15 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt = conn.prepareStatement(colSql);
for (int i = 0; i < table.getSchema().getFieldsCount(); i++) {
ColumnProto col = table.getSchema().getFields(i);
+ TajoDataTypes.DataType dataType = col.getDataType();
+
pstmt.setInt(1, tableId);
pstmt.setString(2, CatalogUtil.extractSimpleName(col.getName()));
pstmt.setInt(3, i);
- pstmt.setString(4, col.getDataType().getType().name());
- pstmt.setInt(5, (col.getDataType().hasLength() ? col.getDataType().getLength() : 0));
+ // the default number of nested fields is 0.
+ pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0);
+ pstmt.setString(5, dataType.getType().name());
+ pstmt.setInt(6, (col.getDataType().hasLength() ? col.getDataType().getLength() : 0));
pstmt.addBatch();
pstmt.clearParameters();
}
@@ -1028,13 +1036,13 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
throws CatalogException {
final String selectColumnSql =
- "SELECT COLUMN_NAME, DATA_TYPE, TYPE_LENGTH, ORDINAL_POSITION from " + TB_COLUMNS +
+ "SELECT COLUMN_NAME, DATA_TYPE, TYPE_LENGTH, ORDINAL_POSITION, NESTED_FIELD_NUM from " + TB_COLUMNS +
" WHERE " + COL_TABLES_PK + " = ?" + " AND COLUMN_NAME = ?" ;
final String deleteColumnNameSql =
"DELETE FROM " + TB_COLUMNS + " WHERE TID = ? AND COLUMN_NAME = ?";
final String insertNewColumnSql =
"INSERT INTO " + TB_COLUMNS +
- " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) ";
+ " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?, ?) ";
if (LOG.isDebugEnabled()) {
LOG.debug(selectColumnSql);
@@ -1058,13 +1066,15 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
resultSet = pstmt.executeQuery();
CatalogProtos.ColumnProto columnProto = null;
- int ordinalPosition = -1;
+ int ordinalPosition = 0;
+ int nestedFieldNum = 0;
if (resultSet.next()) {
columnProto = resultToColumnProto(resultSet);
//NOTE ==> Setting new column Name
columnProto = columnProto.toBuilder().setName(alterColumnProto.getNewColumnName()).build();
ordinalPosition = resultSet.getInt("ORDINAL_POSITION");
+ nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM");
} else {
throw new NoSuchColumnException(alterColumnProto.getOldColumnName());
}
@@ -1085,8 +1095,9 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt.setInt(1, tableId);
pstmt.setString(2, CatalogUtil.extractSimpleName(columnProto.getName()));
pstmt.setInt(3, ordinalPosition);
- pstmt.setString(4, columnProto.getDataType().getType().name());
- pstmt.setInt(5, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0));
+ pstmt.setInt(4, nestedFieldNum);
+ pstmt.setString(5, columnProto.getDataType().getType().name());
+ pstmt.setInt(6, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0));
pstmt.executeUpdate();
conn.commit();
@@ -1101,8 +1112,11 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) throws CatalogException {
- final String insertNewColumnSql = "INSERT INTO " + TB_COLUMNS + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) ";
- final String columnCountSql = "SELECT COLUMN_NAME, MAX(ORDINAL_POSITION) AS POSITION FROM " + TB_COLUMNS + " WHERE TID = ? GROUP BY COLUMN_NAME";
+ final String insertNewColumnSql =
+ "INSERT INTO " + TB_COLUMNS +
+ " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?, ?) ";
+ final String columnCountSql =
+ "SELECT MAX(ORDINAL_POSITION) AS POSITION FROM " + TB_COLUMNS + " WHERE TID = ?";
if (LOG.isDebugEnabled()) {
LOG.debug(insertNewColumnSql);
@@ -1119,18 +1133,22 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt.setInt(1 , tableId);
resultSet = pstmt.executeQuery();
+ // get the last the ordinal position.
int position = resultSet.next() ? resultSet.getInt("POSITION") : 0;
resultSet.close();
pstmt.close();
resultSet = null;
+ TajoDataTypes.DataType dataType = columnProto.getDataType();
+
pstmt = conn.prepareStatement(insertNewColumnSql);
pstmt.setInt(1, tableId);
pstmt.setString(2, CatalogUtil.extractSimpleName(columnProto.getName()));
pstmt.setInt(3, position + 1);
- pstmt.setString(4, columnProto.getDataType().getType().name());
- pstmt.setInt(5, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0));
+ pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0);
+ pstmt.setString(5, dataType.getType().name());
+ pstmt.setInt(6, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0));
pstmt.executeUpdate();
} catch (SQLException sqlException) {
@@ -1385,7 +1403,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
// Geting Column Descriptions
//////////////////////////////////////////
CatalogProtos.SchemaProto.Builder schemaBuilder = CatalogProtos.SchemaProto.newBuilder();
- sql = "SELECT COLUMN_NAME, DATA_TYPE, TYPE_LENGTH from " + TB_COLUMNS +
+ sql = "SELECT COLUMN_NAME, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH from " + TB_COLUMNS +
" WHERE " + COL_TABLES_PK + " = ? ORDER BY ORDINAL_POSITION ASC";
if (LOG.isDebugEnabled()) {
@@ -1643,7 +1661,8 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
List<ColumnProto> columns = new ArrayList<ColumnProto>();
try {
- String sql = "SELECT TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH FROM " + TB_COLUMNS +
+ String sql =
+ "SELECT TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH FROM " + TB_COLUMNS +
" ORDER BY TID ASC, ORDINAL_POSITION ASC";
conn = getConnection();
@@ -1654,11 +1673,15 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
builder.setTid(resultSet.getInt("TID"));
builder.setName(resultSet.getString("COLUMN_NAME"));
-
+
+ int nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM");
+
Type type = getDataType(resultSet.getString("DATA_TYPE").trim());
int typeLength = resultSet.getInt("TYPE_LENGTH");
-
- if (typeLength > 0) {
+
+ if (nestedFieldNum > 0) {
+ builder.setDataType(CatalogUtil.newRecordType(nestedFieldNum));
+ } else if (typeLength > 0) {
builder.setDataType(CatalogUtil.newDataTypeWithLen(type, typeLength));
} else {
builder.setDataType(CatalogUtil.newSimpleDataType(type));
@@ -2309,9 +2332,14 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
ColumnProto.Builder builder = ColumnProto.newBuilder();
builder.setName(res.getString("column_name").trim());
+ int nestedFieldNum = res.getInt("NESTED_FIELD_NUM");
+
Type type = getDataType(res.getString("data_type").trim());
int typeLength = res.getInt("type_length");
- if (typeLength > 0) {
+
+ if (nestedFieldNum > 0) {
+ builder.setDataType(CatalogUtil.newRecordType(nestedFieldNum));
+ } else if (typeLength > 0) {
builder.setDataType(CatalogUtil.newDataTypeWithLen(type, typeLength));
} else {
builder.setDataType(CatalogUtil.newSimpleDataType(type));
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java
index 8cb3858..0159b31 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MariaDBStore.java
@@ -26,8 +26,9 @@ import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.exception.InternalException;
public class MariaDBStore extends AbstractMySQLMariaDBStore {
- /** 2014-06-09: First versioning */
- private static final int MARIADB_CATALOG_STORE_VERSION = 2;
+ /** 3 - 2015-03-12: Nested Schema (TAJO-1329) */
+ /** 2 - First versioning */
+ private static final int MARIADB_CATALOG_STORE_VERSION = 3;
private static final String CATALOG_DRIVER = "org.mariadb.jdbc.Driver";
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java
index cedc0fe..f7f1b1b 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MySQLStore.java
@@ -26,10 +26,11 @@ import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.exception.InternalException;
public class MySQLStore extends AbstractMySQLMariaDBStore {
- /** 2014-03-20: First versioning */
- private static final int MYSQL_CATALOG_STORE_VERSION_2 = 2;
- /** Before 2013-03-20 */
- private static final int MYSQL_CATALOG_STORE_VERSION_1 = 1;
+
+ /** 3 - 2015-03-12: Nested Schema (TAJO-1329) */
+ /** 2 - 2014-06-09: First versioning */
+ /** 1- Before 2013-03-20 */
+ private static final int MYSQL_CATALOG_STORE_VERSION = 3;
private static final String CATALOG_DRIVER = "com.mysql.jdbc.Driver";
@Override
@@ -43,7 +44,7 @@ public class MySQLStore extends AbstractMySQLMariaDBStore {
@Override
public int getDriverVersion() {
- return MYSQL_CATALOG_STORE_VERSION_2;
+ return MYSQL_CATALOG_STORE_VERSION;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java
index 45c153c..4b7e6a3 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/OracleStore.java
@@ -27,7 +27,7 @@ import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.exception.InternalException;
public class OracleStore extends AbstractDBStore {
-
+
private static final String CATALOG_DRIVER = "oracle.jdbc.OracleDriver";
public OracleStore(Configuration conf) throws InternalException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java
index 41f2909..6089fdd 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/PostgreSQLStore.java
@@ -27,7 +27,7 @@ import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.exception.InternalException;
public class PostgreSQLStore extends AbstractDBStore {
-
+
private static final String CATALOG_DRIVER = "org.postgresql.Driver";
public PostgreSQLStore(Configuration conf) throws InternalException {
@@ -48,7 +48,7 @@ public class PostgreSQLStore extends AbstractDBStore {
protected Connection createConnection(Configuration conf) throws SQLException {
return DriverManager.getConnection(getCatalogUri(), this.connectionId, this.connectionPassword);
}
-
+
@Override
protected void createDatabaseDependants() throws CatalogException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
index db2473b..a0bd9cd 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
@@ -17,7 +17,12 @@
limitations under the License.
-->
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
- <tns:base version="2">
+
+ <!-- Version History -->
+ <!-- 3 - 2015-03-12: Nested Schema (TAJO-1329) -->
+ <!-- 2 - First versioning -->
+
+ <tns:base version="3">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -72,14 +77,15 @@
TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE,
COLUMN_NAME VARCHAR(128) NOT NULL,
ORDINAL_POSITION INTEGER NOT NULL,
+ NESTED_FIELD_NUM INTEGER NOT NULL,
DATA_TYPE CHAR(16),
TYPE_LENGTH INTEGER,
- CONSTRAINT COLUMNS_PK PRIMARY KEY (TID, COLUMN_NAME)
+ CONSTRAINT COLUMNS_PK PRIMARY KEY (TID, ORDINAL_POSITION)
)]]>
</tns:sql>
</tns:Object>
<tns:Object order="8" type="index" name="IDX_FK_COLUMNS_TABLE_NAME" dependsOn="COLUMNS">
- <tns:sql><![CDATA[CREATE UNIQUE INDEX idx_fk_columns_table_name on COLUMNS (TID, COLUMN_NAME)]]></tns:sql>
+ <tns:sql><![CDATA[CREATE UNIQUE INDEX idx_fk_columns_table_name on COLUMNS (TID, ORDINAL_POSITION)]]></tns:sql>
</tns:Object>
<tns:Object order="9" type="table" name="OPTIONS">
<tns:sql><![CDATA[
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql
index aec1553..5b8ed26 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/columns.sql
@@ -2,8 +2,9 @@ CREATE TABLE COLUMNS (
TID INT NOT NULL,
COLUMN_NAME VARCHAR(255) BINARY NOT NULL,
ORDINAL_POSITION INT NOT NULL,
+ NESTED_FIELD_NUM INT NOT NULL,
DATA_TYPE CHAR(16),
TYPE_LENGTH INTEGER,
- PRIMARY KEY (TID, COLUMN_NAME),
+ PRIMARY KEY (TID, ORDINAL_POSITION),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql
index aec1553..5b8ed26 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/columns.sql
@@ -2,8 +2,9 @@ CREATE TABLE COLUMNS (
TID INT NOT NULL,
COLUMN_NAME VARCHAR(255) BINARY NOT NULL,
ORDINAL_POSITION INT NOT NULL,
+ NESTED_FIELD_NUM INT NOT NULL,
DATA_TYPE CHAR(16),
TYPE_LENGTH INTEGER,
- PRIMARY KEY (TID, COLUMN_NAME),
+ PRIMARY KEY (TID, ORDINAL_POSITION),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
index 8945fca..880a14e 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
@@ -17,7 +17,12 @@
limitations under the License.
-->
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
- <tns:base version="2">
+
+ <!-- Version History -->
+ <!-- 3 - 2015-03-12: Nested Schema (TAJO-1329) -->
+ <!-- 2 - First versioning -->
+
+ <tns:base version="3">
<tns:objects>
<tns:Object order="0" type="table" name="meta">
<tns:sql><![CDATA[
@@ -117,9 +122,10 @@
TID INT NOT NULL,
COLUMN_NAME VARCHAR2(255) NOT NULL,
ORDINAL_POSITION INT NOT NULL,
+ NESTED_FIELD_NUM INT NOT NULL,
DATA_TYPE CHAR(16),
TYPE_LENGTH INTEGER,
- CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, COLUMN_NAME),
+ CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, ORDINAL_POSITION),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)]]>
</tns:sql>
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
index 8e5cbcc..821527b 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
@@ -19,7 +19,12 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
- <tns:base version="2">
+
+ <!-- Version History -->
+ <!-- 3 - 2015-03-12: Nested Schema (TAJO-1329) -->
+ <!-- 2 - First versioning -->
+
+ <tns:base version="3">
<tns:objects>
<tns:Object name="META" type="table" order="0">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -76,9 +81,10 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.
TID INT NOT NULL,
COLUMN_NAME VARCHAR(255) NOT NULL,
ORDINAL_POSITION INT NOT NULL,
+ NESTED_FIELD_NUM INT NOT NULL,
DATA_TYPE CHAR(16),
TYPE_LENGTH INTEGER,
- CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, COLUMN_NAME),
+ CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, ORDINAL_POSITION),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)]]>
</tns:sql>
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 305742f..c3bfc99 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -431,6 +431,107 @@ public class TestCatalog {
assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable"));
}
+ /**
+ * It asserts the equality between an original table desc and a restored table desc.
+ */
+ private static void assertSchemaEquality(String tableName, Schema schema) throws IOException {
+ Path path = new Path(CommonTestingUtil.getTestDir(), tableName);
+ TableDesc tableDesc = new TableDesc(
+ CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName),
+ schema,
+ StoreType.CSV,
+ new KeyValueSet(),
+ path.toUri());
+
+ // schema creation
+ assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
+ catalog.createTable(tableDesc);
+ assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
+
+ // change it for the equals test.
+ schema.setQualifier(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
+ TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
+ assertEquals(schema, restored.getSchema());
+
+ // drop test
+ catalog.dropTable(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
+ assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
+ }
+
+ @Test
+ public void testCreateAndGetNestedTable1() throws Exception {
+ // schema creation
+ // three level nested schema
+ //
+ // s1
+ // |- s2
+ // |- s3
+ // |- s4
+ // |- s7
+ // |- s5
+ // |- s6
+ // |- s8
+ // |- s9
+
+ Schema nestedSchema = new Schema();
+ nestedSchema.addColumn("s1", Type.INT8);
+
+ nestedSchema.addColumn("s2", Type.INT8);
+
+ Schema s5 = new Schema();
+ s5.addColumn("s6", Type.INT8);
+
+ Schema s7 = new Schema();
+ s7.addColumn("s5", new TypeDesc(s5));
+
+ Schema s3 = new Schema();
+ s3.addColumn("s4", Type.INT8);
+ s3.addColumn("s7", new TypeDesc(s7));
+ s3.addColumn("s8", Type.INT8);
+
+ nestedSchema.addColumn("s3", new TypeDesc(s3));
+ nestedSchema.addColumn("s9", Type.INT8);
+
+ assertSchemaEquality("nested_schema1", nestedSchema);
+ }
+
+ @Test
+ public void testCreateAndGetNestedTable2() throws Exception {
+ // schema creation
+ // three level nested schema
+ //
+ // s1
+ // |- s2
+ // |- s3
+ // |- s1
+ // |- s2
+ // |- s3
+ // |- s1
+ // |- s3
+ // |- s4
+
+ Schema nestedSchema = new Schema();
+ nestedSchema.addColumn("s1", Type.INT8);
+
+ nestedSchema.addColumn("s2", Type.INT8);
+
+ Schema s5 = new Schema();
+ s5.addColumn("s6", Type.INT8);
+
+ Schema s7 = new Schema();
+ s7.addColumn("s5", new TypeDesc(s5));
+
+ Schema s3 = new Schema();
+ s3.addColumn("s4", Type.INT8);
+ s3.addColumn("s7", new TypeDesc(s7));
+ s3.addColumn("s8", Type.INT8);
+
+ nestedSchema.addColumn("s3", new TypeDesc(s3));
+ nestedSchema.addColumn("s9", Type.INT8);
+
+ assertSchemaEquality("nested_schema2", nestedSchema);
+ }
+
static IndexDesc desc1;
static IndexDesc desc2;
static IndexDesc desc3;
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index b7d9334..a3960e6 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@ -114,10 +114,7 @@ public class DescTableCommand extends TajoShellCommand {
for(int i = 0; i < desc.getSchema().size(); i++) {
Column col = desc.getSchema().getColumn(i);
- sb.append(col.getSimpleName()).append("\t").append(col.getDataType().getType());
- if (col.getDataType().hasLength()) {
- sb.append("(").append(col.getDataType().getLength()).append(")");
- }
+ sb.append(col.getSimpleName()).append("\t").append(col.getTypeDesc());
sb.append("\n");
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-common/src/main/proto/DataTypes.proto
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/proto/DataTypes.proto b/tajo-common/src/main/proto/DataTypes.proto
index 04f1e12..fc5ac9a 100644
--- a/tajo-common/src/main/proto/DataTypes.proto
+++ b/tajo-common/src/main/proto/DataTypes.proto
@@ -101,5 +101,15 @@ enum Type {
message DataType {
required Type type = 1;
optional int32 length = 2;
+
+ /* Auxiliary information */
optional string code = 3;
+
+ /**
+ * Nested fields. Since Protobuf does not support nested fields,
+ * the nesting is flattened to a single list by a depth-first traversal.
+ * The children count is used to construct the nested relationship.
+ * This field is not set when the element is a primitive type
+ */
+ optional int32 num_nested_fields = 4;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml
index 743180f..38bddec 100644
--- a/tajo-core/pom.xml
+++ b/tajo-core/pom.xml
@@ -492,6 +492,62 @@
</plugins>
</build>
</profile>
+
+ <profile> <!-- This is for testing or debugging in IDE -->
+ <id>all-dependencies</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </dependency>
+
+ </dependencies>
+ </profile>
+
<profile>
<id>hcatalog-0.12.0</id>
<activation>
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
index d6631f6..5819bd2 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
@@ -41,6 +41,27 @@ public class TestTajoDump extends QueryTestCaseBase {
printWriter.close();
assertStrings(new String(bos.toByteArray()));
bos.close();
+
+ executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\"");
+ }
+ }
+
+ @Test
+ public void testDump2() throws Exception {
+ if (!testingCluster.isHCatalogStoreRunning()) {
+ executeString("CREATE TABLE \"" + getCurrentDatabase() +
+ "\".\"TableName2\" (\"Age\" int, \"Name\" Record (\"FirstName\" TEXT, lastname TEXT))");
+
+ UserRoleInfo userInfo = UserRoleInfo.getCurrentUser();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintWriter printWriter = new PrintWriter(bos);
+ TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter);
+ printWriter.flush();
+ printWriter.close();
+ assertStrings(new String(bos.toByteArray()));
+ bos.close();
+
+ executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName2\"");
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
index d4a5a1f..aee5a02 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
@@ -205,17 +205,7 @@ public class TestTajoCli {
assertEquals(databaseName, tajoCli.getContext().getCurrentDatabase());
}
- @Test
- public void testDescTable() throws Exception {
- String tableName;
- if (cluster.isHCatalogStoreRunning()) {
- tableName = "TEST_DESC_TABLE".toLowerCase();
- } else {
- tableName = "TEST_DESC_TABLE";
- }
-
- String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4);";
-
+ private void verifyDescTable(String sql, String tableName, String resultFileName) throws Exception {
setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
tajoCli.executeScript(sql);
@@ -226,12 +216,38 @@ public class TestTajoCli {
FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration());
if (!cluster.isHCatalogStoreRunning()) {
- assertOutputResult("testDescTable.result", consoleResult, new String[]{"${table.path}"},
+ assertOutputResult(resultFileName, consoleResult, new String[]{"${table.path}"},
new String[]{fs.getUri() + "/tajo/warehouse/default/" + tableName});
}
}
@Test
+ public void testDescTable() throws Exception {
+ String tableName;
+ if (cluster.isHCatalogStoreRunning()) {
+ tableName = "TEST_DESC_TABLE".toLowerCase();
+ } else {
+ tableName = "TEST_DESC_TABLE";
+ }
+
+ String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4);";
+ verifyDescTable(sql, tableName, "testDescTable.result");
+ }
+
+ @Test
+ public void testDescTableForNestedSchema() throws Exception {
+ String tableName;
+ if (cluster.isHCatalogStoreRunning()) {
+ tableName = "TEST_DESC_TABLE_NESTED".toLowerCase();
+ } else {
+ tableName = "TEST_DESC_TABLE_NESTED";
+ }
+
+ String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4, col3 record (col4 record (col5 text)));";
+ verifyDescTable(sql, tableName, "testDescTableForNestedSchema.result");
+ }
+
+ @Test
public void testSelectResultWithNullFalse() throws Exception {
String sql =
"select\n" +
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
index 21b3910..1fbe7c5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
@@ -600,4 +600,28 @@ public class TestCreateTable extends QueryTestCaseBase {
*/
}
}
+
+ @Test
+ public final void testNestedRecord1() throws Exception {
+ executeString("CREATE DATABASE D9;").close();
+
+ assertTableNotExists("d9.nested_table");
+ executeQuery().close();
+ assertTableExists("d9.nested_table");
+
+ executeString("DROP TABLE D9.nested_table");
+ executeString("DROP DATABASE D9").close();
+ }
+
+ @Test
+ public final void testNestedRecord2() throws Exception {
+ executeString("CREATE DATABASE D9;").close();
+
+ assertTableNotExists("d9.nested_table2");
+ executeQuery();
+ assertTableExists("d9.nested_table2");
+
+ executeString("DROP TABLE D9.nested_table2");
+ executeString("DROP DATABASE D9").close();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord1.sql b/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord1.sql
new file mode 100644
index 0000000..d147361
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord1.sql
@@ -0,0 +1 @@
+CREATE TABLE D9.nested_table (f1 int, nested_field record (f2 int4, f3 int8), f3 text);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord2.sql b/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord2.sql
new file mode 100644
index 0000000..f794d21
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestCreateTable/testNestedRecord2.sql
@@ -0,0 +1 @@
+CREATE TABLE D9.nested_table2 (f1 int, nf1 record (f1 int4, f3 double), nf2 record (f1 int4, nf1 record (f1 int4, f2 text), nf2 record (f1 int4, f2 text), f2 double), f2 text);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result
index a6aa8f4..2dc746d 100644
--- a/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result
+++ b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelect.result
@@ -2,5 +2,5 @@ explain
-------------------------------
SCAN(0) on default.lineitem
=> target list: default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4)
- => out schema: {(2) default.lineitem.l_orderkey (INT4),default.lineitem.l_partkey (INT4)}
- => in schema: {(16) default.lineitem.l_orderkey (INT4),default.lineitem.l_partkey (INT4),default.lineitem.l_suppkey (INT4),default.lineitem.l_linenumber (INT4),default.lineitem.l_quantity (FLOAT8),default.lineitem.l_extendedprice (FLOAT8),default.lineitem.l_discount (FLOAT8),default.lineitem.l_tax (FLOAT8),default.lineitem.l_returnflag (TEXT),default.lineitem.l_linestatus (TEXT),default.lineitem.l_shipdate (TEXT),default.lineitem.l_commitdate (TEXT),default.lineitem.l_receiptdate (TEXT),default.lineitem.l_shipinstruct (TEXT),default.lineitem.l_shipmode (TEXT),default.lineitem.l_comment (TEXT)}
\ No newline at end of file
+ => out schema: {(2) default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4)}
+ => in schema: {(16) default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4), default.lineitem.l_suppkey (INT4), default.lineitem.l_linenumber (INT4), default.lineitem.l_quantity (FLOAT8), default.lineitem.l_extendedprice (FLOAT8), default.lineitem.l_discount (FLOAT8), default.lineitem.l_tax (FLOAT8), default.lineitem.l_returnflag (TEXT), default.lineitem.l_linestatus (TEXT), default.lineitem.l_shipdate (TEXT), default.lineitem.l_commitdate (TEXT), default.lineitem.l_receiptdate (TEXT), default.lineitem.l_shipinstruct (TEXT), default.lineitem.l_shipmode (TEXT), default.lineitem.l_comment (TEXT)}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result b/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result
new file mode 100644
index 0000000..83f360b
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result
@@ -0,0 +1,29 @@
+OK
+
+table name: default.TEST_DESC_TABLE_NESTED
+table path: ${table.path}
+store type: CSV
+number of rows: 0
+volume: 0 B
+Options:
+ 'text.delimiter'='|'
+
+schema:
+col1 INT4
+col2 INT4
+col3 RECORD (col4 RECORD (col5 TEXT))
+
+
+
+table name: default.TEST_DESC_TABLE_NESTED
+table path: ${table.path}
+store type: CSV
+number of rows: 0
+volume: 0 B
+Options:
+ 'text.delimiter'='|'
+
+schema:
+col1 INT4
+col2 INT4
+col3 RECORD (col4 RECORD (col5 TEXT))
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result b/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result
new file mode 100644
index 0000000..6c15e3e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result
@@ -0,0 +1,16 @@
+--
+-- Tajo database dump
+--
+
+
+--
+-- Database name: "TestTajoDump"
+--
+
+CREATE DATABASE IF NOT EXISTS "TestTajoDump";
+
+--
+-- Name: "TestTajoDump"."TableName2"; Type: TABLE; Storage: CSV
+--
+CREATE TABLE "TestTajoDump"."TableName2" ("Age" INT4, "Name" RECORD ("FirstName" TEXT, lastname TEXT)) USING CSV WITH ('text.delimiter'='|');
+
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
index 235bebf..5166e80 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
@@ -793,10 +793,11 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva
}
return new ConstEval(
- DatumFactory.cast(constEval.getValue(), LogicalPlanner.convertDataType(expr.getTarget()), tz));
+ DatumFactory.cast(constEval.getValue(),
+ LogicalPlanner.convertDataType(expr.getTarget()).getDataType(), tz));
} else {
- return new CastEval(ctx.queryContext, child, LogicalPlanner.convertDataType(expr.getTarget()));
+ return new CastEval(ctx.queryContext, child, LogicalPlanner.convertDataType(expr.getTarget()).getDataType());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 14fea08..8395c3d 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -1860,7 +1860,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
* @param elements to be transformed
* @return schema transformed from table definition elements
*/
- private Schema convertTableElementsSchema(ColumnDefinition[] elements) {
+ private static Schema convertTableElementsSchema(ColumnDefinition[] elements) {
Schema schema = new Schema();
for (ColumnDefinition columnDefinition: elements) {
@@ -1870,19 +1870,29 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return schema;
}
- private Column convertColumn(ColumnDefinition columnDefinition) {
+ private static Column convertColumn(ColumnDefinition columnDefinition) {
return new Column(columnDefinition.getColumnName(), convertDataType(columnDefinition));
}
- public static TajoDataTypes.DataType convertDataType(DataTypeExpr dataType) {
+ public static TypeDesc convertDataType(DataTypeExpr dataType) {
TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(dataType.getTypeName());
TajoDataTypes.DataType.Builder builder = TajoDataTypes.DataType.newBuilder();
builder.setType(type);
+
if (dataType.hasLengthOrPrecision()) {
builder.setLength(dataType.getLengthOrPrecision());
}
- return builder.build();
+
+ TypeDesc typeDesc;
+ if (type == TajoDataTypes.Type.RECORD) {
+ Schema nestedRecordSchema = convertTableElementsSchema(dataType.getNestedRecordTypes());
+ typeDesc = new TypeDesc(nestedRecordSchema);
+ } else {
+ typeDesc = new TypeDesc(builder.build());
+ }
+
+ return typeDesc;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f48d4bd0/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java
index 6222734..7c468bb 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java
@@ -59,7 +59,7 @@ public class TypeDeterminant extends SimpleAlgebraVisitor<LogicalPlanner.PlanCon
dataType = BOOL_TYPE;
break;
case Cast:
- dataType = LogicalPlanner.convertDataType(((CastExpr)expr).getTarget());
+ dataType = LogicalPlanner.convertDataType(((CastExpr)expr).getTarget()).getDataType();
break;
default:
dataType = visit(ctx, stack, expr.getChild());
@@ -270,7 +270,7 @@ public class TypeDeterminant extends SimpleAlgebraVisitor<LogicalPlanner.PlanCon
@Override
public DataType visitDataType(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DataTypeExpr expr)
throws PlanningException {
- return LogicalPlanner.convertDataType(expr);
+ return LogicalPlanner.convertDataType(expr).getDataType();
}
@Override
[3/3] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Conflicts:
tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5bb4108c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5bb4108c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5bb4108c
Branch: refs/heads/index_support
Commit: 5bb4108c9cc1b2ea7fe08743f3c3298719676071
Parents: 292cd9e f48d4bd
Author: Jihoon Son <ji...@apache.org>
Authored: Fri Mar 13 12:23:57 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Fri Mar 13 12:23:57 2015 +0900
----------------------------------------------------------------------
CHANGES | 5 +
.../apache/tajo/algebra/ColumnDefinition.java | 2 +-
.../org/apache/tajo/algebra/DataTypeExpr.java | 18 ++-
.../org/apache/tajo/catalog/CatalogUtil.java | 28 ++--
.../java/org/apache/tajo/catalog/Column.java | 45 ++++--
.../org/apache/tajo/catalog/DDLBuilder.java | 9 +-
.../java/org/apache/tajo/catalog/Schema.java | 156 +++++++++++++------
.../org/apache/tajo/catalog/SchemaUtil.java | 42 +++++
.../java/org/apache/tajo/catalog/TypeDesc.java | 75 +++++++++
.../org/apache/tajo/catalog/TestSchema.java | 137 ++++++++++++++++
.../dictionary/ColumnsTableDescriptor.java | 7 +-
.../tajo/catalog/store/AbstractDBStore.java | 64 +++++---
.../apache/tajo/catalog/store/MariaDBStore.java | 5 +-
.../apache/tajo/catalog/store/MySQLStore.java | 11 +-
.../apache/tajo/catalog/store/OracleStore.java | 2 +-
.../tajo/catalog/store/PostgreSQLStore.java | 4 +-
.../src/main/resources/schemas/derby/derby.xml | 12 +-
.../main/resources/schemas/mariadb/columns.sql | 3 +-
.../main/resources/schemas/mysql/columns.sql | 3 +-
.../main/resources/schemas/oracle/columns.sql | 9 --
.../main/resources/schemas/oracle/databases.sql | 6 -
.../main/resources/schemas/oracle/indexes.sql | 18 ---
.../main/resources/schemas/oracle/oracle.xml | 10 +-
.../schemas/oracle/partition_methods.sql | 7 -
.../resources/schemas/oracle/partitions.sql | 10 --
.../src/main/resources/schemas/oracle/stats.sql | 6 -
.../schemas/oracle/table_properties.sql | 7 -
.../main/resources/schemas/oracle/tables.sql | 9 --
.../resources/schemas/oracle/tablespaces.sql | 6 -
.../resources/schemas/postgresql/columns.sql | 9 --
.../resources/schemas/postgresql/databases.sql | 6 -
.../schemas/postgresql/partition_methods.sql | 7 -
.../resources/schemas/postgresql/partitions.sql | 10 --
.../resources/schemas/postgresql/postgresql.xml | 10 +-
.../schemas/postgresql/table_properties.sql | 7 -
.../resources/schemas/postgresql/tables.sql | 9 --
.../schemas/postgresql/tablespaces.sql | 6 -
.../org/apache/tajo/catalog/TestCatalog.java | 101 ++++++++++++
.../cli/tsql/commands/DescTableCommand.java | 5 +-
tajo-common/src/main/proto/DataTypes.proto | 10 ++
tajo-core/pom.xml | 56 +++++++
.../org/apache/tajo/cli/tools/TestTajoDump.java | 21 +++
.../org/apache/tajo/cli/tsql/TestTajoCli.java | 40 +++--
.../tajo/engine/query/TestCreateTable.java | 24 +++
.../TestCreateTable/testNestedRecord1.sql | 1 +
.../TestCreateTable/testNestedRecord2.sql | 1 +
.../TestSelectQuery/testExplainSelect.result | 4 +-
.../testDescTableForNestedSchema.result | 29 ++++
.../results/TestTajoDump/testDump2.result | 16 ++
.../org/apache/tajo/plan/ExprAnnotator.java | 5 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 18 ++-
.../org/apache/tajo/plan/TypeDeterminant.java | 4 +-
52 files changed, 830 insertions(+), 285 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5bb4108c/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5bb4108c/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5bb4108c/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5bb4108c/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5bb4108c/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5bb4108c/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5bb4108c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5bb4108c/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------