You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/01/27 09:29:39 UTC
[4/4] git commit: TAJO-475: Table partition catalog recap. (Min Zhou
and hyunsik)
TAJO-475: Table partition catalog recap. (Min Zhou and hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/eb563add
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/eb563add
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/eb563add
Branch: refs/heads/master
Commit: eb563addd336019417d045b95277dec9f89b8f5f
Parents: accd0e5
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Jan 27 03:44:55 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Jan 27 17:23:45 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../org/apache/tajo/algebra/CreateTable.java | 18 +-
.../tajo/catalog/AbstractCatalogClient.java | 35 +-
.../org/apache/tajo/catalog/CatalogService.java | 6 +
.../src/main/proto/CatalogProtocol.proto | 14 +-
.../apache/tajo/catalog/CatalogConstants.java | 2 +
.../org/apache/tajo/catalog/CatalogUtil.java | 15 +
.../org/apache/tajo/catalog/DDLBuilder.java | 66 +-
.../java/org/apache/tajo/catalog/IndexDesc.java | 4 +-
.../java/org/apache/tajo/catalog/Schema.java | 4 +-
.../java/org/apache/tajo/catalog/TableDesc.java | 67 +-
.../tajo/catalog/partition/PartitionDesc.java | 173 ++--
.../catalog/partition/PartitionMethodDesc.java | 141 +++
.../tajo/catalog/partition/Specifier.java | 128 ---
.../src/main/proto/CatalogProtos.proto | 30 +-
.../tajo/catalog/store/HCatalogStore.java | 95 +-
.../tajo/catalog/store/TestHCatalogStore.java | 16 +-
.../org/apache/tajo/catalog/CatalogServer.java | 100 ++-
.../tajo/catalog/store/AbstractDBStore.java | 813 +++++++++++------
.../apache/tajo/catalog/store/CatalogStore.java | 43 +-
.../apache/tajo/catalog/store/DerbyStore.java | 865 ++-----------------
.../org/apache/tajo/catalog/store/MemStore.java | 84 +-
.../apache/tajo/catalog/store/MySQLStore.java | 80 +-
.../org/apache/tajo/catalog/TestCatalog.java | 124 ++-
.../org/apache/tajo/catalog/TestDBStore.java | 113 +--
.../org/apache/tajo/cli/DescTableCommand.java | 37 +-
tajo-client/src/main/proto/ClientProtos.proto | 2 +-
tajo-core/tajo-core-backend/pom.xml | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 7 +-
.../apache/tajo/engine/planner/InsertNode.java | 10 +-
.../apache/tajo/engine/planner/LogicalPlan.java | 2 +-
.../tajo/engine/planner/LogicalPlanner.java | 340 +++-----
.../engine/planner/PhysicalPlannerImpl.java | 7 +-
.../apache/tajo/engine/planner/PlannerUtil.java | 4 +-
.../engine/planner/global/GlobalPlanner.java | 18 +-
.../engine/planner/logical/CreateTableNode.java | 14 +-
.../logical/PartitionedTableScanNode.java | 2 +-
.../planner/logical/PersistentStoreNode.java | 32 -
.../tajo/engine/planner/logical/ScanNode.java | 16 +-
.../planner/logical/ShuffleFileWriteNode.java | 4 +-
.../engine/planner/logical/StoreTableNode.java | 25 +-
.../ColumnPartitionedTableStoreExec.java | 41 +-
.../planner/physical/HashAggregateExec.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 25 +-
.../rewrite/PartitionedTableRewriter.java | 12 +-
.../planner/rewrite/ProjectionPushDownRule.java | 2 +-
.../apache/tajo/engine/query/QueryContext.java | 12 +-
.../apache/tajo/engine/utils/SchemaUtil.java | 9 +
.../org/apache/tajo/master/GlobalEngine.java | 15 +-
.../tajo/master/TajoMasterClientService.java | 8 +-
.../apache/tajo/master/querymaster/Query.java | 293 +++++--
.../org/apache/tajo/client/TestDDLBuilder.java | 13 +-
.../org/apache/tajo/client/TestTajoClient.java | 2 +-
.../tajo/engine/parser/TestHiveConverter.java | 1 -
.../tajo/engine/parser/TestSQLAnalyzer.java | 20 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 2 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 20 +-
.../tajo/engine/query/TestTablePartitions.java | 38 +-
.../tajo/engine/query/TestUnionQuery.java | 6 -
.../test/resources/results/testBuildDDL.result | 2 +-
60 files changed, 1909 insertions(+), 2174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c66504d..a867658 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -125,6 +125,8 @@ Release 0.8.0 - unreleased
IMPROVEMENTS
+ TAJO-475: Table partition catalog recap. (Min Zhou and hyunsik)
+
TAJO-539: Change some EvalNode::eval to directly return a Datum value.
(hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
index 87405b0..1836f51 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
@@ -32,7 +32,7 @@ public class CreateTable extends Expr {
private String location;
private Expr subquery;
private Map<String, String> params;
- private PartitionDescExpr partition;
+ private PartitionMethodDescExpr partition;
public CreateTable(final String tableName) {
super(OpType.CreateTable);
@@ -108,11 +108,11 @@ public class CreateTable extends Expr {
return partition != null;
}
- public void setPartition(PartitionDescExpr partition) {
+ public void setPartitionMethod(PartitionMethodDescExpr partition) {
this.partition = partition;
}
- public <T extends PartitionDescExpr> T getPartition() {
+ public <T extends PartitionMethodDescExpr> T getPartitionMethod() {
return (T) this.partition;
}
@@ -194,10 +194,10 @@ public class CreateTable extends Expr {
COLUMN
}
- public static abstract class PartitionDescExpr {
+ public static abstract class PartitionMethodDescExpr {
PartitionType type;
- public PartitionDescExpr(PartitionType type) {
+ public PartitionMethodDescExpr(PartitionType type) {
this.type = type;
}
@@ -206,7 +206,7 @@ public class CreateTable extends Expr {
}
}
- public static class RangePartition extends PartitionDescExpr {
+ public static class RangePartition extends PartitionMethodDescExpr {
ColumnReferenceExpr [] columns;
List<RangePartitionSpecifier> specifiers;
@@ -239,7 +239,7 @@ public class CreateTable extends Expr {
}
}
- public static class HashPartition extends PartitionDescExpr {
+ public static class HashPartition extends PartitionMethodDescExpr {
ColumnReferenceExpr [] columns;
Expr quantity;
List<PartitionSpecifier> specifiers;
@@ -291,7 +291,7 @@ public class CreateTable extends Expr {
}
}
- public static class ListPartition extends PartitionDescExpr {
+ public static class ListPartition extends PartitionMethodDescExpr {
ColumnReferenceExpr [] columns;
List<ListPartitionSpecifier> specifiers;
@@ -324,7 +324,7 @@ public class CreateTable extends Expr {
}
}
- public static class ColumnPartition extends PartitionDescExpr {
+ public static class ColumnPartition extends PartitionMethodDescExpr {
private ColumnDefinition [] columns;
private boolean isOmitValues;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index 981e062..9176a88 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService;
import org.apache.tajo.catalog.exception.NoSuchFunctionException;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.conf.TajoConf;
@@ -52,7 +53,7 @@ public abstract class AbstractCatalogClient implements CatalogService {
public AbstractCatalogClient(TajoConf conf, InetSocketAddress catalogServerAddr) {
this.pool = RpcConnectionPool.getPool(conf);
this.catalogServerAddr = catalogServerAddr;
- this.conf= conf;
+ this.conf = conf;
}
@Override
@@ -71,6 +72,38 @@ public abstract class AbstractCatalogClient implements CatalogService {
}
@Override
+ public final PartitionMethodDesc getPartitionMethod(final String tableName) {
+ try {
+ return new ServerCallable<PartitionMethodDesc>(conf, catalogServerAddr, CatalogProtocol.class, false) {
+ public PartitionMethodDesc call(NettyClientBase client) throws ServiceException {
+ CatalogProtocolService.BlockingInterface stub = getStub(client);
+ return CatalogUtil.newPartitionMethodDesc(stub.getPartitionMethodByTableName(null,
+ StringProto.newBuilder().setValue(tableName).build()));
+ }
+ }.withRetries();
+ } catch (ServiceException e) {
+ LOG.error(e.getMessage(), e);
+ return null;
+ }
+ }
+
+ @Override
+ public final boolean existPartitionMethod(final String tableId) {
+ try {
+ return new ServerCallable<Boolean>(conf, catalogServerAddr, CatalogProtocol.class, false) {
+ public Boolean call(NettyClientBase client) throws ServiceException {
+ CatalogProtocolService.BlockingInterface stub = getStub(client);
+ return stub.existPartitionMethod(null, StringProto.newBuilder().
+ setValue(tableId).build()).getValue();
+ }
+ }.withRetries();
+ } catch (ServiceException e) {
+ LOG.error(e.getMessage(), e);
+ return false;
+ }
+ }
+
+ @Override
public final Collection<String> getAllTableNames() {
try {
return new ServerCallable<Collection<String>>(conf, catalogServerAddr, CatalogProtocol.class, false) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
index c6bd7fb..494ac0b 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
@@ -18,6 +18,7 @@
package org.apache.tajo.catalog;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.common.TajoDataTypes.DataType;
import java.util.Collection;
@@ -56,6 +57,7 @@ public interface CatalogService {
*/
boolean addTable(TableDesc desc);
+
/**
* Drop a table by name
*
@@ -66,6 +68,10 @@ public interface CatalogService {
boolean existsTable(String tableId);
+ PartitionMethodDesc getPartitionMethod(String tableId);
+
+ boolean existPartitionMethod(String tableId);
+
boolean addIndex(IndexDesc index);
boolean existIndex(String indexName);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
index 6374278..6a96b4e 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
+++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
@@ -27,18 +27,30 @@ import "PrimitiveProtos.proto";
service CatalogProtocolService {
rpc getTableDesc(StringProto) returns (TableDescProto);
rpc getAllTableNames(NullProto) returns (GetAllTableNamesResponse);
- rpc getFunctions(NullProto) returns (GetFunctionsResponse);
rpc addTable(TableDescProto) returns (BoolProto);
rpc deleteTable(StringProto) returns (BoolProto);
rpc existsTable(StringProto) returns (BoolProto);
+
+ rpc getPartitionMethodByTableName(StringProto) returns (PartitionMethodProto);
+ rpc existPartitionMethod(StringProto) returns (BoolProto);
+ rpc delPartitionMethod(StringProto) returns (BoolProto);
+
+ rpc addPartitions(PartitionsProto) returns (BoolProto);
+ rpc addPartition(PartitionDescProto) returns (BoolProto);
+ rpc getPartitionByPartitionName(StringProto) returns (PartitionDescProto);
+ rpc getPartitionsByTableName(StringProto) returns (PartitionsProto);
+ rpc delAllPartitions(StringProto) returns (PartitionsProto);
+
rpc addIndex(IndexDescProto) returns (BoolProto);
rpc existIndexByName(StringProto) returns (BoolProto);
rpc existIndex(GetIndexRequest) returns (BoolProto);
rpc getIndexByName(StringProto) returns (IndexDescProto);
rpc getIndex(GetIndexRequest) returns (IndexDescProto);
rpc delIndex(StringProto) returns (BoolProto);
+
rpc createFunction(FunctionDescProto) returns (BoolProto);
rpc dropFunction(UnregisterFunctionRequest) returns (BoolProto);
+ rpc getFunctions(NullProto) returns (GetFunctionsResponse);
rpc getFunctionMeta(GetFunctionMetaRequest) returns (FunctionDescProto);
rpc containFunction(ContainFunctionRequest) returns (BoolProto);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
index ed23b08..0a40640 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
@@ -35,7 +35,9 @@ public class CatalogConstants {
public static final String TB_OPTIONS = "OPTIONS";
public static final String TB_INDEXES = "INDEXES";
public static final String TB_STATISTICS = "STATS";
+ public static final String TB_PARTITION_METHODS = "PARTITION_METHODS";
public static final String TB_PARTTIONS = "PARTITIONS";
+
public static final String C_TABLE_ID = "TABLE_ID";
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index dc91035..4a3fc27 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -19,6 +19,8 @@
package org.apache.tajo.catalog;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
@@ -88,6 +90,10 @@ public class CatalogUtil {
return new TableDesc(proto);
}
+ public static PartitionMethodDesc newPartitionMethodDesc(CatalogProtos.PartitionMethodProto proto) {
+ return new PartitionMethodDesc(proto);
+ }
+
public static TableDesc newTableDesc(String tableName, Schema schema, StoreType type, Options options, Path path) {
return new TableDesc(tableName, schema, type, options, path);
}
@@ -142,6 +148,15 @@ public class CatalogUtil {
return DataType.newBuilder().setType(type).setLength(length).build();
}
+ public static String columnToDDLString(Column column) {
+ StringBuilder sb = new StringBuilder(column.getColumnName());
+ sb.append(" ").append(column.getDataType().getType());
+ if (column.getDataType().hasLength()) {
+ sb.append(" (").append(column.getDataType().getLength()).append(")");
+ }
+ return sb.toString();
+ }
+
public static void closeSQLWrapper(Wrapper... wrapper) {
if(wrapper == null) return;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index c818be7..1ae77b4 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -18,9 +18,7 @@
package org.apache.tajo.catalog;
-import org.apache.tajo.catalog.partition.PartitionDesc;
-import org.apache.tajo.catalog.partition.Specifier;
-import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.common.TajoDataTypes;
import java.util.Map;
@@ -39,12 +37,13 @@ public class DDLBuilder {
buildSchema(sb, desc.getSchema());
buildUsingClause(sb, desc.getMeta());
buildWithClause(sb, desc.getMeta());
- buildLocationClause(sb, desc);
- if (desc.getPartitions() != null) {
+ if (desc.hasPartition()) {
buildPartitionClause(sb, desc);
}
+ buildLocationClause(sb, desc);
+
sb.append(";");
return sb.toString();
}
@@ -96,65 +95,16 @@ public class DDLBuilder {
}
private static void buildPartitionClause(StringBuilder sb, TableDesc desc) {
- PartitionDesc partitionDesc = desc.getPartitions();
+ PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
sb.append(" PARTITION BY ");
- sb.append(partitionDesc.getPartitionsType().name());
+ sb.append(partitionDesc.getPartitionType().name());
// columns
sb.append("(");
- int columnCount = 0;
- for(Column column: partitionDesc.getColumns()) {
- for(Column targetColumn: desc.getSchema().getColumns()) {
- if (column.getColumnName().equals(targetColumn.getColumnName())) {
- if (columnCount > 0)
- sb.append(",");
-
- sb.append(column.getColumnName());
- columnCount++;
- }
- }
+ for (Column column : partitionDesc.getExpressionSchema().toArray()) {
+ sb.append(CatalogUtil.columnToDDLString(column));
}
sb.append(")");
-
- // specifier
- if (partitionDesc.getSpecifiers() != null
- && !partitionDesc.getPartitionsType().equals(CatalogProtos.PartitionsType.COLUMN)) {
-
- sb.append(" (");
- for(int i = 0; i < partitionDesc.getSpecifiers().size(); i++) {
- Specifier specifier = partitionDesc.getSpecifiers().get(i);
- if (i > 0)
- sb.append(",");
-
- sb.append(" PARTITION");
-
- if (!specifier.getName().isEmpty())
- sb.append(" ").append(specifier.getName());
-
- if (partitionDesc.getPartitionsType().equals(CatalogProtos.PartitionsType.LIST)) {
- if (!specifier.getExpressions().isEmpty()) {
- sb.append(" VALUES (");
- String[] expressions = specifier.getExpressions().split("\\,");
- for(int j = 0; j < expressions.length; j++) {
- if (j > 0)
- sb.append(",");
- sb.append("'").append(expressions[j]).append("'");
- }
- sb.append(")");
-
- }
- } else if (partitionDesc.getPartitionsType().equals(CatalogProtos.PartitionsType.RANGE)) {
- sb.append(" VALUES LESS THAN (");
- if (!specifier.getExpressions().isEmpty()) {
- sb.append(specifier.getExpressions());
- } else {
- sb.append("MAXVALUE");
- }
- sb.append(")");
- }
- }
- sb.append(")");
- }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
index 9465946..6199259 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
@@ -120,7 +120,7 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
return Objects.hashCode(getName(), getTableId(), getColumn(),
getIndexMethod(), isUnique(), isClustered(), isAscending());
}
-
+
public Object clone() throws CloneNotSupportedException {
IndexDesc desc = (IndexDesc) super.clone();
desc.name = name;
@@ -130,7 +130,7 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
desc.isUnique = isUnique;
desc.isClustered = isClustered;
desc.isAscending = isAscending;
- return desc;
+ return desc;
}
public String toString() {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index a19d352..c971b63 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@ -199,10 +199,10 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public boolean contains(String name) {
- if (fieldsByQualifiedName.containsKey(name)) {
+ if (fieldsByQualifiedName.containsKey(name.toLowerCase())) {
return true;
}
- if (fieldsByName.containsKey(name)) {
+ if (fieldsByName.containsKey(name.toLowerCase())) {
if (fieldsByName.size() > 1) {
throw new RuntimeException("Ambiguous Column name");
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
index 8fffc6e..1b0a9fe 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
@@ -25,7 +25,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
-import org.apache.tajo.catalog.partition.PartitionDesc;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
import org.apache.tajo.catalog.statistics.TableStats;
@@ -38,13 +38,13 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
protected TableDescProto.Builder builder = null;
- @Expose protected String tableName; // required
+ @Expose protected String tableName; // required
@Expose protected Schema schema;
- @Expose protected TableMeta meta; // required
- @Expose protected Path uri; // required
- @Expose protected TableStats stats; // optional
- @Expose protected PartitionDesc partitionDesc; //optional
-
+ @Expose protected TableMeta meta; // required
+ @Expose protected Path uri; // required
+ @Expose protected TableStats stats; // optional
+ @Expose protected PartitionMethodDesc partitionMethodDesc; // optional
+
public TableDesc() {
builder = TableDescProto.newBuilder();
}
@@ -64,9 +64,11 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
public TableDesc(TableDescProto proto) {
this(proto.getId(), new Schema(proto.getSchema()), new TableMeta(proto.getMeta()), new Path(proto.getPath()));
- this.stats = new TableStats(proto.getStats());
- if (proto.getPartitions() != null && !proto.getPartitions().toString().isEmpty()) {
- this.partitionDesc = new PartitionDesc(proto.getPartitions());
+ if(proto.hasStats()) {
+ this.stats = new TableStats(proto.getStats());
+ }
+ if (proto.hasPartition()) {
+ this.partitionMethodDesc = new PartitionMethodDesc(proto.getPartition());
}
}
@@ -103,6 +105,17 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
return schema;
}
+ public Schema getLogicalSchema() {
+ if (hasPartition()) {
+ Schema logicalSchema = new Schema(schema);
+ logicalSchema.addColumns(getPartitionMethod().getExpressionSchema());
+ logicalSchema.setQualifier(tableName);
+ return logicalSchema;
+ } else {
+ return schema;
+ }
+ }
+
public void setStats(TableStats stats) {
this.stats = stats;
}
@@ -115,16 +128,16 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
return this.stats;
}
- public boolean hasPartitions() {
- return this.partitionDesc != null;
+ public boolean hasPartition() {
+ return this.partitionMethodDesc != null;
}
- public PartitionDesc getPartitions() {
- return partitionDesc;
+ public PartitionMethodDesc getPartitionMethod() {
+ return partitionMethodDesc;
}
- public void setPartitions(PartitionDesc partitionDesc) {
- this.partitionDesc = partitionDesc;
+ public void setPartitionMethod(PartitionMethodDesc partitionMethodDesc) {
+ this.partitionMethodDesc = partitionMethodDesc;
}
public boolean equals(Object object) {
@@ -135,14 +148,14 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
eq = eq && schema.equals(other.schema);
eq = eq && meta.equals(other.meta);
eq = eq && uri.equals(other.uri);
- eq = eq && TUtil.checkEquals(partitionDesc, other.partitionDesc);
+ eq = eq && TUtil.checkEquals(partitionMethodDesc, other.partitionMethodDesc);
return eq && TUtil.checkEquals(stats, other.stats);
}
return false;
}
- public Object clone() throws CloneNotSupportedException {
+ public Object clone() throws CloneNotSupportedException {
TableDesc desc = (TableDesc) super.clone();
desc.builder = TableDescProto.newBuilder();
desc.tableName = tableName;
@@ -150,16 +163,15 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
desc.meta = (TableMeta) meta.clone();
desc.uri = uri;
desc.stats = stats != null ? (TableStats) stats.clone() : null;
- desc.partitionDesc = partitionDesc != null ? (PartitionDesc) partitionDesc.clone() : null;
-
+ desc.partitionMethodDesc = partitionMethodDesc != null ? (PartitionMethodDesc) partitionMethodDesc.clone() : null;
return desc;
}
-
- public String toString() {
- Gson gson = new GsonBuilder().setPrettyPrinting().
- excludeFieldsWithoutExposeAnnotation().create();
+
+ public String toString() {
+ Gson gson = new GsonBuilder().setPrettyPrinting().
+ excludeFieldsWithoutExposeAnnotation().create();
return gson.toJson(this);
- }
+ }
public String toJson() {
return CatalogGsonHelper.toJson(this, TableDesc.class);
@@ -184,9 +196,10 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
if (this.stats != null) {
builder.setStats(this.stats.getProto());
}
- if (this.partitionDesc != null) {
- builder.setPartitions(this.partitionDesc.getProto());
+ if (this.partitionMethodDesc != null) {
+ builder.setPartition(this.partitionMethodDesc.getProto());
}
+
return builder.build();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
index c3e31d8..f7c1342 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
@@ -18,28 +18,22 @@
package org.apache.tajo.catalog.partition;
-import com.google.common.collect.ImmutableList;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
import com.google.gson.annotations.Expose;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.json.GsonObject;
-import org.apache.tajo.util.TUtil;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
+/**
+ * <code>PartitionDesc</code> presents a table partition.
+ */
public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescProto>, Cloneable, GsonObject {
- @Expose protected CatalogProtos.PartitionsType partitionsType; //required
- @Expose protected Schema schema;
- @Expose protected int numPartitions; //optional
- @Expose protected List<Specifier> specifiers; //optional
- @Expose protected boolean isOmitValues = false; // optional;
+
+ @Expose protected String tableId; // required
+ @Expose protected String partitionName; // optional
+ @Expose protected int ordinalPosition; // required
+ @Expose protected String partitionValue; // optional
+ @Expose protected String path; // optional
private CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder();
@@ -47,149 +41,120 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro
}
public PartitionDesc(PartitionDesc partition) {
- this();
- this.partitionsType = partition.partitionsType;
- this.schema = partition.schema;
- this.numPartitions = partition.numPartitions;
- this.specifiers = partition.specifiers;
- }
-
- public PartitionDesc(CatalogProtos.PartitionsType partitionsType, Column[] columns, int numPartitions,
- List<Specifier> specifiers) {
- this();
- this.partitionsType = partitionsType;
- for (Column c : columns) {
- addColumn(c);
- }
- this.numPartitions = numPartitions;
- this.specifiers = specifiers;
+ this.tableId = partition.tableId;
+ this.partitionName = partition.partitionName;
+ this.ordinalPosition = partition.ordinalPosition;
+ this.partitionValue = partition.partitionValue;
+ this.path = partition.path;
}
public PartitionDesc(CatalogProtos.PartitionDescProto proto) {
- this.partitionsType = proto.getPartitionsType();
- this.schema = new Schema(proto.getSchema());
- this.numPartitions = proto.getNumPartitions();
- this.isOmitValues = proto.getIsOmitValues();
- if(proto.getSpecifiersList() != null) {
- this.specifiers = TUtil.newList();
- for(CatalogProtos.SpecifierProto specifier: proto.getSpecifiersList()) {
- this.specifiers.add(new Specifier(specifier));
- }
+ this.tableId = proto.getTableId();
+ if(proto.hasPartitionName()) {
+ this.partitionName = proto.getPartitionName();
+ }
+ this.ordinalPosition = proto.getOrdinalPosition();
+ if(proto.hasPartitionValue()) {
+ this.partitionValue = proto.getPartitionValue();
+ }
+ if(proto.hasPath()) {
+ this.path = proto.getPath();
}
}
- public Schema getSchema() {
- return schema;
- }
-
- public List<Column> getColumns() {
- return ImmutableList.copyOf(schema.toArray());
+ public String getName() {
+ return partitionName;
}
- public void setColumns(Collection<Column> columns) {
- this.schema = new Schema(columns.toArray(new Column[columns.size()]));
+ public String getTableId() {
+ return tableId;
}
- public synchronized void addColumn(Column column) {
- if (schema == null) {
- schema = new Schema();
- }
- schema.addColumn(column);
+ public int getOrdinalPosition() {
+ return ordinalPosition;
}
- public synchronized void addSpecifier(Specifier specifier) {
- if(specifiers == null)
- specifiers = TUtil.newList();
-
- specifiers.add(specifier);
+ public String getPartitionValue() {
+ return partitionValue;
}
- public CatalogProtos.PartitionsType getPartitionsType() {
- return partitionsType;
+ public void setPartitionValue(String partitionValue) {
+ this.partitionValue = partitionValue;
}
- public void setPartitionsType(CatalogProtos.PartitionsType partitionsType) {
- this.partitionsType = partitionsType;
+ public String getPath() {
+ return path;
}
- public int getNumPartitions() {
- return numPartitions;
- }
- public void setNumPartitions(int numPartitions) {
- this.numPartitions = numPartitions;
+ public void setTable(String tableId) {
+ this.tableId = tableId;
}
- public List<Specifier> getSpecifiers() {
- return specifiers;
+ public void setName(String partitionName) {
+ this.partitionName = partitionName;
}
- public void setSpecifiers(List<Specifier> specifiers) {
- this.specifiers = specifiers;
- }
- public void setOmitValues(boolean flag) {
- isOmitValues = flag;
+ public void setOrdinalPosition(int ordinalPosition) {
+ this.ordinalPosition = ordinalPosition;
}
- public boolean isOmitValues() {
- return isOmitValues;
+ public void setPath(String path) {
+ this.path = path;
}
public boolean equals(Object o) {
if (o instanceof PartitionDesc) {
PartitionDesc another = (PartitionDesc) o;
- boolean eq = partitionsType == another.partitionsType;
- eq = eq && schema.equals(another.schema);
- eq = eq && numPartitions == another.numPartitions;
- eq = eq && TUtil.checkEquals(specifiers, another.specifiers);
- eq = eq && isOmitValues == another.isOmitValues;
+ boolean eq = tableId.equals(another.tableId);
+ eq = eq && ((partitionName != null && another.partitionName != null
+ && partitionName.equals(another.partitionName)) ||
+ (partitionName == null && another.partitionName == null));
+ eq = eq && (ordinalPosition == another.ordinalPosition);
+ eq = eq && ((partitionValue != null && another.partitionValue != null
+ && partitionValue.equals(another.partitionValue))
+ || (partitionValue == null && another.partitionValue == null));
+ eq = eq && ((path != null && another.path != null && path.equals(another.path)) ||
+ (path == null && another.path == null));
return eq;
}
return false;
}
- public Object clone() throws CloneNotSupportedException {
- PartitionDesc copy = (PartitionDesc) super.clone();
- copy.builder = CatalogProtos.PartitionDescProto.newBuilder();
- copy.setPartitionsType(this.partitionsType);
- copy.schema = new Schema(schema.getProto());
- copy.setNumPartitions(this.numPartitions);
- copy.specifiers = new ArrayList<Specifier>(this.specifiers);
- copy.isOmitValues = isOmitValues;
-
- return copy;
- }
@Override
public CatalogProtos.PartitionDescProto getProto() {
if (builder == null) {
builder = CatalogProtos.PartitionDescProto.newBuilder();
}
- if (this.partitionsType != null) {
- builder.setPartitionsType(this.partitionsType);
+
+ builder.setTableId(tableId);
+ if(this.partitionName != null) {
+ builder.setPartitionName(partitionName);
}
- builder.setSchema(schema.getProto());
- builder.setNumPartitions(numPartitions);
- builder.setIsOmitValues(isOmitValues);
- if (this.specifiers != null) {
- for(Specifier specifier: specifiers) {
- builder.addSpecifiers(specifier.getProto());
- }
+
+ builder.setOrdinalPosition(this.ordinalPosition);
+
+ if (this.partitionValue != null) {
+ builder.setPartitionValue(this.partitionValue);
+ }
+
+ if(this.path != null) {
+ builder.setPath(this.path);
}
+
return builder.build();
}
public String toString() {
- StringBuilder sb = new StringBuilder("Partition Type: " + partitionsType.name()).append(", key=");
- sb.append(schema);
+ StringBuilder sb = new StringBuilder("name: " + partitionName);
return sb.toString();
}
@Override
public String toJson() {
return CatalogGsonHelper.toJson(this, PartitionDesc.class);
-
}
public static PartitionDesc fromJson(String strVal) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java
new file mode 100644
index 0000000..d4a2c3e
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.catalog.partition;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.json.CatalogGsonHelper;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.common.ProtoObject;
+import org.apache.tajo.json.GsonObject;
+import org.apache.tajo.util.TUtil;
+
+/**
+ * <code>PartitionMethodDesc</code> presents a table description, including partition type, and partition keys.
+ */
+public class PartitionMethodDesc implements ProtoObject<CatalogProtos.PartitionMethodProto>, Cloneable, GsonObject {
+ private CatalogProtos.PartitionMethodProto.Builder builder;
+
+ @Expose private String tableId; // required
+ @Expose private CatalogProtos.PartitionType partitionType; // required
+ @Expose private String expression; // required
+ @Expose private Schema expressionSchema; // required
+
+ public PartitionMethodDesc() {
+ builder = CatalogProtos.PartitionMethodProto.newBuilder();
+ }
+
+ public PartitionMethodDesc(String tableId, CatalogProtos.PartitionType partitionType, String expression,
+ Schema expressionSchema) {
+ this.tableId = tableId;
+ this.partitionType = partitionType;
+ this.expression = expression;
+ this.expressionSchema = expressionSchema;
+ }
+
+ public PartitionMethodDesc(CatalogProtos.PartitionMethodProto proto) {
+ this(proto.getTableId(), proto.getPartitionType(), proto.getExpression(), new Schema(proto.getExpressionSchema()));
+ }
+
+ public String getTableId() {
+ return tableId;
+ }
+
+ public String getExpression() {
+ return expression;
+ }
+
+ public Schema getExpressionSchema() {
+ return expressionSchema;
+ }
+
+ public CatalogProtos.PartitionType getPartitionType() {
+ return partitionType;
+ }
+
+ public void setTableId(String tableId) {
+ this.tableId = tableId;
+ }
+
+ public void setExpressionSchema(Schema expressionSchema) {
+ this.expressionSchema = expressionSchema;
+ }
+
+ public void setPartitionType(CatalogProtos.PartitionType partitionsType) {
+ this.partitionType = partitionsType;
+ }
+
+ public void setExpression(String expression) {
+ this.expression = expression;
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if(object instanceof PartitionMethodDesc) {
+ PartitionMethodDesc other = (PartitionMethodDesc) object;
+ boolean eq = tableId.equals(other.tableId);
+ eq = eq && partitionType.equals(other.partitionType);
+ eq = eq && expression.equals(other.expression);
+ eq = eq && TUtil.checkEquals(expressionSchema, other.expressionSchema);
+ return eq;
+ }
+
+ return false;
+ }
+
+ @Override
+ public CatalogProtos.PartitionMethodProto getProto() {
+ if(builder == null) {
+ builder = CatalogProtos.PartitionMethodProto.newBuilder();
+ }
+ builder.setTableId(tableId);
+ builder.setPartitionType(partitionType);
+ builder.setExpression(expression);
+ builder.setExpressionSchema(expressionSchema.getProto());
+ return builder.build();
+ }
+
+
+ public Object clone() throws CloneNotSupportedException {
+ PartitionMethodDesc desc = (PartitionMethodDesc) super.clone();
+ desc.builder = builder;
+ desc.tableId = tableId;
+ desc.partitionType = partitionType;
+ desc.expression = expression;
+ desc.expressionSchema = (Schema) expressionSchema.clone();
+ return desc;
+ }
+
+ public String toString() {
+ Gson gson = new GsonBuilder().setPrettyPrinting().
+ excludeFieldsWithoutExposeAnnotation().create();
+ return gson.toJson(this);
+ }
+
+ @Override
+ public String toJson() {
+ return CatalogGsonHelper.toJson(this, PartitionMethodDesc.class);
+ }
+
+ public static PartitionMethodDesc fromJson(String strVal) {
+ return strVal != null ? CatalogGsonHelper.fromJson(strVal, PartitionMethodDesc.class) : null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/Specifier.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/Specifier.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/Specifier.java
deleted file mode 100644
index feb8a33..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/Specifier.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.catalog.partition;
-
-import com.google.common.base.Objects;
-import com.google.gson.Gson;
-import com.google.gson.annotations.Expose;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.tajo.catalog.json.CatalogGsonHelper;
-import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.json.GsonObject;
-import org.apache.tajo.util.TUtil;
-
-public class Specifier implements ProtoObject<CatalogProtos.SpecifierProto>, Cloneable,
- GsonObject {
-
- private static final Log LOG = LogFactory.getLog(Specifier.class);
- protected CatalogProtos.SpecifierProto.Builder builder = null;
-
-
- @Expose protected String name;
- @Expose protected String expressions;
-
- public Specifier() {
- builder = CatalogProtos.SpecifierProto.newBuilder();
- }
-
- public Specifier(String name) {
- this();
- this.name = name;
- }
-
- public Specifier(String name, String expressions) {
- this();
- this.name = name;
- this.expressions = expressions;
- }
-
- public Specifier(CatalogProtos.SpecifierProto proto) {
- this();
- this.name = proto.getName().toLowerCase();
- this.expressions = proto.getExpressions();
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getExpressions() {
- return expressions;
- }
-
- public void setExpressions(String expressions) {
- this.expressions = expressions;
- }
-
- public boolean equals(Object o) {
- if (o instanceof Specifier) {
- Specifier other = (Specifier)o;
- boolean eq = TUtil.checkEquals(this.name, other.name);
- eq = eq && TUtil.checkEquals(this.expressions, other.expressions);
- return eq;
- }
- return false;
- }
-
- public int hashCode() {
- return Objects.hashCode(this.name, this.expressions);
-
- }
-
- public Object clone() throws CloneNotSupportedException {
- Specifier clone = (Specifier) super.clone();
- clone.builder = CatalogProtos.SpecifierProto.newBuilder();
- clone.name = this.name;
- clone.expressions = this.expressions;
- return clone;
- }
-
- public String toString() {
- Gson gson = CatalogGsonHelper.getPrettyInstance();
- return gson.toJson(this);
- }
-
- @Override
- public CatalogProtos.SpecifierProto getProto() {
- if(builder == null) {
- builder = CatalogProtos.SpecifierProto.newBuilder();
- }
-
- if(this.name != null) {
- builder.setName(this.name);
- }
-
- if(this.expressions != null) {
- builder.setExpressions(this.expressions);
- }
-
- return builder.build();
- }
-
- @Override
- public String toJson() {
- return CatalogGsonHelper.toJson(this, Specifier.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index beff4c7..47792a6 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -50,7 +50,7 @@ enum CompressType {
LZ = 6;
}
-enum PartitionsType {
+enum PartitionType {
RANGE = 0;
HASH = 1;
LIST = 2;
@@ -113,7 +113,7 @@ message TableDescProto {
required TableProto meta = 3;
required SchemaProto schema = 4;
optional TableStatsProto stats = 5;
- optional PartitionDescProto partitions = 6;
+ optional PartitionMethodProto partition = 6;
}
enum FunctionType {
@@ -237,15 +237,23 @@ message SortSpecProto {
optional bool nullFirst = 3 [default = false];
}
-message PartitionDescProto {
- required PartitionsType partitionsType = 1;
- optional SchemaProto schema = 2;
- optional int32 numPartitions = 3;
- repeated SpecifierProto specifiers = 4;
- optional bool isOmitValues = 5;
+
+message PartitionsProto {
+ repeated PartitionDescProto partition = 1;
+}
+
+message PartitionMethodProto {
+ required string tableId = 1;
+ required PartitionType partitionType = 2;
+ required string expression = 3;
+ required SchemaProto expressionSchema = 4;
}
-message SpecifierProto {
- optional string name = 1;
- optional string expressions = 2;
+message PartitionDescProto {
+ required string tableId = 1;
+ optional string partitionName = 2;
+ required int32 ordinalPosition = 3;
+ optional string partitionValue = 4;
+ optional string path = 5;
}
+
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index e24412e..ab20f42 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -30,7 +30,8 @@ import org.apache.hcatalog.data.Pair;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.partition.PartitionDesc;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
@@ -39,6 +40,8 @@ import org.apache.tajo.exception.InternalException;
import java.io.IOException;
import java.util.*;
+import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType;
+
public class HCatalogStore extends CatalogConstants implements CatalogStore {
protected final Log LOG = LogFactory.getLog(getClass());
protected Configuration conf;
@@ -95,7 +98,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
@Override
- public final TableDesc getTable(final String name) throws IOException {
+ public final CatalogProtos.TableDescProto getTable(final String name) throws IOException {
String dbName = null, tableName = null;
Pair<String, String> tablePair = null;
org.apache.hadoop.hive.ql.metadata.Table table = null;
@@ -105,7 +108,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
org.apache.tajo.catalog.Schema schema = null;
Options options = null;
TableStats stats = null;
- PartitionDesc partitions = null;
+ PartitionMethodDesc partitions = null;
// get db name and table name.
try {
@@ -183,15 +186,24 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
// set partition keys
if (table.getPartitionKeys() != null) {
+ Schema expressionSchema = new Schema();
+ StringBuilder sb = new StringBuilder();
if (table.getPartitionKeys().size() > 0) {
- partitions = new PartitionDesc();
List<FieldSchema> partitionKeys = table.getPartitionKeys();
for(int i = 0; i < partitionKeys.size(); i++) {
FieldSchema fieldSchema = partitionKeys.get(i);
TajoDataTypes.Type dataType = HCatalogUtil.getTajoFieldType(fieldSchema.getType().toString());
- partitions.addColumn(new Column(fieldSchema.getName(), dataType));
+ expressionSchema.addColumn(new Column(fieldSchema.getName(), dataType));
+ if (i > 0) {
+ sb.append(",");
+ }
+ sb.append(fieldSchema.getName());
}
- partitions.setPartitionsType(CatalogProtos.PartitionsType.COLUMN);
+ partitions = new PartitionMethodDesc(
+ tableName,
+ PartitionType.COLUMN,
+ sb.toString(),
+ expressionSchema);
}
}
} finally {
@@ -204,9 +216,9 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
tableDesc.setStats(stats);
}
if (partitions != null) {
- tableDesc.setPartitions(partitions);
+ tableDesc.setPartitionMethod(partitions);
}
- return tableDesc;
+ return tableDesc.getProto();
}
@@ -247,14 +259,14 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
@Override
- public final void addTable(final TableDesc tableDesc) throws IOException {
+ public final void addTable(final CatalogProtos.TableDescProto tableDesc) throws IOException {
String dbName = null, tableName = null;
Pair<String, String> tablePair = null;
HCatalogStoreClientPool.HCatalogStoreClient client = null;
// get db name and table name.
try {
- tablePair = HCatUtil.getDbAndTableName(tableDesc.getName());
+ tablePair = HCatUtil.getDbAndTableName(tableDesc.getId());
dbName = tablePair.first;
tableName = tablePair.second;
} catch (IOException ioe) {
@@ -281,12 +293,12 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
//sd.setLocation(tableDesc.getPath().toString());
// set column information
- ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(tableDesc.getSchema().getColumns
- ().size());
- for (Column col : tableDesc.getSchema().getColumns()) {
- cols.add(new FieldSchema(col.getColumnName(), HCatalogUtil.getHiveFieldType(col
- .getDataType
- ().getType().name()), ""));
+ ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(tableDesc.getSchema().getFieldsCount());
+ for (CatalogProtos.ColumnProto col : tableDesc.getSchema().getFieldsList()) {
+ cols.add(new FieldSchema(
+ col.getColumnName(),
+ HCatalogUtil.getHiveFieldType(col.getDataType().getType().name()),
+ ""));
}
sd.setCols(cols);
@@ -350,6 +362,57 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
client.release();
}
}
+
+ @Override
+ public void addPartitionMethod(CatalogProtos.PartitionMethodProto partitionMethodProto) throws IOException {
+ // TODO - not implemented yet
+ }
+
+ @Override
+ public CatalogProtos.PartitionMethodProto getPartitionMethod(String tableName) throws IOException {
+ return null; // TODO - not implemented yet
+ }
+
+ @Override
+ public boolean existPartitionMethod(String tableName) throws IOException {
+ return false; // TODO - not implemented yet
+ }
+
+ @Override
+ public void delPartitionMethod(String tableName) throws IOException {
+ // TODO - not implemented yet
+ }
+
+ @Override
+ public void addPartitions(CatalogProtos.PartitionsProto partitionsProto) throws IOException {
+ // TODO - not implemented yet
+ }
+
+ @Override
+ public void addPartition(CatalogProtos.PartitionDescProto partitionDescProto) throws IOException {
+ // TODO - not implemented yet
+ }
+
+ @Override
+ public CatalogProtos.PartitionsProto getPartitions(String tableName) throws IOException {
+ return null; // TODO - not implemented yet
+ }
+
+ @Override
+ public CatalogProtos.PartitionDescProto getPartition(String partitionName) throws IOException {
+ return null; // TODO - not implemented yet
+ }
+
+ @Override
+ public void delPartition(String partitionName) throws IOException {
+ // TODO - not implemented yet
+ }
+
+ @Override
+ public void delPartitions(String tableName) throws IOException {
+ // TODO - not implemented yet
+ }
+
@Override
public final void addFunction(final FunctionDesc func) throws IOException {
// TODO - not implemented yet
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index 225d92e..e0afd7a 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -241,7 +241,7 @@ public class TestHCatalogStore {
@Test
public void testGetTable() throws Exception {
- TableDesc table = store.getTable(DB_NAME + "." + CUSTOMER);
+ TableDesc table = new TableDesc(store.getTable(DB_NAME + "." + CUSTOMER));
List<Column> columns = table.getSchema().getColumns();
assertEquals(DB_NAME + "." + CUSTOMER, table.getName());
@@ -262,9 +262,9 @@ public class TestHCatalogStore {
assertEquals(TajoDataTypes.Type.TEXT, columns.get(6).getDataType().getType());
assertEquals("c_comment", columns.get(7).getColumnName());
assertEquals(TajoDataTypes.Type.TEXT, columns.get(7).getDataType().getType());
- assertNull(table.getPartitions());
+ assertNull(table.getPartitionMethod());
- table = store.getTable(DB_NAME + "." + NATION);
+ table = new TableDesc(store.getTable(DB_NAME + "." + NATION));
columns = table.getSchema().getColumns();
assertEquals(DB_NAME + "." + NATION, table.getName());
assertEquals(5, columns.size());
@@ -278,9 +278,9 @@ public class TestHCatalogStore {
assertEquals(TajoDataTypes.Type.TEXT, columns.get(3).getDataType().getType());
assertEquals("type", columns.get(4).getColumnName());
assertEquals(TajoDataTypes.Type.TEXT, columns.get(4).getDataType().getType());
- assertNotNull(table.getPartitions());
- assertEquals("type", table.getPartitions().getSchema().getColumn(0).getColumnName());
- assertEquals(CatalogProtos.PartitionsType.COLUMN, table.getPartitions().getPartitionsType());
+ assertNotNull(table.getPartitionMethod());
+ assertEquals("type", table.getPartitionMethod().getExpressionSchema().getColumn(0).getColumnName());
+ assertEquals(CatalogProtos.PartitionType.COLUMN, table.getPartitionMethod().getPartitionType());
}
@Test
@@ -293,10 +293,10 @@ public class TestHCatalogStore {
@Test
public void testDeleteTable() throws Exception {
- TableDesc table = store.getTable(DB_NAME + "." + CUSTOMER);
+ TableDesc table = new TableDesc(store.getTable(DB_NAME + "." + CUSTOMER));
Path customerPath = table.getPath();
- table = store.getTable(DB_NAME + "." + NATION);
+ table = new TableDesc(store.getTable(DB_NAME + "." + NATION));
Path nationPath = table.getPath();
store.deleteTable(DB_NAME + "." + CUSTOMER);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/eb563add/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index c69487a..5d01a16 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -199,11 +199,8 @@ public class CatalogServer extends AbstractService {
if (!store.existTable(tableId)) {
throw new NoSuchTableException(tableId);
}
- TableDesc desc = store.getTable(tableId);
- SchemaProto schemaProto = desc.getSchema().getProto();
- SchemaProto qualifiedSchema = CatalogUtil.getQualfiedSchema(tableId, schemaProto);
- desc.setSchema(new Schema(qualifiedSchema));
- return desc.getProto();
+
+ return store.getTable(tableId);
} catch (IOException ioe) {
// TODO - handle exception
LOG.error(ioe);
@@ -244,32 +241,21 @@ public class CatalogServer extends AbstractService {
}
@Override
- public BoolProto addTable(RpcController controller, TableDescProto tableDesc)
+ public BoolProto addTable(RpcController controller, TableDescProto proto)
throws ServiceException {
wlock.lock();
try {
- if (store.existTable(tableDesc.getId())) {
- throw new AlreadyExistsTableException(tableDesc.getId());
- }
-
- // rewrite schema
- TableDescProto.Builder descBuilder = TableDescProto.newBuilder(tableDesc);
- descBuilder.setMeta(tableDesc.getMeta());
- descBuilder.setSchema(tableDesc.getSchema());
-
- if( tableDesc.getPartitions() != null
- && !tableDesc.getPartitions().toString().isEmpty()) {
- descBuilder.setPartitions(tableDesc.getPartitions());
+ if (store.existTable(proto.getId().toLowerCase())) {
+ throw new AlreadyExistsTableException(proto.getId());
}
-
- store.addTable(new TableDesc(descBuilder.build()));
+ store.addTable(proto);
} catch (IOException ioe) {
LOG.error(ioe.getMessage(), ioe);
return BOOL_FALSE;
} finally {
wlock.unlock();
- LOG.info("Table " + tableDesc.getId() + " is added to the catalog ("
+ LOG.info("Table " + proto.getId() + " is added to the catalog ("
+ bindAddressStr + ")");
}
@@ -313,6 +299,78 @@ public class CatalogServer extends AbstractService {
}
@Override
+ public PartitionMethodProto getPartitionMethodByTableName(RpcController controller,
+ StringProto name)
+ throws ServiceException {
+ rlock.lock();
+ try {
+ String tableId = name.getValue().toLowerCase();
+ return store.getPartitionMethod(tableId);
+ } catch (IOException ioe) {
+ // TODO - handle exception
+ LOG.error(ioe);
+ return null;
+ } finally {
+ rlock.unlock();
+ }
+ }
+
+ @Override
+ public BoolProto existPartitionMethod(RpcController controller, StringProto tableName)
+ throws ServiceException {
+ rlock.lock();
+ try {
+ String tableId = tableName.getValue().toLowerCase();
+ return BoolProto.newBuilder().setValue(
+ store.existPartitionMethod(tableId)).build();
+ } catch (IOException e) {
+ LOG.error(e);
+ return BoolProto.newBuilder().setValue(false).build();
+ } finally {
+ rlock.unlock();
+ }
+ }
+
+ @Override
+ public BoolProto delPartitionMethod(RpcController controller, StringProto request)
+ throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public BoolProto addPartitions(RpcController controller, PartitionsProto request)
+ throws ServiceException {
+
+ return null;
+ }
+
+ @Override
+ public BoolProto addPartition(RpcController controller, PartitionDescProto request)
+ throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public PartitionDescProto getPartitionByPartitionName(RpcController controller,
+ StringProto request)
+ throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public PartitionsProto getPartitionsByTableName(RpcController controller,
+ StringProto request)
+ throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public PartitionsProto delAllPartitions(RpcController controller, StringProto request)
+ throws ServiceException {
+ return null;
+ }
+
+ @Override
public BoolProto addIndex(RpcController controller, IndexDescProto indexDesc)
throws ServiceException {
rlock.lock();