You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by bl...@apache.org on 2015/10/16 15:36:22 UTC
[3/3] tajo git commit: TAJO-1926: Disable partition pruning using
catalog temporarily.
TAJO-1926: Disable partition pruning using catalog temporarily.
Closes #824
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/13569f00
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/13569f00
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/13569f00
Branch: refs/heads/branch-0.11.0
Commit: 13569f0054e73ceeedf20d6e946626c711486f49
Parents: 77ac53c
Author: JaeHwa Jung <bl...@apache.org>
Authored: Fri Oct 16 22:33:47 2015 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Fri Oct 16 22:33:47 2015 +0900
----------------------------------------------------------------------
CHANGES | 5 +-
.../tajo/catalog/AbstractCatalogClient.java | 54 +-
.../src/main/proto/CatalogProtocol.proto | 4 +-
.../apache/tajo/catalog/CatalogConstants.java | 3 +-
.../org/apache/tajo/catalog/CatalogService.java | 10 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 30 +-
.../org/apache/tajo/catalog/DDLBuilder.java | 38 +-
.../tajo/catalog/partition/PartitionDesc.java | 30 +-
.../src/main/proto/CatalogProtos.proto | 16 +-
.../tajo/catalog/store/HiveCatalogStore.java | 220 +------
.../catalog/store/TestHiveCatalogStore.java | 138 -----
tajo-catalog/tajo-catalog-server/pom.xml | 8 -
.../org/apache/tajo/catalog/CatalogServer.java | 72 +--
.../tajo/catalog/store/AbstractDBStore.java | 307 +---------
.../apache/tajo/catalog/store/CatalogStore.java | 88 +--
.../src/main/resources/schemas/derby/derby.xml | 4 +-
.../main/resources/schemas/mariadb/mariadb.xml | 4 +-
.../src/main/resources/schemas/mysql/mysql.xml | 4 +-
.../main/resources/schemas/oracle/oracle.xml | 4 +-
.../resources/schemas/postgresql/postgresql.xml | 4 +-
.../org/apache/tajo/catalog/TestCatalog.java | 109 +---
.../TestCatalogAgainstCaseSensitivity.java | 2 +-
.../org/apache/tajo/cli/tools/TajoDump.java | 13 +-
.../apache/tajo/client/CatalogAdminClient.java | 2 +-
.../tajo/client/CatalogAdminClientImpl.java | 2 +-
.../org/apache/tajo/client/TajoClientImpl.java | 5 +-
.../apache/tajo/exception/ErrorMessages.java | 1 -
.../apache/tajo/exception/ExceptionUtil.java | 1 -
.../apache/tajo/exception/ReturnStateUtil.java | 4 -
.../UndefinedPartitionMethodException.java | 4 +-
tajo-common/src/main/proto/errors.proto | 1 -
.../planner/TestEvalNodeToExprConverter.java | 406 -------------
.../planner/TestPartitionedTableRewriter.java | 432 ++++++++++++++
.../tajo/engine/query/TestAlterTable.java | 14 +-
.../tajo/engine/query/TestTablePartitions.java | 517 +----------------
.../TestTajoDump/testPartitionsDump.result | 7 +-
.../tajo/master/TajoMasterClientService.java | 2 +-
.../apache/tajo/master/exec/DDLExecutor.java | 27 +-
.../java/org/apache/tajo/querymaster/Query.java | 23 +-
.../apache/tajo/plan/expr/AlgebraicUtil.java | 148 +----
.../rewrite/rules/PartitionedTableRewriter.java | 172 ++----
.../tajo/plan/util/EvalNodeToExprConverter.java | 297 ----------
.../util/PartitionFilterAlgebraVisitor.java | 573 -------------------
43 files changed, 599 insertions(+), 3206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 04d4ab5..8863224 100644
--- a/CHANGES
+++ b/CHANGES
@@ -285,9 +285,10 @@ Release 0.11.0 - unreleased
BUG FIXES
+ TAJO-1926: Disable partition pruning using catalog temporarily. (jaehwa)
+
TAJO-1924: Repair partition need to calculate partition volume. (jaehwa)
-
TAJO-1923: Selecting on information_schema.table_stats throws an internal
error. (jihoon)
@@ -722,8 +723,6 @@ Release 0.11.0 - unreleased
TAJO-1887: Disable the alter table add partition statement temporarily.
(jaehwa)
- TAJO-1493: Make partition pruning based on catalog informations. (jaehwa)
-
TAJO-1673: Implement recover partitions. (jaehwa)
TAJO-1465: Add ORCFileAppender to write into ORCFile table.
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index ba4f37e..7af110c 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@ -407,29 +407,6 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable
}
@Override
- public boolean existPartitions(String databaseName, String tableName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException {
-
- try {
- final BlockingInterface stub = getStub();
- final TableIdentifierProto request = buildTableIdentifier(databaseName, tableName);
- final ReturnState state = stub.existsPartitions(null, request);
-
- if (isThisError(state, UNDEFINED_PARTITIONS)) {
- return false;
- }
- throwsIfThisError(state, UndefinedDatabaseException.class);
- throwsIfThisError(state, UndefinedTableException.class);
- throwsIfThisError(state, UndefinedPartitionMethodException.class);
- ensureOk(state);
- return true;
-
- } catch (ServiceException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
public final PartitionDescProto getPartition(final String databaseName, final String tableName,
final String partitionName)
throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException,
@@ -457,16 +434,15 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable
}
@Override
- public final List<PartitionDescProto> getPartitionsOfTable(final String databaseName, final String tableName) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException {
+ public final List<PartitionDescProto> getPartitions(final String databaseName, final String tableName) {
try {
final BlockingInterface stub = getStub();
- final TableIdentifierProto request = buildTableIdentifier(databaseName, tableName);
+ final PartitionIdentifierProto request = PartitionIdentifierProto.newBuilder()
+ .setDatabaseName(databaseName)
+ .setTableName(tableName)
+ .build();
final GetPartitionsResponse response = stub.getPartitionsByTableName(null, request);
- throwsIfThisError(response.getState(), UndefinedDatabaseException.class);
- throwsIfThisError(response.getState(), UndefinedTableException.class);
- throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class);
ensureOk(response.getState());
return response.getPartitionList();
@@ -476,26 +452,6 @@ public abstract class AbstractCatalogClient implements CatalogService, Closeable
}
@Override
- public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UnsupportedException {
- try {
- final BlockingInterface stub = getStub();
- GetPartitionsResponse response = stub.getPartitionsByAlgebra(null, request);
-
- throwsIfThisError(response.getState(), UndefinedDatabaseException.class);
- throwsIfThisError(response.getState(), UndefinedTableException.class);
- throwsIfThisError(response.getState(), UndefinedPartitionMethodException.class);
- throwsIfThisError(response.getState(), UnsupportedException.class);
- ensureOk(response.getState());
- return response.getPartitionList();
- } catch (ServiceException e) {
- LOG.error(e.getMessage(), e);
- return null;
- }
- }
-
- @Override
public List<TablePartitionProto> getAllPartitions() {
try {
final BlockingInterface stub = getStub();
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
index a3a904b..8cc8e2f 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
+++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
@@ -120,12 +120,10 @@ service CatalogProtocolService {
rpc getPartitionMethodByTableName(TableIdentifierProto) returns (GetPartitionMethodResponse);
rpc existPartitionMethod(TableIdentifierProto) returns (ReturnState);
- rpc existsPartitions(TableIdentifierProto) returns (ReturnState);
rpc getPartitionByPartitionName(PartitionIdentifierProto) returns (GetPartitionDescResponse);
- rpc getPartitionsByTableName(TableIdentifierProto) returns (GetPartitionsResponse);
+ rpc getPartitionsByTableName(PartitionIdentifierProto) returns (GetPartitionsResponse);
rpc getAllPartitions(NullProto) returns (GetTablePartitionsResponse);
rpc addPartitions(AddPartitionsProto) returns (ReturnState);
- rpc getPartitionsByAlgebra(PartitionsByAlgebraProto) returns (GetPartitionsResponse);
rpc createIndex(IndexDescProto) returns (ReturnState);
rpc dropIndex(IndexNameProto) returns (ReturnState);
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
index f2acf98..721bcf1 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
@@ -59,7 +59,6 @@ public class CatalogConstants {
public static final String COL_PARTITIONS_PK = "PARTITION_ID";
public static final String COL_COLUMN_NAME = "COLUMN_NAME";
public static final String COL_PARTITION_VALUE = "PARTITION_VALUE";
- public static final String COL_PARTITION_BYTES = "NUM_BYTES";
-
+
public static final String INFORMATION_SCHEMA_DB_NAME = "information_schema";
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
index 54d22e7..9173dc1 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
@@ -168,19 +168,11 @@ public interface CatalogService {
boolean existPartitionMethod(String databaseName, String tableName) throws UndefinedTableException,
UndefinedDatabaseException;
- boolean existPartitions(String databaseName, String tableName) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException;
-
PartitionDescProto getPartition(String databaseName, String tableName, String partitionName)
throws UndefinedPartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException,
UndefinedTableException;
- List<PartitionDescProto> getPartitionsOfTable(String databaseName, String tableName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException;
-
- List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UnsupportedException;
+ List<PartitionDescProto> getPartitions(String databaseName, String tableName);
List<TablePartitionProto> getAllPartitions();
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 12fb23d..d4ff69d 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -811,33 +811,16 @@ public class CatalogUtil {
/**
* Converts passed parameters to a AlterTableDesc. This method would be called when adding a partition or dropping
* a table. This creates AlterTableDesc that is a wrapper class for protocol buffer.
- * *
- * @param tableName
- * @param columns
- * @param values
- * @param path
- * @param alterTableType
- * @return
- */
- public static AlterTableDesc addOrDropPartition(String tableName, String[] columns, String[] values, @Nullable
- String path, AlterTableType alterTableType) {
- return addOrDropPartition(tableName, columns, values, path, alterTableType, 0L);
- }
- /**
- * Converts passed parameters to a AlterTableDesc. This method would be called when adding a partition or dropping
- * a table. This creates AlterTableDesc that is a wrapper class for protocol buffer.
*
* @param tableName table name
* @param columns partition column names
* @param values partition values
- * @param path partition directory path
+ * @param location partition location
* @param alterTableType ADD_PARTITION or DROP_PARTITION
- * @param numBytes contents length
* @return AlterTableDesc
*/
- public static AlterTableDesc addOrDropPartition(String tableName, String[] columns, String[] values,
- @Nullable String path, AlterTableType alterTableType, long numBytes) {
-
+ public static AlterTableDesc addOrDropPartition(String tableName, String[] columns,
+ String[] values, String location, AlterTableType alterTableType) {
final AlterTableDesc alterTableDesc = new AlterTableDesc();
alterTableDesc.setTableName(tableName);
@@ -847,11 +830,8 @@ public class CatalogUtil {
partitionDesc.setPartitionKeys(pair.getFirst());
partitionDesc.setPartitionName(pair.getSecond());
- if (alterTableType.equals(AlterTableType.ADD_PARTITION)) {
- if (path != null) {
- partitionDesc.setPath(path);
- }
- partitionDesc.setNumBytes(numBytes);
+ if (alterTableType.equals(AlterTableType.ADD_PARTITION) && location != null) {
+ partitionDesc.setPath(location);
}
alterTableDesc.setPartitionDesc(partitionDesc);
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index 7159aae..4b56e8c 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -18,11 +18,14 @@
package org.apache.tajo.catalog;
+import com.google.common.base.Function;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.StringUtils;
-import java.io.File;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -115,19 +118,31 @@ public class DDLBuilder {
sb.append(" USING " + CatalogUtil.getBackwardCompitableDataFormat(meta.getDataFormat()));
}
- private static void buildWithClause(StringBuilder sb, TableMeta meta) {
+ private static void buildWithClause(final StringBuilder sb, TableMeta meta) {
KeyValueSet options = meta.getOptions();
if (options != null && options.size() > 0) {
- boolean first = true;
+
sb.append(" WITH (");
- for (Map.Entry<String, String> entry : meta.getOptions().getAllKeyValus().entrySet()) {
- if (first) {
- first = false;
- } else {
- sb.append(", ");
+
+ // sort table properties in an lexicographic order of the property keys.
+ Map.Entry<String, String>[] entries = meta.getOptions().getAllKeyValus().entrySet().toArray(
+ new Map.Entry[meta.getOptions().size()]);
+
+ Arrays.sort(entries, new Comparator<Map.Entry<String, String>>() {
+ @Override
+ public int compare(Map.Entry<String, String> o1, Map.Entry<String, String> o2) {
+ return o1.getKey().compareTo(o2.getKey());
}
- sb.append("'").append(entry.getKey()).append("'='").append(entry.getValue()).append("'");
- }
+ });
+
+ // Join all properties by comma (',')
+ sb.append(StringUtils.join(entries, ", ", new Function<Map.Entry<String, String>, String>() {
+ @Override
+ public String apply(Map.Entry<String, String> e) {
+ return "'" + e.getKey() + "'='" + e.getValue() + "'";
+ }
+ }));
+
sb.append(")");
}
}
@@ -167,7 +182,8 @@ public class DDLBuilder {
List<Column> colums = table.getPartitionMethod().getExpressionSchema().getAllColumns();
- String[] splitPartitionName = partition.getPartitionName().split(File.separator);
+ String[] splitPartitionName = partition.getPartitionName().split("/");
+
for(int i = 0; i < splitPartitionName.length; i++) {
String[] partitionColumnValue = splitPartitionName[i].split("=");
if (i > 0) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
index e41ac85..7287fce 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
@@ -27,7 +27,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.json.GsonObject;
import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto;
-import org.apache.tajo.util.TUtil;
import java.util.List;
@@ -57,7 +56,6 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro
@Expose protected String partitionName;
@Expose protected List<PartitionKeyProto> partitionKeys;
@Expose protected String path; //optional
- @Expose private Long numBytes = null; // optional
private CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder();
@@ -85,30 +83,27 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro
this.partitionKeys = partitionKeys;
}
- public Long getNumBytes() {
- return numBytes;
- }
-
- public void setNumBytes(Long numBytes) {
- this.numBytes = numBytes;
- }
-
public int hashCode() {
- return Objects.hashCode(partitionName, partitionKeys, path, numBytes);
+ return Objects.hashCode(partitionName, partitionKeys, path);
}
public boolean equals(Object o) {
if (o instanceof PartitionDesc) {
PartitionDesc another = (PartitionDesc) o;
- boolean eq = this.partitionName.equals(another.partitionName);
- eq = eq && this.partitionKeys.equals(another.partitionKeys);
- eq = eq && this.path.equals(another.path);
- eq = eq && TUtil.checkEquals(this.numBytes, another.numBytes);
+ boolean eq = ((partitionName != null && another.partitionName != null
+ && partitionName.equals(another.partitionName)) ||
+ (partitionName == null && another.partitionName == null));
+ eq = eq && ((partitionKeys != null && another.partitionKeys != null
+ && partitionKeys.equals(another.partitionKeys))
+ || (partitionKeys == null && another.partitionKeys == null));
+ eq = eq && ((path != null && another.path != null && path.equals(another.path)) ||
+ (path == null && another.path == null));
return eq;
}
return false;
}
+
@Override
public CatalogProtos.PartitionDescProto getProto() {
if (builder == null) {
@@ -130,10 +125,6 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro
builder.setPath(this.path);
}
- if(this.numBytes != null) {
- builder.setNumBytes(this.numBytes);
- }
-
return builder.build();
}
@@ -158,7 +149,6 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro
desc.partitionName = partitionName;
desc.partitionKeys = partitionKeys;
desc.path = path;
- desc.numBytes = numBytes;
return desc;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 9a7b66a..e89c672 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -248,12 +248,12 @@ message PartitionDescProto {
repeated PartitionKeyProto partitionKeys = 2;
optional string path = 3;
optional int32 id = 4;
- optional int64 numBytes = 5;
}
message PartitionKeyProto {
required string columnName = 1;
- required string partitionValue = 2;
+ optional string parentColumnName = 2;
+ required string partitionValue = 3;
}
message PartitionIdentifierProto {
@@ -262,18 +262,6 @@ message PartitionIdentifierProto {
optional string partitionName = 3;
}
-message PartitionsByAlgebraProto {
- required string databaseName = 1;
- required string tableName = 2;
- required string algebra = 3; // json object which contains algebra expressions
-}
-
-message PartitionsByFilterProto {
- required string databaseName = 1;
- required string tableName = 2;
- required string filter = 3; // filter string: (col1 ='1' or col1 = '100') and col3 > 20
-}
-
message TablespaceProto {
required string spaceName = 1;
required string uri = 2;
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index e377502..139e650 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -25,7 +25,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -35,9 +34,6 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.TajoConstants;
-import org.apache.tajo.algebra.Expr;
-import org.apache.tajo.algebra.IsNullPredicate;
-import org.apache.tajo.algebra.JsonHelper;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -46,14 +42,11 @@ import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.*;
-import org.apache.tajo.plan.expr.AlgebraicUtil;
-import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
import org.apache.thrift.TException;
-import java.io.File;
import java.io.IOException;
import java.util.*;
@@ -700,7 +693,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
Table table = client.getHiveClient().getTable(databaseName, tableName);
List<FieldSchema> columns = table.getSd().getCols();
columns.add(new FieldSchema(columnProto.getName(),
- HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), ""));
+ HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), ""));
client.getHiveClient().alter_table(databaseName, tableName, table);
@@ -725,10 +718,6 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
partition.setDbName(databaseName);
partition.setTableName(tableName);
- Map<String, String> params = TUtil.newHashMap();
- params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(partitionDescProto.getNumBytes()));
- partition.setParameters(params);
-
List<String> values = Lists.newArrayList();
for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) {
values.add(keyProto.getPartitionValue());
@@ -856,204 +845,11 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
}
@Override
- public boolean existPartitions(String databaseName, String tableName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException {
-
- HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
- boolean result = false;
-
- try {
- client = clientPool.getClient();
- List<Partition> partitions = client.getHiveClient().listPartitionsByFilter(databaseName, tableName,
- "", (short) -1);
-
- if (partitions.size() > 0) {
- result = true;
- }
- } catch (Exception e) {
- throw new TajoInternalError(e);
- } finally {
- if (client != null) {
- client.release();
- }
- }
-
- return result;
- }
-
- @Override
- public List<CatalogProtos.PartitionDescProto> getPartitionsOfTable(String databaseName, String tableName)
- throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException {
- List<PartitionDescProto> list = null;
-
- try {
- if (!existDatabase(databaseName)) {
- throw new UndefinedDatabaseException(tableName);
- }
-
- if (!existTable(databaseName, tableName)) {
- throw new UndefinedTableException(tableName);
- }
-
- if (!existPartitionMethod(databaseName, tableName)) {
- throw new UndefinedPartitionMethodException(tableName);
- }
-
- list = getPartitionsFromHiveMetaStore(databaseName, tableName, "");
- } catch (Exception se) {
- throw new TajoInternalError(se);
- }
-
- return list;
- }
-
- @Override
- public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, UnsupportedException {
-
- List<PartitionDescProto> list = null;
-
- try {
- String databaseName = request.getDatabaseName();
- String tableName = request.getTableName();
-
- if (!existDatabase(databaseName)) {
- throw new UndefinedDatabaseException(tableName);
- }
-
- if (!existTable(databaseName, tableName)) {
- throw new UndefinedTableException(tableName);
- }
-
- if (!existPartitionMethod(databaseName, tableName)) {
- throw new UndefinedPartitionMethodException(tableName);
- }
-
- TableDescProto tableDesc = getTable(databaseName, tableName);
- String filter = getFilter(databaseName, tableName, tableDesc.getPartition().getExpressionSchema().getFieldsList()
- , request.getAlgebra());
- list = getPartitionsFromHiveMetaStore(databaseName, tableName, filter);
- } catch (UnsupportedException ue) {
- throw ue;
- } catch (Exception se) {
- throw new TajoInternalError(se);
- }
-
- return list;
- }
-
- private String getFilter(String databaseName, String tableName, List<ColumnProto> partitionColumns
- , String json) throws TajoException {
-
- Expr[] exprs = null;
-
- if (json != null && !json.isEmpty()) {
- Expr algebra = JsonHelper.fromJson(json, Expr.class);
- exprs = AlgebraicUtil.toConjunctiveNormalFormArray(algebra);
- }
-
- PartitionFilterAlgebraVisitor visitor = new PartitionFilterAlgebraVisitor();
- visitor.setIsHiveCatalog(true);
-
- Expr[] filters = AlgebraicUtil.getRearrangedCNFExpressions(databaseName + "." + tableName, partitionColumns, exprs);
-
- StringBuffer sb = new StringBuffer();
-
- // Write join clause from second column to last column.
- Column target;
-
- String result;
- for (int i = 0; i < partitionColumns.size(); i++) {
- target = new Column(partitionColumns.get(i));
-
- if (!(filters[i] instanceof IsNullPredicate)) {
- visitor.setColumn(target);
- visitor.visit(null, new Stack<Expr>(), filters[i]);
- result = visitor.getResult();
-
- // If visitor build filter successfully, add filter to be used for executing hive api.
- if (result.length() > 0) {
- if (sb.length() > 0) {
- sb.append(" AND ");
- }
- sb.append(" ( ").append(result).append(" ) ");
- } else {
- throw new TajoInternalError("Filter does not exist : " + filters[i].toJson());
- }
- }
- }
-
- return sb.toString();
+ public List<CatalogProtos.PartitionDescProto> getPartitions(String databaseName,
+ String tableName) {
+ throw new UnsupportedOperationException();
}
- /**
- * Get list of partitions matching specified filter.
- *
- * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
- * Assume that an user want to give a condition WHERE (col1 ='1' or col1 = '100') and col3 > 20 .
- *
- * Then, the filter string would be written as following:
- * (col1 =\"1\" or col1 = \"100\") and col3 > 20
- *
- *
- * @param databaseName
- * @param tableName
- * @param filter
- * @return
- */
- private List<PartitionDescProto> getPartitionsFromHiveMetaStore(String databaseName, String tableName,
- String filter) {
- HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
- List<PartitionDescProto> partitions = null;
- TableDescProto tableDesc = null;
- List<ColumnProto> parititonColumns = null;
-
- try {
- partitions = TUtil.newList();
- client = clientPool.getClient();
-
- List<Partition> hivePartitions = client.getHiveClient().listPartitionsByFilter(databaseName, tableName
- , filter, (short) -1);
-
- tableDesc = getTable(databaseName, tableName);
- parititonColumns = tableDesc.getPartition().getExpressionSchema().getFieldsList();
-
- StringBuilder partitionName = new StringBuilder();
- for (Partition hivePartition : hivePartitions) {
- CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder();
- builder.setPath(hivePartition.getSd().getLocation());
-
- partitionName.delete(0, partitionName.length());
- for (int i = 0; i < parititonColumns.size(); i++) {
- if (i > 0) {
- partitionName.append(File.separator);
- }
- partitionName.append(CatalogUtil.extractSimpleName(parititonColumns.get(i).getName()));
- partitionName.append("=");
- partitionName.append(hivePartition.getValues().get(i));
- }
-
- builder.setPartitionName(partitionName.toString());
-
- Map<String, String> params = hivePartition.getParameters();
- if (params != null) {
- if (params.get(StatsSetupConst.TOTAL_SIZE) != null) {
- builder.setNumBytes(Long.parseLong(params.get(StatsSetupConst.TOTAL_SIZE)));
- }
- }
-
- partitions.add(builder.build());
- }
- } catch (Exception e) {
- throw new TajoInternalError(e);
- } finally {
- if (client != null) {
- client.release();
- }
- }
-
- return partitions;
- }
@Override
public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName,
@@ -1079,14 +875,6 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
keyBuilder.setColumnName(columnName);
keyBuilder.setPartitionValue(value);
builder.addPartitionKeys(keyBuilder);
-
- Map<String, String> params = partition.getParameters();
- if (params != null) {
- if (params.get(StatsSetupConst.TOTAL_SIZE) != null) {
- builder.setNumBytes(Long.parseLong(params.get(StatsSetupConst.TOTAL_SIZE)));
- }
- }
-
}
} catch (NoSuchObjectException e) {
return null;
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
index c843b21..af1b0b1 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
@@ -34,7 +34,6 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
-import org.apache.tajo.util.TUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -262,27 +261,10 @@ public class TestHiveCatalogStore {
}
testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150101");
- testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150102");
- testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150101");
testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150102");
- testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150101");
- testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150102");
-
- List<String> partitionNames = TUtil.newList();
- partitionNames.add("n_nationkey=40/n_date=20150801");
- partitionNames.add("n_nationkey=40/n_date=20150802");
- partitionNames.add("n_nationkey=50/n_date=20150801");
- partitionNames.add("n_nationkey=50/n_date=20150802");
- testAddPartitions(table1.getUri(), NATION, partitionNames);
-
- testGetPartitionsByAlgebra(DB_NAME, NATION);
testDropPartition(NATION, "n_nationkey=10/n_date=20150101");
- testDropPartition(NATION, "n_nationkey=10/n_date=20150102");
- testDropPartition(NATION, "n_nationkey=20/n_date=20150101");
testDropPartition(NATION, "n_nationkey=20/n_date=20150102");
- testDropPartition(NATION, "n_nationkey=30/n_date=20150101");
- testDropPartition(NATION, "n_nationkey=30/n_date=20150102");
CatalogProtos.PartitionDescProto partition = store.getPartition(DB_NAME, NATION, "n_nationkey=10/n_date=20150101");
assertNull(partition);
@@ -293,87 +275,6 @@ public class TestHiveCatalogStore {
store.dropTable(DB_NAME, NATION);
}
- private void testGetPartitionsByAlgebra(String databaseName, String tableName) throws Exception {
- String qfTableName = databaseName + "." + tableName;
-
- // Equals Operator
- CatalogProtos.PartitionsByAlgebraProto.Builder request = CatalogProtos.PartitionsByAlgebraProto.newBuilder();
- request.setDatabaseName(databaseName);
- request.setTableName(tableName);
-
- String algebra = "{\n" +
- " \"LeftExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"n_nationkey\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"10\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"n_date\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"20150101\",\n" +
- " \"ValueType\": \"String\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"OpType\": \"And\"\n" +
- "}";
-
- request.setAlgebra(algebra);
-
- List<CatalogProtos.PartitionDescProto> partitions = store.getPartitionsByAlgebra(request.build());
- assertNotNull(partitions);
- assertEquals(1, partitions.size());
-
- // OR
- algebra = "{\n" +
- " \"LeftExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"n_nationkey\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"20\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"n_nationkey\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"30\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"OpType\": \"Or\"\n" +
- "}";
-
- request.setAlgebra(algebra);
-
- partitions = store.getPartitionsByAlgebra(request.build());
- assertNotNull(partitions);
- assertEquals(4, partitions.size());
- }
private void testAddPartition(URI uri, String tableName, String partitionName) throws Exception {
AlterTableDesc alterTableDesc = new AlterTableDesc();
@@ -415,45 +316,6 @@ public class TestHiveCatalogStore {
}
}
- private void testAddPartitions(URI uri, String tableName, List<String> partitionNames) throws Exception {
- List<CatalogProtos.PartitionDescProto> partitions = TUtil.newList();
- for (String partitionName : partitionNames) {
- CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder();
- builder.setPartitionName(partitionName);
- Path path = new Path(uri.getPath(), partitionName);
- builder.setPath(path.toString());
-
- List<PartitionKeyProto> partitionKeyList = new ArrayList<PartitionKeyProto>();
- String[] split = partitionName.split("/");
- for(int i = 0; i < split.length; i++) {
- String[] eachPartitionName = split[i].split("=");
-
- PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder();
- keyBuilder.setColumnName(eachPartitionName[0]);
- keyBuilder.setPartitionValue(eachPartitionName[1]);
- partitionKeyList.add(keyBuilder.build());
- }
- builder.addAllPartitionKeys(partitionKeyList);
- partitions.add(builder.build());
- }
-
- store.addPartitions(DB_NAME, tableName, partitions, true);
-
- for (String partitionName : partitionNames) {
- CatalogProtos.PartitionDescProto resultDesc = store.getPartition(DB_NAME, NATION, partitionName);
- assertNotNull(resultDesc);
- assertEquals(resultDesc.getPartitionName(), partitionName);
- assertEquals(resultDesc.getPath(), uri.toString() + "/" + partitionName);
- assertEquals(resultDesc.getPartitionKeysCount(), 2);
-
- String[] split = partitionName.split("/");
- for (int i = 0; i < resultDesc.getPartitionKeysCount(); i++) {
- CatalogProtos.PartitionKeyProto keyProto = resultDesc.getPartitionKeys(i);
- String[] eachName = split[i].split("=");
- assertEquals(keyProto.getPartitionValue(), eachName[1]);
- }
- }
- }
private void testDropPartition(String tableName, String partitionName) throws Exception {
AlterTableDesc alterTableDesc = new AlterTableDesc();
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml
index 3f93236..eff5f8f 100644
--- a/tajo-catalog/tajo-catalog-server/pom.xml
+++ b/tajo-catalog/tajo-catalog-server/pom.xml
@@ -148,14 +148,6 @@
<artifactId>tajo-rpc-protobuf</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-algebra</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-plan</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index 552168d..5a63bbb 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -47,7 +47,6 @@ import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.rpc.BlockingRpcServer;
-import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListResponse;
@@ -933,38 +932,6 @@ public class CatalogServer extends AbstractService {
}
@Override
- public ReturnState existsPartitions(RpcController controller, TableIdentifierProto request) throws
- ServiceException {
-
- String dbName = request.getDatabaseName();
- String tbName = request.getTableName();
-
- // linked meta data do not support partition.
- // So, the request that wants to get partitions in this db will be failed.
- if (linkedMetadataManager.existsDatabase(dbName)) {
- return errUndefinedPartitionMethod(tbName);
- }
-
- if (metaDictionary.isSystemDatabase(dbName)) {
- return errUndefinedPartitionMethod(tbName);
- } else {
- rlock.lock();
- try {
- if (store.existPartitions(dbName, tbName)) {
- return OK;
- } else {
- return errUndefinedPartitions(tbName);
- }
- } catch (Throwable t) {
- printStackTraceIfError(LOG, t);
- return returnError(t);
- } finally {
- rlock.unlock();
- }
- }
- }
-
- @Override
public GetPartitionDescResponse getPartitionByPartitionName(RpcController controller,
PartitionIdentifierProto request)
throws ServiceException {
@@ -1015,7 +982,7 @@ public class CatalogServer extends AbstractService {
}
@Override
- public GetPartitionsResponse getPartitionsByTableName(RpcController controller, TableIdentifierProto request)
+ public GetPartitionsResponse getPartitionsByTableName(RpcController controller, PartitionIdentifierProto request)
throws ServiceException {
String dbName = request.getDatabaseName();
String tbName = request.getTableName();
@@ -1038,7 +1005,7 @@ public class CatalogServer extends AbstractService {
rlock.lock();
try {
- List<PartitionDescProto> partitions = store.getPartitionsOfTable(dbName, tbName);
+ List<PartitionDescProto> partitions = store.getPartitions(dbName, tbName);
GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder();
for (PartitionDescProto partition : partitions) {
@@ -1085,41 +1052,6 @@ public class CatalogServer extends AbstractService {
}
@Override
- public GetPartitionsResponse getPartitionsByAlgebra(RpcController controller,
- PartitionsByAlgebraProto request) throws ServiceException {
- String dbName = request.getDatabaseName();
- String tbName = request.getTableName();
-
- // linked meta data do not support partition.
- // So, the request that wants to get partitions in this db will be failed.
- if (linkedMetadataManager.existsDatabase(dbName)) {
- return GetPartitionsResponse.newBuilder().setState(errUndefinedPartitionMethod(tbName)).build();
- }
-
- if (metaDictionary.isSystemDatabase(dbName)) {
- return GetPartitionsResponse.newBuilder().setState(errUndefinedPartitionMethod(tbName)).build();
- } else {
- rlock.lock();
- try {
- GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder();
- List<PartitionDescProto> partitions = store.getPartitionsByAlgebra(request);
- builder.addAllPartition(partitions);
- builder.setState(OK);
- return builder.build();
- } catch (Throwable t) {
- printStackTraceIfError(LOG, t);
-
- return GetPartitionsResponse.newBuilder()
- .setState(returnError(t))
- .build();
-
- } finally {
- rlock.unlock();
- }
- }
- }
-
- @Override
public ReturnState addPartitions(RpcController controller, AddPartitionsProto request) {
TableIdentifierProto identifier = request.getTableIdentifier();
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index f9b04ef..4038c9d 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -25,27 +25,25 @@ import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tajo.algebra.Expr;
-import org.apache.tajo.algebra.JsonHelper;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.common.TajoDataTypes.*;
+import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.*;
import org.apache.tajo.util.JavaResourceUtil;
-import org.apache.tajo.plan.expr.*;
-import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import java.io.IOException;
import java.net.URI;
import java.sql.*;
-import java.sql.Date;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto;
@@ -59,8 +57,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
protected final String catalogUri;
protected final String insertPartitionSql = "INSERT INTO " + TB_PARTTIONS
- + "(" + COL_TABLES_PK + ", PARTITION_NAME, PATH, " + COL_PARTITION_BYTES
- + ") VALUES (?, ? , ?, ?)";
+ + "(" + COL_TABLES_PK + ", PARTITION_NAME, PATH) VALUES (?, ? , ?)";
protected final String insertPartitionKeysSql = "INSERT INTO " + TB_PARTTION_KEYS + "("
+ COL_PARTITIONS_PK + ", " + COL_TABLES_PK + ", "
@@ -1383,7 +1380,6 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt1.setInt(1, tableId);
pstmt1.setString(2, partition.getPartitionName());
pstmt1.setString(3, partition.getPath());
- pstmt1.setLong(4, partition.getNumBytes());
pstmt1.executeUpdate();
pstmt2 = conn.prepareStatement(insertPartitionKeysSql);
@@ -2137,7 +2133,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
PartitionDescProto.Builder builder = null;
try {
- String sql = "SELECT PATH, " + COL_PARTITIONS_PK + ", " + COL_PARTITION_BYTES + " FROM " + TB_PARTTIONS +
+ String sql = "SELECT PATH, " + COL_PARTITIONS_PK + " FROM " + TB_PARTTIONS +
" WHERE " + COL_TABLES_PK + " = ? AND PARTITION_NAME = ? ";
if (LOG.isDebugEnabled()) {
@@ -2155,7 +2151,6 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
builder.setId(res.getInt(COL_PARTITIONS_PK));
builder.setPath(res.getString("PATH"));
builder.setPartitionName(partitionName);
- builder.setNumBytes(res.getLong(COL_PARTITION_BYTES));
setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder);
} else {
throw new UndefinedPartitionException(partitionName);
@@ -2196,8 +2191,9 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
@Override
- public List<PartitionDescProto> getPartitionsOfTable(String databaseName, String tableName)
+ public List<PartitionDescProto> getPartitions(String databaseName, String tableName)
throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException {
+
Connection conn = null;
ResultSet res = null;
PreparedStatement pstmt = null;
@@ -2209,8 +2205,8 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
ensurePartitionTable(tableName, tableId);
try {
- String sql = "SELECT PATH, PARTITION_NAME, " + COL_PARTITIONS_PK + ", " + COL_PARTITION_BYTES
- + " FROM " + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? ";
+ String sql = "SELECT PATH, PARTITION_NAME, " + COL_PARTITIONS_PK + " FROM "
+ + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? ";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
@@ -2225,7 +2221,6 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
builder = PartitionDescProto.newBuilder();
builder.setPath(res.getString("PATH"));
builder.setPartitionName(res.getString("PARTITION_NAME"));
- builder.setNumBytes(res.getLong(COL_PARTITION_BYTES));
setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder);
partitions.add(builder.build());
}
@@ -2238,256 +2233,6 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
@Override
- public boolean existPartitions(String databaseName, String tableName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException {
-
- String sql = null;
- Connection conn = null;
- ResultSet res = null;
- PreparedStatement pstmt = null;
- boolean result = false;
-
- final int databaseId = getDatabaseId(databaseName);
- final int tableId = getTableId(databaseId, databaseName, tableName);
- ensurePartitionTable(tableName, tableId);
-
- try {
- if (this instanceof DerbyStore) {
- sql = "SELECT 1 FROM " + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? FETCH FIRST ROW ONLY ";
- } else {
- sql = "SELECT 1 FROM " + TB_PARTTIONS +" WHERE " + COL_TABLES_PK + " = ? LIMIT 1 ";
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(sql);
- }
-
- conn = getConnection();
- pstmt = conn.prepareStatement(sql);
- pstmt.setInt(1, tableId);
- res = pstmt.executeQuery();
-
- if (res.next()) {
- result = true;
- }
- } catch (SQLException se) {
- throw new TajoInternalError(se);
- } finally {
- CatalogUtil.closeQuietly(pstmt, res);
- }
- return result;
- }
-
- @Override
- public List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UnsupportedException {
- Connection conn = null;
- PreparedStatement pstmt = null;
- ResultSet res = null;
- int currentIndex = 1;
- String selectStatement = null;
- Pair<String, List<PartitionFilterSet>> pair = null;
-
- List<PartitionDescProto> partitions = TUtil.newList();
- List<PartitionFilterSet> filterSets = null;
-
- int databaseId = getDatabaseId(request.getDatabaseName());
- int tableId = getTableId(databaseId, request.getDatabaseName(), request.getTableName());
- if (!existPartitionMethod(request.getDatabaseName(), request.getTableName())) {
- throw new UndefinedPartitionMethodException(request.getTableName());
- }
-
- try {
- TableDescProto tableDesc = getTable(request.getDatabaseName(), request.getTableName());
-
- pair = getSelectStatementAndPartitionFilterSet(tableDesc.getTableName(), tableDesc.getPartition()
- .getExpressionSchema().getFieldsList(), request.getAlgebra());
-
- selectStatement = pair.getFirst();
- filterSets = pair.getSecond();
-
- conn = getConnection();
- pstmt = conn.prepareStatement(selectStatement);
-
- // Set table id by force because first parameter of all direct sql is table id
- pstmt.setInt(currentIndex, tableId);
- currentIndex++;
-
- for (PartitionFilterSet filter : filterSets) {
- // Set table id by force because all filters have table id as first parameter.
- pstmt.setInt(currentIndex, tableId);
- currentIndex++;
-
- for (Pair<Type, Object> parameter : filter.getParameters()) {
- switch (parameter.getFirst()) {
- case BOOLEAN:
- pstmt.setBoolean(currentIndex, (Boolean)parameter.getSecond());
- break;
- case INT8:
- pstmt.setLong(currentIndex, (Long) parameter.getSecond());
- break;
- case FLOAT8:
- pstmt.setDouble(currentIndex, (Double) parameter.getSecond());
- break;
- case DATE:
- pstmt.setDate(currentIndex, (Date) parameter.getSecond());
- break;
- case TIMESTAMP:
- pstmt.setTimestamp(currentIndex, (Timestamp) parameter.getSecond());
- break;
- case TIME:
- pstmt.setTime(currentIndex, (Time) parameter.getSecond());
- break;
- default:
- pstmt.setString(currentIndex, (String) parameter.getSecond());
- break;
- }
- currentIndex++;
- }
- }
-
- res = pstmt.executeQuery();
-
- while (res.next()) {
- PartitionDescProto.Builder builder = PartitionDescProto.newBuilder();
-
- builder.setId(res.getInt(COL_PARTITIONS_PK));
- builder.setPartitionName(res.getString("PARTITION_NAME"));
- builder.setPath(res.getString("PATH"));
- builder.setNumBytes(res.getLong(COL_PARTITION_BYTES));
-
- partitions.add(builder.build());
- }
- } catch (SQLException se) {
- throw new TajoInternalError(se);
- } finally {
- CatalogUtil.closeQuietly(pstmt, res);
- }
-
- return partitions;
- }
-
- /**
- * Create a select statement and parameters for querying partitions and partition keys in CatalogStore.
- *
- * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
- * Assume that an user gives a condition WHERE (col1 ='1' or col1 = '100') and col3 > 20.
- * There is no filter condition corresponding to col2.
- *
- * Then, the sql would be generated as following:
- *
- * SELECT A.PARTITION_ID, A.PARTITION_NAME, A.PATH FROM PARTITIONS A
- * WHERE A.TID = ?
- * AND A.PARTITION_ID IN (
- * SELECT T1.PARTITION_ID FROM PARTITION_KEYS T1
- * JOIN PARTITION_KEYS T2 ON T1.TID=T2.TID AND T1.PARTITION_ID = T2.PARTITION_ID AND T2.TID = ?
- * AND ( T2.COLUMN_NAME = 'col2' AND T2.PARTITION_VALUE IS NOT NULL )
- * JOIN PARTITION_KEYS T3 ON T1.TID=T3.TID AND T1.PARTITION_ID = T3.PARTITION_ID AND T3.TID = ?
- * AND ( T3.COLUMN_NAME = 'col3' AND T3.PARTITION_VALUE > ? )
- * WHERE T1.TID = ? AND ( T1.COLUMN_NAME = 'col1' AND T1.PARTITION_VALUE = ? )
- * OR ( T1.COLUMN_NAME = 'col1' AND T1.PARTITION_VALUE = ? )
- )
- *
- * @param tableName the table name
- * @param partitionColumns list of partition column
- * @param json the algebra expression
- * @return the select statement and partition filter sets
- */
- private Pair<String, List<PartitionFilterSet>> getSelectStatementAndPartitionFilterSet(String tableName,
- List<ColumnProto> partitionColumns, String json) {
-
- Pair<String, List<PartitionFilterSet>> result = null;
- Expr[] exprs = null;
-
- try {
- List<PartitionFilterSet> filterSets = TUtil.newList();
-
- if (json != null && !json.isEmpty()) {
- Expr algebra = JsonHelper.fromJson(json, Expr.class);
- exprs = AlgebraicUtil.toConjunctiveNormalFormArray(algebra);
- }
-
- // Write table alias for all levels
- String tableAlias;
-
- PartitionFilterAlgebraVisitor visitor = new PartitionFilterAlgebraVisitor();
- visitor.setIsHiveCatalog(false);
-
- Expr[] filters = AlgebraicUtil.getRearrangedCNFExpressions(tableName, partitionColumns, exprs);
-
- StringBuffer sb = new StringBuffer();
- sb.append("\n SELECT A.").append(CatalogConstants.COL_PARTITIONS_PK)
- .append(", A.PARTITION_NAME, A.PATH ").append(", ").append(COL_PARTITION_BYTES)
- .append(" FROM ").append(CatalogConstants.TB_PARTTIONS).append(" A ")
- .append("\n WHERE A.").append(CatalogConstants.COL_TABLES_PK).append(" = ? ")
- .append("\n AND A.").append(CatalogConstants.COL_PARTITIONS_PK).append(" IN (")
- .append("\n SELECT T1.").append(CatalogConstants.COL_PARTITIONS_PK)
- .append(" FROM ").append(CatalogConstants.TB_PARTTION_KEYS).append(" T1 ");
-
- // Write join clause from second column to last column.
- Column target;
-
- for (int i = 1; i < partitionColumns.size(); i++) {
- target = new Column(partitionColumns.get(i));
- tableAlias = "T" + (i+1);
-
- visitor.setColumn(target);
- visitor.setTableAlias(tableAlias);
- visitor.visit(null, new Stack<Expr>(), filters[i]);
-
- sb.append("\n JOIN ").append(CatalogConstants.TB_PARTTION_KEYS).append(" ").append(tableAlias)
- .append(" ON T1.").append(CatalogConstants.COL_TABLES_PK).append("=")
- .append(tableAlias).append(".").append(CatalogConstants.COL_TABLES_PK)
- .append(" AND T1.").append(CatalogConstants.COL_PARTITIONS_PK)
- .append(" = ").append(tableAlias).append(".").append(CatalogConstants.COL_PARTITIONS_PK)
- .append(" AND ").append(tableAlias).append(".").append(CatalogConstants.COL_TABLES_PK).append(" = ? AND ");
- sb.append(visitor.getResult());
-
- // Set parameters for executing PrepareStament
- PartitionFilterSet filterSet = new PartitionFilterSet();
- filterSet.setColumnName(target.getSimpleName());
-
- List<Pair<Type, Object>> list = TUtil.newList();
- list.addAll(visitor.getParameters());
- filterSet.addParameters(list);
-
- filterSets.add(filterSet);
- visitor.clearParameters();
- }
-
- // Write where clause for first column
- target = new Column(partitionColumns.get(0));
- tableAlias = "T1";
- visitor.setColumn(target);
- visitor.setTableAlias(tableAlias);
- visitor.visit(null, new Stack<Expr>(), filters[0]);
-
- sb.append("\n WHERE T1.").append(CatalogConstants.COL_TABLES_PK).append(" = ? AND ");
- sb.append(visitor.getResult())
- .append("\n )");
- sb.append("\n ORDER BY A.PARTITION_NAME");
-
- // Set parameters for executing PrepareStament
- PartitionFilterSet filterSet = new PartitionFilterSet();
- filterSet.setColumnName(target.getSimpleName());
-
- List<Pair<Type, Object>> list = TUtil.newList();
- list.addAll(visitor.getParameters());
- filterSet.addParameters(list);
-
- filterSets.add(filterSet);
-
- result = new Pair<>(sb.toString(), filterSets);
- } catch (TajoException e) {
- throw new TajoInternalError(e);
- }
-
- return result;
- }
-
-
- @Override
public List<TablePartitionProto> getAllPartitions() {
Connection conn = null;
Statement stmt = null;
@@ -2578,7 +2323,6 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt3.setInt(1, tableId);
pstmt3.setString(2, partition.getPartitionName());
pstmt3.setString(3, partition.getPath());
- pstmt3.setLong(4, partition.getNumBytes());
pstmt3.addBatch();
pstmt3.clearParameters();
@@ -3207,33 +2951,4 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
return exist;
}
-
- class PartitionFilterSet {
- private String columnName;
- private List<Pair<Type, Object>> parameters;
-
- public PartitionFilterSet() {
- parameters = TUtil.newList();
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
- public List<Pair<Type, Object>> getParameters() {
- return parameters;
- }
-
- public void setParameters(List<Pair<Type, Object>> parameters) {
- this.parameters = parameters;
- }
-
- public void addParameters(List<Pair<Type, Object>> parameters) {
- this.parameters.addAll(parameters);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
index 5288979..a067a53 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
@@ -92,96 +92,18 @@ public interface CatalogStore extends Closeable {
/************************** PARTITIONS *****************************/
/**
- * Check if list of partitions exist on catalog.
- *
- * @param databaseName
- * @param tableName
- * @return
- * @throws UndefinedDatabaseException
- * @throws UndefinedTableException
- * @throws UndefinedPartitionMethodException
- */
- boolean existPartitions(String databaseName, String tableName) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException;
-
- /**
* Get all partitions of a table
* @param tableName the table name
* @return
* @throws TajoException
*/
- List<CatalogProtos.PartitionDescProto> getPartitionsOfTable(String databaseName, String tableName) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UnsupportedException;
+ List<CatalogProtos.PartitionDescProto> getPartitions(String databaseName, String tableName) throws
+ UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException;
CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName,
- String partitionName) throws UndefinedDatabaseException,
- UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException;
-
- /**
- * Get list of partitions matching specified algrbra expression.
- *
- * For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
- * Assume that an user want to give a condition WHERE (col1 ='1' or col1 = '100') and col3 > 20 .
- *
- * Then, the algebra expression would be written as following:
- *
- * {
- * "LeftExpr": {
- * "LeftExpr": {
- * "Qualifier": "default.table1",
- * "ColumnName": "col3",
- * "OpType": "Column"
- * },
- * "RightExpr": {
- * "Value": "20.0",
- * "ValueType": "Unsigned_Integer",
- * "OpType": "Literal"
- * },
- * "OpType": "GreaterThan"
- * },
- * "RightExpr": {
- * "LeftExpr": {
- * "LeftExpr": {
- * "Qualifier": "default.table1",
- * "ColumnName": "col1",
- * "OpType": "Column"
- * },
- * "RightExpr": {
- * "Value": "1",
- * "ValueType": "String",
- * "OpType": "Literal"
- * },
- * "OpType": "Equals"
- * },
- * "RightExpr": {
- * "LeftExpr": {
- * "Qualifier": "default.table1",
- * "ColumnName": "col1",
- * "OpType": "Column"
- * },
- * "RightExpr": {
- * "Value": "100",
- * "ValueType": "String",
- * "OpType": "Literal"
- * },
- * "OpType": "Equals"
- * },
- * "OpType": "Or"
- * },
- * "OpType": "And"
- * }
- *
- * @param request the database name, the table name, the algebra expression
- * @return list of PartitionDescProto
- * @throws UndefinedDatabaseException
- * @throws UndefinedTableException
- * @throws UndefinedPartitionMethodException
- * @throws UndefinedOperatorException
- */
- List<PartitionDescProto> getPartitionsByAlgebra(PartitionsByAlgebraProto request) throws
- UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException,
- UndefinedOperatorException, UnsupportedException;
+ String partitionName)
+ throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException,
+ UndefinedPartitionMethodException;
List<TablePartitionProto> getAllPartitions();
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
index 96100e8..2ae85d1 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -32,7 +33,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -177,7 +178,6 @@
TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE,
PARTITION_NAME VARCHAR(767),
PATH VARCHAR(1024),
- NUM_BYTES BIGINT,
CONSTRAINT C_PARTITIONS_PK PRIMARY KEY (PARTITION_ID),
CONSTRAINT C_PARTITIONS_UNIQ UNIQUE (TID, PARTITION_NAME)
)]]>
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
index a94489d..6fb2610 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -32,7 +33,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -154,7 +155,6 @@
TID INT NOT NULL,
PARTITION_NAME VARCHAR(255) BINARY,
PATH VARCHAR(4096) BINARY,
- NUM_BYTES BIGINT,
UNIQUE INDEX PARTITION_UNIQUE_IDX (TID, PARTITION_NAME),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)]]>
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
index c0dadaa..d4ad970 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -32,7 +33,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -158,7 +159,6 @@
TID INT NOT NULL,
PARTITION_NAME VARCHAR(255) BINARY,
PATH VARCHAR(4096) BINARY,
- NUM_BYTES BIGINT,
UNIQUE INDEX PARTITION_UNIQUE_IDX (TID, PARTITION_NAME),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)]]>
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
index 190270c..8f23514 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -32,7 +33,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object order="0" type="table" name="meta">
<tns:sql><![CDATA[
@@ -211,7 +212,6 @@
TID INT NOT NULL,
PARTITION_NAME VARCHAR2(767),
PATH VARCHAR2(4000),
- NUM_BYTES NUMBER(38),
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
CONSTRAINT C_PARTITIONS_UNIQ UNIQUE (TID, PARTITION_NAME)
)]]>
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
index 33a1fd2..6c0f26a 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
@@ -21,6 +21,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 13 - 2015-10-14: Disable partition pruning using catalog temporarily. (TAJO-1926)
* 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663)
* 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493)
* 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
@@ -35,7 +36,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="12">
+ <tns:base version="13">
<tns:objects>
<tns:Object name="META" type="table" order="0">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -174,7 +175,6 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.
PARTITION_NAME VARCHAR(128),
PARTITION_VALUE VARCHAR(1024),
PATH VARCHAR(4096),
- NUM_BYTES BIGINT,
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
UNIQUE (TID, PARTITION_NAME)
)]]>
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index cbddab8..1671f57 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -59,7 +59,7 @@ public class TestCatalog {
static CatalogServer server;
static CatalogService catalog;
- @BeforeClass
+ @BeforeClass
public static void setUp() throws Exception {
server = new MiniCatalogServer();
@@ -590,7 +590,7 @@ public class TestCatalog {
assertEquals(retrived.getFunctionName(),"test2");
assertEquals(retrived.getLegacyFuncClass(),TestFunc1.class);
- assertEquals(retrived.getFuncType(), FunctionType.UDF);
+ assertEquals(retrived.getFuncType(),FunctionType.UDF);
}
@Test
@@ -785,8 +785,7 @@ public class TestCatalog {
.addColumn("age", Type.INT4)
.addColumn("score", Type.FLOAT8);
- String simpleTableName = "addedtable";
- String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName);
+ String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
@@ -801,7 +800,7 @@ public class TestCatalog {
TableDesc desc =
new TableDesc(tableName, schema, meta,
- new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri());
+ new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionMethodDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
@@ -816,17 +815,14 @@ public class TestCatalog {
testAddPartition(tableName, "id=10/name=aaa");
testAddPartition(tableName, "id=20/name=bbb");
- List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
+ List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable");
assertNotNull(partitions);
assertEquals(partitions.size(), 2);
- assertEquals(partitions.get(0).getNumBytes(), 0L);
-
- testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName);
testDropPartition(tableName, "id=10/name=aaa");
testDropPartition(tableName, "id=20/name=bbb");
- partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
+ partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable");
assertNotNull(partitions);
assertEquals(partitions.size(), 0);
@@ -834,99 +830,6 @@ public class TestCatalog {
assertFalse(catalog.existsTable(tableName));
}
- private void testGetPartitionsByAlgebra(String databaseName, String tableName) throws Exception {
- String qfTableName = databaseName + "." + tableName;
-
- // Equals Operator
- CatalogProtos.PartitionsByAlgebraProto.Builder request = CatalogProtos.PartitionsByAlgebraProto.newBuilder();
- request.setDatabaseName(databaseName);
- request.setTableName(tableName);
-
- String algebra = "{\n" +
- " \"LeftExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"id\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"10\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"name\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"aaa\",\n" +
- " \"ValueType\": \"String\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"Equals\"\n" +
- " },\n" +
- " \"OpType\": \"And\"\n" +
- "}";
-
- request.setAlgebra(algebra);
-
- List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitionsByAlgebra(request.build());
- assertNotNull(partitions);
- assertEquals(1, partitions.size());
-
- // GreaterThan Operator and InPredicate Operatior
- algebra = "{\n" +
- " \"LeftExpr\": {\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"id\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Value\": \"0\",\n" +
- " \"ValueType\": \"Unsigned_Integer\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " \"OpType\": \"GreaterThan\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"IsNot\": false,\n" +
- " \"LeftExpr\": {\n" +
- " \"Qualifier\": \"" + qfTableName + "\",\n" +
- " \"ColumnName\": \"name\",\n" +
- " \"OpType\": \"Column\"\n" +
- " },\n" +
- " \"RightExpr\": {\n" +
- " \"Values\": [\n" +
- " {\n" +
- " \"Value\": \"aaa\",\n" +
- " \"ValueType\": \"String\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " },\n" +
- " {\n" +
- " \"Value\": \"bbb\",\n" +
- " \"ValueType\": \"String\",\n" +
- " \"OpType\": \"Literal\"\n" +
- " }\n" +
- " ],\n" +
- " \"OpType\": \"ValueList\"\n" +
- " },\n" +
- " \"OpType\": \"InPredicate\"\n" +
- " },\n" +
- " \"OpType\": \"And\"\n" +
- "}";
-
- request.setAlgebra(algebra);
-
- partitions = catalog.getPartitionsByAlgebra(request.build());
- assertNotNull(partitions);
- assertEquals(2, partitions.size());
- }
-
private void testAddPartition(String tableName, String partitionName) throws Exception {
AlterTableDesc alterTableDesc = new AlterTableDesc();
alterTableDesc.setTableName(tableName);
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
index 99a7b43..a8b3feb 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
@@ -212,7 +212,7 @@ public class TestCatalogAgainstCaseSensitivity {
// Test get partitions of a table
//////////////////////////////////////////////////////////////////////////////
- List<PartitionDescProto> partitionDescs = catalog.getPartitionsOfTable("TestDatabase1", "TestPartition1");
+ List<PartitionDescProto> partitionDescs = catalog.getPartitions("TestDatabase1", "TestPartition1");
assertEquals(2, partitionDescs.size());
Map<String, PartitionDescProto> tablePartitionMap = new HashMap<>();
for (PartitionDescProto eachPartition : partitionDescs) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/13569f00/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
index 8652478..0c946ba 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
@@ -20,6 +20,8 @@ package org.apache.tajo.cli.tools;
import com.google.protobuf.ServiceException;
import org.apache.commons.cli.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.tajo.auth.UserRoleInfo;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -42,6 +44,7 @@ import java.util.Collections;
import java.util.List;
public class TajoDump {
+ private static final Log LOG = LogFactory.getLog(TajoDump.class);
private static final org.apache.commons.cli.Options options;
static {
@@ -180,25 +183,24 @@ public class TajoDump {
if (table.getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")) {
continue;
}
-
+
if (table.isExternal()) {
writer.write(DDLBuilder.buildDDLForExternalTable(table));
} else {
writer.write(DDLBuilder.buildDDLForBaseTable(table));
}
+
+ // TODO: This should be improved at TAJO-1891
if (table.hasPartition()) {
writer.write("\n\n");
writer.write("--\n");
writer.write(String.format("-- Table Partitions: %s%n", tableName));
+ writer.write("-- Partition dump and restore are not supported yet\n");
writer.write("--\n");
- // TODO: This should be improved at TAJO-1891
// List<PartitionDescProto> partitionProtos = client.getPartitionsOfTable(fqName);
// for (PartitionDescProto eachPartitionProto : partitionProtos) {
// writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto));
// }
- writer.write(String.format("ALTER TABLE %s REPAIR PARTITION;",
- CatalogUtil.denormalizeIdentifier(databaseName) + "." + CatalogUtil.denormalizeIdentifier(tableName)));
-
writer.write("\n\n");
}
@@ -218,6 +220,7 @@ public class TajoDump {
}
}
+
private static String toDateString() {
DateFormat df = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
java.util.Date today = Calendar.getInstance().getTime();