You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/05/12 05:14:12 UTC
[13/14] tajo git commit: TAJO-1591: Change StoreType represented as
Enum to String type. (hyunsik)
TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik)
Closes #564
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/53ed1c37
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/53ed1c37
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/53ed1c37
Branch: refs/heads/index_support
Commit: 53ed1c376f4969b5a2c54e30ddbd8113847a8768
Parents: 70df3a8 0d1bf41
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon May 11 15:29:17 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon May 11 15:29:17 2015 -0700
----------------------------------------------------------------------
CHANGES | 11 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 12 +-
.../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++
.../java/org/apache/tajo/catalog/Schema.java | 109 +++++-
.../org/apache/tajo/catalog/SchemaUtil.java | 33 +-
.../tajo/catalog/store/HiveCatalogStore.java | 4 +-
.../cli/tsql/commands/DescTableCommand.java | 4 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Tuple.java | 2 +-
.../java/org/apache/tajo/util/StringUtils.java | 70 ++++
.../main/java/org/apache/tajo/util/TUtil.java | 32 --
.../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 17 +-
.../engine/planner/PhysicalPlannerImpl.java | 17 +-
.../tajo/engine/planner/global/DataChannel.java | 4 +-
.../engine/planner/global/GlobalPlanner.java | 6 +-
.../global/builder/DistinctGroupbyBuilder.java | 6 +-
.../DistinctGroupbyThirdAggregationExec.java | 2 +-
.../planner/physical/PhysicalPlanUtil.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 27 +-
.../tajo/engine/utils/TupleCacheScanner.java | 2 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 8 +-
.../exec/NonForwardQueryResultFileScanner.java | 2 +-
.../NonForwardQueryResultSystemScanner.java | 20 +-
.../tajo/worker/ExecutionBlockContext.java | 15 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 1 +
.../resources/webapps/admin/catalogview.jsp | 5 +-
.../engine/function/TestFunctionLoader.java | 5 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 8 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 4 +-
.../tajo/engine/query/TestJoinBroadcast.java | 2 +-
.../engine/query/TestSelectNestedRecord.java | 71 ++++
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +-
.../rs/resources/TestQueryResultResource.java | 6 +-
.../TestSelectNestedRecord/sample1/table.json | 3 +
.../TestSelectNestedRecord/tweets/sample1.json | 4 +
.../TestSelectNestedRecord/sample1_ddl.sql | 7 +
.../TestSelectNestedRecord/sample2_ddl.sql | 19 ++
.../testNestedFieldAsGroupbyKey1.sql | 7 +
.../testNestedFieldAsJoinKey1.sql | 7 +
.../TestSelectNestedRecord/testSelect1.sql | 1 +
.../TestSelectNestedRecord/testSelect2.sql | 61 ++++
.../TestSelectNestedRecord/tweets_ddl.sql | 74 +++++
.../testSelectWithParentheses2.sql | 2 +-
.../testNestedFieldAsGroupbyKey1.result | 6 +
.../testNestedFieldAsJoinKey1.result | 6 +
.../TestSelectNestedRecord/testSelect1.result | 5 +
.../TestSelectNestedRecord/testSelect2.result | 6 +
.../apache/tajo/jdbc/TajoDatabaseMetaData.java | 2 +-
.../tajo/plan/LogicalPlanPreprocessor.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 32 +-
.../apache/tajo/plan/expr/RowConstantEval.java | 3 +-
.../tajo/plan/expr/WindowFunctionEval.java | 3 +-
.../GreedyHeuristicJoinOrderAlgorithm.java | 8 +-
.../apache/tajo/plan/joinorder/JoinEdge.java | 4 +-
.../apache/tajo/plan/joinorder/JoinGraph.java | 6 +-
.../tajo/plan/logical/DistinctGroupbyNode.java | 4 +-
.../apache/tajo/plan/logical/EvalExprNode.java | 3 +-
.../apache/tajo/plan/logical/GroupbyNode.java | 10 +-
.../tajo/plan/logical/ProjectionNode.java | 3 +-
.../apache/tajo/plan/logical/RelationNode.java | 12 +
.../tajo/plan/logical/ShuffleFileWriteNode.java | 3 +-
.../tajo/plan/logical/TruncateTableNode.java | 4 +-
.../apache/tajo/plan/logical/WindowAggNode.java | 7 +-
.../tajo/plan/nameresolver/NameResolver.java | 194 ++++++++---
.../plan/nameresolver/ResolverByLegacy.java | 8 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 4 +-
.../rewrite/rules/PartitionedTableRewriter.java | 4 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 2 +-
.../tajo/plan/verifier/LogicalPlanVerifier.java | 24 +-
.../plan/verifier/PreLogicalPlanVerifier.java | 6 +-
.../org/apache/tajo/rpc/AsyncRpcServer.java | 6 +-
.../org/apache/tajo/rpc/BlockingRpcServer.java | 6 +-
.../org/apache/tajo/storage/MergeScanner.java | 2 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Scanner.java | 9 +
.../org/apache/tajo/storage/StorageManager.java | 4 +-
.../storage/hbase/AbstractHBaseAppender.java | 2 -
.../tajo/storage/hbase/ColumnMapping.java | 2 +-
.../HBaseBinarySerializerDeserializer.java | 10 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 22 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 6 +-
.../java/org/apache/tajo/storage/CSVFile.java | 2 +-
.../org/apache/tajo/storage/FileScanner.java | 2 +-
.../apache/tajo/storage/FileStorageManager.java | 2 +-
.../apache/tajo/storage/avro/AvroScanner.java | 25 +-
.../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++--------
.../apache/tajo/storage/json/JsonLineSerDe.java | 5 +-
.../tajo/storage/json/JsonLineSerializer.java | 2 -
.../storage/parquet/TajoRecordConverter.java | 7 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileScanner.java | 16 +-
.../tajo/storage/text/CSVLineDeserializer.java | 26 +-
.../apache/tajo/storage/text/CSVLineSerDe.java | 5 +-
.../tajo/storage/text/DelimitedTextFile.java | 11 +-
.../tajo/storage/text/TextLineDeserializer.java | 4 +-
.../apache/tajo/storage/text/TextLineSerDe.java | 3 +-
.../apache/tajo/storage/TestMergeScanner.java | 16 +-
.../org/apache/tajo/storage/TestStorages.java | 36 +-
99 files changed, 1292 insertions(+), 462 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 9788307,84be66a..73fd2f3
--- a/CHANGES
+++ b/CHANGES
@@@ -24,6 -24,6 +24,8 @@@ Release 0.11.0 - unrelease
IMPROVEMENT
++ TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik)
++
TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun,
Committed by hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --cc tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index e2cfe8f,322818d..2c856e1
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@@ -175,7 -175,19 +175,19 @@@ public class TestMergeScanner
Tuple tuple;
while ((tuple = scanner.next()) != null) {
totalCounts++;
- if (isProjectableStorage(meta.getStoreType())) {
+
- if (storeType == StoreType.RAW) {
++ if (storeType.equalsIgnoreCase("RAW")) {
+ assertEquals(4, tuple.size());
+ assertNotNull(tuple.get(0));
+ assertNotNull(tuple.get(1));
+ assertNotNull(tuple.get(2));
+ assertNotNull(tuple.get(3));
+ } else if (scanner.isProjectable()) {
+ assertEquals(2, tuple.size());
+ assertNotNull(tuple.get(0));
+ assertNotNull(tuple.get(1));
+ } else {
+ assertEquals(4, tuple.size());
assertNotNull(tuple.get(0));
assertNull(tuple.get(1));
assertNotNull(tuple.get(2));
@@@ -187,15 -199,15 +199,13 @@@
assertEquals(tupleNum * 2, totalCounts);
}
- private static boolean isProjectableStorage(StoreType type) {
- switch (type) {
- case CSV:
- case SEQUENCEFILE:
- case RAW:
- case ROWFILE:
- return false;
- default:
+ private static boolean isProjectableStorage(String type) {
+ if (type.equalsIgnoreCase("RCFILE") ||
+ type.equalsIgnoreCase("PARQUET") ||
- type.equalsIgnoreCase("SEQUENCEFILE") ||
- type.equalsIgnoreCase("CSV") ||
+ type.equalsIgnoreCase("AVRO")) {
return true;
+ } else {
+ return false;
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --cc tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 7c75973,a735307..949e2e8
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@@ -313,9 -305,23 +305,23 @@@ public class TestStorages
assertEquals(tupleNum, tupleCnt);
}
+ private void verifyProjectedFields(boolean projectable, Tuple tuple, int tupleCnt) {
+ if (projectable) {
+ assertTrue(tupleCnt + 2 == tuple.get(0).asInt8());
+ assertTrue(tupleCnt + 3 == tuple.get(1).asFloat4());
+ } else {
+ // RAW and ROW always project all fields.
- if (storeType != StoreType.RAW && storeType != StoreType.ROWFILE) {
++ if (!storeType.equalsIgnoreCase("RAW") && !storeType.equalsIgnoreCase("ROWFILE")) {
+ assertTrue(tuple.get(0) == null);
+ }
+ assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
+ assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
+ }
+ }
+
@Test
public void testVariousTypes() throws IOException {
- boolean handleProtobuf = storeType != StoreType.JSON;
+ boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@@ -956,7 -962,7 +962,8 @@@
@Test
public void testLessThanSchemaSize() throws IOException {
/* RAW is internal storage. It must be same with schema size */
- if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")){
- if (storeType == StoreType.RAW || storeType == StoreType.AVRO || storeType == StoreType.PARQUET) {
++ if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")
++ || storeType.equalsIgnoreCase("PARQUET")){
return;
}