You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/05/16 14:15:41 UTC
[10/10] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/2cbc1b9c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/2cbc1b9c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/2cbc1b9c
Branch: refs/heads/index_support
Commit: 2cbc1b9c7dc1ec04c4f6c8d7c8f746225d02421d
Parents: 6f44096 5491f0e
Author: Jihoon Son <ji...@apache.org>
Authored: Sat May 16 21:05:31 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Sat May 16 21:05:31 2015 +0900
----------------------------------------------------------------------
CHANGES | 15 +
.../java/org/apache/tajo/catalog/Schema.java | 16 +
.../java/org/apache/tajo/catalog/TableMeta.java | 6 +-
.../org/apache/tajo/catalog/TestTableMeta.java | 27 +
.../org/apache/tajo/cli/tools/TajoAdmin.java | 6 +-
.../org/apache/tajo/cli/tools/TajoHAAdmin.java | 9 +-
.../cli/tsql/commands/DescTableCommand.java | 1 +
.../apache/tajo/client/DummyServiceTracker.java | 19 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 24 +-
.../java/org/apache/tajo/conf/TajoConf.java | 2 +
.../ValueTooLongForTypeCharactersException.java | 27 +
.../java/org/apache/tajo/ha/HAConstants.java | 1 +
.../java/org/apache/tajo/ha/HAServiceUtil.java | 253 --------
.../apache/tajo/service/BaseServiceTracker.java | 31 +-
.../apache/tajo/service/HAServiceTracker.java | 25 +-
.../org/apache/tajo/service/ServiceTracker.java | 28 +-
.../org/apache/tajo/storage/EmptyTuple.java | 140 +---
.../java/org/apache/tajo/storage/NullTuple.java | 175 +++++
.../java/org/apache/tajo/storage/VTuple.java | 20 +-
.../engine/planner/PhysicalPlannerImpl.java | 25 +-
.../planner/physical/BSTIndexScanExec.java | 4 +-
.../physical/BasicPhysicalExecutorVisitor.java | 8 -
.../planner/physical/ColPartitionStoreExec.java | 7 +-
.../planner/physical/CommonHashJoinExec.java | 191 ++++++
.../engine/planner/physical/CommonJoinExec.java | 172 ++++-
.../planner/physical/HashFullOuterJoinExec.java | 247 +++----
.../engine/planner/physical/HashJoinExec.java | 212 +-----
.../planner/physical/HashLeftAntiJoinExec.java | 59 +-
.../planner/physical/HashLeftOuterJoinExec.java | 292 +--------
.../planner/physical/HashLeftSemiJoinExec.java | 48 +-
.../planner/physical/NLLeftOuterJoinExec.java | 101 ---
.../physical/PhysicalExecutorVisitor.java | 3 -
.../physical/RangeShuffleFileWriteExec.java | 3 +-
.../physical/RightOuterMergeJoinExec.java | 40 +-
.../engine/planner/physical/SeqScanExec.java | 2 +-
.../engine/planner/physical/StoreTableExec.java | 9 +-
.../apache/tajo/engine/utils/CacheHolder.java | 3 +-
.../org/apache/tajo/ha/HdfsServiceTracker.java | 322 +++++----
.../org/apache/tajo/master/GlobalEngine.java | 4 +-
.../java/org/apache/tajo/master/TajoMaster.java | 24 +-
.../apache/tajo/master/exec/DDLExecutor.java | 5 +-
.../exec/NonForwardQueryResultFileScanner.java | 7 +-
.../apache/tajo/master/exec/QueryExecutor.java | 4 +-
.../java/org/apache/tajo/querymaster/Query.java | 7 +-
.../tajo/querymaster/QueryMasterTask.java | 4 +-
.../apache/tajo/querymaster/Repartitioner.java | 24 +-
.../java/org/apache/tajo/querymaster/Stage.java | 6 +-
.../main/java/org/apache/tajo/util/JSPUtil.java | 2 +-
.../java/org/apache/tajo/worker/TajoWorker.java | 6 +-
.../main/java/org/apache/tajo/worker/Task.java | 3 +-
.../tajo/worker/WorkerHeartbeatService.java | 2 +-
.../resources/webapps/admin/catalogview.jsp | 11 +-
.../main/resources/webapps/admin/cluster.jsp | 10 +-
.../src/main/resources/webapps/admin/index.jsp | 10 +-
.../src/main/resources/webapps/admin/query.jsp | 7 +-
.../resources/webapps/admin/query_executor.jsp | 9 +-
.../org/apache/tajo/BackendTestingUtil.java | 3 +-
.../planner/global/TestBroadcastJoinPlan.java | 3 +-
.../planner/physical/TestBNLJoinExec.java | 5 +-
.../planner/physical/TestExternalSortExec.java | 3 +-
.../physical/TestFullOuterHashJoinExec.java | 9 +-
.../physical/TestFullOuterMergeJoinExec.java | 10 +-
.../planner/physical/TestHashAntiJoinExec.java | 5 +-
.../planner/physical/TestHashJoinExec.java | 5 +-
.../planner/physical/TestHashSemiJoinExec.java | 13 +-
.../physical/TestLeftOuterHashJoinExec.java | 113 ++--
.../physical/TestLeftOuterNLJoinExec.java | 474 --------------
.../planner/physical/TestMergeJoinExec.java | 5 +-
.../engine/planner/physical/TestNLJoinExec.java | 5 +-
.../planner/physical/TestPhysicalPlanner.java | 11 +-
.../physical/TestProgressExternalSortExec.java | 3 +-
.../physical/TestRightOuterHashJoinExec.java | 7 +-
.../physical/TestRightOuterMergeJoinExec.java | 11 +-
.../engine/planner/physical/TestSortExec.java | 5 +-
.../tajo/engine/query/TestHBaseTable.java | 12 +-
.../tajo/engine/query/TestInsertQuery.java | 45 +-
.../tajo/engine/query/TestJoinBroadcast.java | 2 +-
.../apache/tajo/engine/util/TestTupleUtil.java | 18 +
.../apache/tajo/ha/TestHAServiceHDFSImpl.java | 28 +-
.../org/apache/tajo/jdbc/TestResultSet.java | 2 +-
.../tajo/master/TestExecutionBlockCursor.java | 4 +-
.../org/apache/tajo/storage/TestRowFile.java | 5 +-
.../queries/TestInsertQuery/test1_ddl.sql | 1 +
.../TestInsertQuery/test1_nolength_ddl.sql | 1 +
.../testInsertIntoSelectWithFixedSizeChar.sql | 4 +
...tIntoSelectWithFixedSizeCharWithNoLength.sql | 2 +
.../testJoinFilterOfRowPreservedTable1.sql | 2 +-
.../testJoinFilterOfRowPreservedTable1.result | 2 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 4 +
.../plan/expr/AggregationFunctionCallEval.java | 4 +-
.../apache/tajo/plan/expr/AlgebraicUtil.java | 5 +
.../org/apache/tajo/plan/expr/EvalNode.java | 39 +-
.../java/org/apache/tajo/plan/expr/InEval.java | 2 +-
.../plan/expr/PatternMatchPredicateEval.java | 2 +-
.../tajo/plan/expr/WindowFunctionEval.java | 2 +-
.../stream/TextFieldSerializerDeserializer.java | 8 +-
.../storage/BinarySerializerDeserializer.java | 10 +
.../org/apache/tajo/storage/FrameTuple.java | 14 +-
.../org/apache/tajo/storage/MergeScanner.java | 2 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 20 +-
.../org/apache/tajo/storage/StorageManager.java | 645 +------------------
.../org/apache/tajo/storage/TableSpace.java | 74 +++
.../apache/tajo/storage/TableSpaceManager.java | 254 ++++++++
.../storage/TextSerializerDeserializer.java | 10 +-
.../tajo/storage/hbase/HBasePutAppender.java | 4 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 7 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 40 +-
.../storage/hbase/TestHBaseStorageManager.java | 5 +-
.../org/apache/tajo/storage/FileAppender.java | 2 +-
.../apache/tajo/storage/FileStorageManager.java | 377 ++++++++++-
.../storage/HashShuffleAppenderManager.java | 4 +-
.../tajo/storage/parquet/TajoWriteSupport.java | 7 +
.../text/TextFieldSerializerDeserializer.java | 8 +-
.../tajo/storage/TestCompressionStorages.java | 5 +-
.../tajo/storage/TestDelimitedTextFile.java | 9 +-
.../tajo/storage/TestFileStorageManager.java | 11 +-
.../apache/tajo/storage/TestFileSystems.java | 3 +-
.../org/apache/tajo/storage/TestLineReader.java | 9 +-
.../apache/tajo/storage/TestMergeScanner.java | 7 +-
.../org/apache/tajo/storage/TestStorages.java | 94 ++-
.../apache/tajo/storage/index/TestBSTIndex.java | 53 +-
.../index/TestSingleCSVFileBSTIndex.java | 5 +-
.../apache/tajo/storage/json/TestJsonSerDe.java | 8 +-
123 files changed, 2389 insertions(+), 2900 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 42d9b36,21f5e5a..4372e04
--- a/CHANGES
+++ b/CHANGES
@@@ -123,8 -128,16 +128,18 @@@ Release 0.11.0 - unrelease
BUG FIXES
+ TAJO-1594: Catalog schema is invalid for some databases. (jihoon)
+
+ TAJO-1605: Fix master build failure on jdk 1.6. (jinho)
+
+ TAJO-1485: Datum 'Char' returned only 1byte.
+ (Contributed by DaeMyung Kang, Committed by jihoon)
+
+ TAJO-1586: TajoMaster HA startup failure on Yarn. (jaehwa)
+
+ TAJO-1598: TableMeta should change equals mechanism.
+ (Contributed by DaeMyung Kang, Committed by jihoon)
+
TAJO-1593: Add missing stop condition to Taskrunner. (jinho)
TAJO-1556: "insert into select" with reordered column list does not work.
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --cc tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index 31dfb13,6df26b7..3341c68
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@@ -21,6 -21,6 +21,7 @@@ package org.apache.tajo.cli.tsql.comman
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.tajo.TajoConstants;
++import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index 2dd7fa8,ff42d5f..04315eb
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@@ -46,11 -47,8 +46,12 @@@ import org.apache.tajo.ipc.TajoWorkerPr
import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.storage.*;
+import org.apache.tajo.storage.FileStorageManager;
+import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.storage.StorageManager;
++import org.apache.tajo.storage.TableSpaceManager;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
index 211d438,4612d45..7df48a4
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
@@@ -68,21 -54,10 +68,21 @@@ public class BSTIndexScanExec extends P
super(context, scanNode.getInSchema(), scanNode.getOutSchema());
this.scanNode = scanNode;
this.qual = scanNode.getQual();
- this.datum = datum;
+
+ SortSpec[] keySortSpecs = new SortSpec[predicates.length];
+ values = new Datum[predicates.length];
+ for (int i = 0; i < predicates.length; i++) {
+ keySortSpecs[i] = predicates[i].getKeySortSpec();
+ values[i] = predicates[i].getValue();
+ }
+
+ TupleComparator comparator = new BaseTupleComparator(keySchema,
+ keySortSpecs);
+
+ Schema fileScanOutSchema = mergeSubSchemas(inSchema, keySchema, scanNode.getTargets(), qual);
- this.fileScanner = StorageManager.getSeekableScanner(context.getConf(),
+ this.fileScanner = TableSpaceManager.getSeekableScanner(context.getConf(),
- scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema);
+ scanNode.getTableDesc().getMeta(), inSchema, fragment, fileScanOutSchema);
this.fileScanner.init();
this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets());
@@@ -92,23 -67,6 +92,23 @@@
this.reader.open();
}
+ private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, Target[] targets, EvalNode qual) {
+ Schema mergedSchema = new Schema();
+ Set<Column> qualAndTargets = TUtil.newHashSet();
+ qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(qual));
+ for (Target target : targets) {
+ qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree()));
+ }
- for (Column column : originalSchema.getColumns()) {
++ for (Column column : originalSchema.getRootColumns()) {
+ if (subSchema.contains(column)
+ || qualAndTargets.contains(column)
+ || qualAndTargets.contains(column)) {
+ mergedSchema.addColumn(column);
+ }
+ }
+ return mergedSchema;
+ }
+
@Override
public void init() throws IOException {
super.init();
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 991cb25,aeb4166..fc9212c
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@@ -55,8 -54,8 +54,9 @@@ import org.apache.tajo.plan.verifier.Pr
import org.apache.tajo.plan.verifier.VerificationState;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.storage.StorageManager;
+ import org.apache.tajo.storage.TableSpaceManager;
import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.IPCUtil;
import java.io.IOException;
import java.sql.SQLException;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
index 12f4b79,d77001c..640ec60
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
@@@ -36,14 -36,7 +36,13 @@@ import org.apache.tajo.algebra.Expr
import org.apache.tajo.algebra.JsonHelper;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.TableDesc;
- import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.plan.LogicalOptimizer;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.LogicalPlanner;
+import org.apache.tajo.plan.logical.LogicalRootNode;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
+import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.UnimplementedException;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
index ab94aac,5efdede..35cd612
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
@@@ -80,9 -80,9 +80,9 @@@ public class TestExecutionBlockCursor
analyzer = new SQLAnalyzer();
logicalPlanner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(conf);
+ optimizer = new LogicalOptimizer(conf, catalog);
- StorageManager sm = StorageManager.getFileStorageManager(conf);
+ StorageManager sm = TableSpaceManager.getFileStorageManager(conf);
dispatcher = new AsyncDispatcher();
dispatcher.init(conf);
dispatcher.start();
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
----------------------------------------------------------------------
diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
index ed6a570,c6b7354..7517e3a
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
@@@ -18,12 -18,10 +18,13 @@@
package org.apache.tajo.plan.expr;
+import org.apache.tajo.algebra.*;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.plan.PlanningException;
+import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
import java.util.ArrayList;
+ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Stack;