You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/03/23 03:56:17 UTC
[8/9] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fd2f8281
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fd2f8281
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fd2f8281
Branch: refs/heads/index_support
Commit: fd2f8281dd0efcf07322ae94086a1a72ad8b5f87
Parents: 63c8e1c 8d0146b
Author: Jihoon Son <ji...@apache.org>
Authored: Mon Mar 23 11:55:25 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Mon Mar 23 11:55:25 2015 +0900
----------------------------------------------------------------------
CHANGES | 39 +++
tajo-catalog/tajo-catalog-client/pom.xml | 2 +-
.../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 2 +-
tajo-catalog/tajo-catalog-server/pom.xml | 2 +-
tajo-cli/pom.xml | 2 +-
.../org/apache/tajo/cli/tools/TajoAdmin.java | 1 -
.../org/apache/tajo/cli/tools/TajoGetConf.java | 5 -
.../org/apache/tajo/cli/tools/TajoHAAdmin.java | 5 -
.../cli/tsql/DefaultTajoCliOutputFormatter.java | 2 +-
.../org/apache/tajo/cli/tsql/SimpleParser.java | 3 +-
.../java/org/apache/tajo/cli/tsql/TajoCli.java | 10 +-
tajo-client/pom.xml | 2 +-
.../apache/tajo/client/SessionConnection.java | 20 +-
.../java/org/apache/tajo/conf/TajoConf.java | 2 +
.../java/org/apache/tajo/util/BytesUtils.java | 159 +++++----
.../java/org/apache/tajo/util/StringUtils.java | 6 +-
.../org/apache/tajo/util/TestStringUtil.java | 4 +-
tajo-core/pom.xml | 2 +-
.../engine/planner/global/GlobalPlanner.java | 39 ++-
.../engine/planner/physical/HashJoinExec.java | 51 ++-
.../planner/physical/HashLeftOuterJoinExec.java | 74 +++-
.../physical/PartitionMergeScanExec.java | 17 +-
.../tajo/engine/planner/physical/ScanExec.java | 72 ++++
.../engine/planner/physical/SeqScanExec.java | 105 +-----
.../apache/tajo/engine/utils/CacheHolder.java | 97 ++++++
.../apache/tajo/engine/utils/TableCache.java | 84 +++++
.../apache/tajo/engine/utils/TableCacheKey.java | 57 +++
.../apache/tajo/engine/utils/TupleCache.java | 122 -------
.../apache/tajo/engine/utils/TupleCacheKey.java | 57 ---
.../org/apache/tajo/master/QueryInProgress.java | 2 +-
.../org/apache/tajo/master/QueryManager.java | 2 +-
.../exec/NonForwardQueryResultFileScanner.java | 37 +-
.../apache/tajo/master/exec/QueryExecutor.java | 5 +
.../java/org/apache/tajo/querymaster/Stage.java | 3 +-
.../tajo/worker/ExecutionBlockContext.java | 43 ++-
.../worker/ExecutionBlockSharedResource.java | 26 ++
.../main/java/org/apache/tajo/worker/Task.java | 75 ++--
.../apache/tajo/worker/TaskAttemptContext.java | 2 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 9 +-
.../apache/tajo/worker/TaskRunnerManager.java | 4 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 5 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 28 ++
.../tajo/engine/query/TestSelectQuery.java | 48 +++
.../tajo/engine/query/TestUnionQuery.java | 64 +++-
.../apache/tajo/engine/util/TestTableCache.java | 109 ++++++
.../apache/tajo/engine/util/TestTupleCache.java | 89 -----
.../apache/tajo/master/TestGlobalPlanner.java | 3 +-
.../multibytes_delimiter1/table1.tbl | 5 +
.../multibytes_delimiter2/table2.tbl | 5 +
.../TestCTASQuery/CtasWithMultipleUnions.sql | 12 +
.../testCtasWithMultipleUnions.sql | 1 +
.../multibytes_delimiter_table1_ddl.sql | 3 +
.../multibytes_delimiter_table2_ddl.sql | 3 +
.../multibytes_delimiter_table3_ddl.sql | 3 +
.../multibytes_delimiter_table4_ddl.sql | 3 +
.../testMultiBytesDelimiter1.sql | 1 +
.../testMultiBytesDelimiter2.sql | 1 +
.../testMultiBytesDelimiter3.sql | 1 +
.../testMultiBytesDelimiter4.sql | 1 +
.../testMultiBytesDelimiter1.result | 7 +
.../testMultiBytesDelimiter2.result | 7 +
.../testMultiBytesDelimiter3.result | 7 +
.../testMultiBytesDelimiter4.result | 7 +
tajo-dist/pom.xml | 7 +-
tajo-dist/src/main/conf/tajo-env.sh | 2 +-
.../org/apache/tajo/plan/ExprAnnotator.java | 2 +-
.../java/org/apache/tajo/plan/LogicalPlan.java | 2 -
.../plan/expr/AggregationFunctionCallEval.java | 14 +
.../apache/tajo/plan/expr/AlgebraicUtil.java | 5 +
.../tajo/plan/expr/BetweenPredicateEval.java | 14 +
.../org/apache/tajo/plan/expr/CaseWhenEval.java | 25 +-
.../org/apache/tajo/plan/expr/CastEval.java | 10 +
.../org/apache/tajo/plan/expr/EvalTreeUtil.java | 66 +++-
.../java/org/apache/tajo/plan/expr/InEval.java | 10 +
.../org/apache/tajo/plan/expr/IsNullEval.java | 9 +
.../tajo/plan/expr/PartialBinaryExpr.java | 5 +-
.../tajo/plan/expr/RegexPredicateEval.java | 25 ++
.../apache/tajo/plan/expr/RowConstantEval.java | 11 +
.../tajo/plan/expr/WindowFunctionEval.java | 14 +
.../GreedyHeuristicJoinOrderAlgorithm.java | 4 +-
.../tajo/plan/logical/AlterTableNode.java | 14 +
.../tajo/plan/logical/DistinctGroupbyNode.java | 17 +-
.../apache/tajo/plan/logical/EvalExprNode.java | 18 +-
.../apache/tajo/plan/logical/GroupbyNode.java | 16 +-
.../apache/tajo/plan/logical/HavingNode.java | 9 +
.../apache/tajo/plan/logical/IndexScanNode.java | 13 +
.../apache/tajo/plan/logical/InsertNode.java | 13 +
.../org/apache/tajo/plan/logical/JoinNode.java | 13 +
.../org/apache/tajo/plan/logical/LimitNode.java | 11 +-
.../apache/tajo/plan/logical/LogicalNode.java | 27 +-
.../tajo/plan/logical/PersistentStoreNode.java | 10 +
.../tajo/plan/logical/ProjectionNode.java | 28 +-
.../apache/tajo/plan/logical/SelectionNode.java | 9 +
.../tajo/plan/logical/ShuffleFileWriteNode.java | 13 +
.../org/apache/tajo/plan/logical/SortNode.java | 14 +-
.../tajo/plan/logical/StoreTableNode.java | 10 +
.../tajo/plan/nameresolver/NameResolver.java | 4 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 10 +-
.../tajo/plan/serder/EvalNodeDeserializer.java | 17 +-
.../plan/serder/LogicalNodeDeserializer.java | 3 +
.../tajo/plan/serder/LogicalNodeSerializer.java | 2 +
.../org/apache/tajo/plan/util/PlannerUtil.java | 26 +-
.../tajo/plan/verifier/LogicalPlanVerifier.java | 7 -
.../plan/verifier/PreLogicalPlanVerifier.java | 19 +-
.../plan/visitor/BasicLogicalPlanVisitor.java | 6 +-
tajo-plan/src/main/proto/Plan.proto | 1 +
tajo-project/pom.xml | 31 ++
tajo-project/src/site/site.xml | 2 +-
tajo-pullserver/pom.xml | 2 +-
tajo-rpc/pom.xml | 180 ++--------
.../org/apache/tajo/rpc/AsyncRpcClient.java | 267 --------------
.../org/apache/tajo/rpc/AsyncRpcServer.java | 148 --------
.../org/apache/tajo/rpc/BlockingRpcClient.java | 315 -----------------
.../org/apache/tajo/rpc/BlockingRpcServer.java | 147 --------
.../java/org/apache/tajo/rpc/CallFuture.java | 84 -----
.../apache/tajo/rpc/DefaultRpcController.java | 65 ----
.../org/apache/tajo/rpc/NettyClientBase.java | 173 ---------
.../org/apache/tajo/rpc/NettyRpcController.java | 63 ----
.../org/apache/tajo/rpc/NettyServerBase.java | 205 -----------
.../java/org/apache/tajo/rpc/NullCallback.java | 38 --
.../tajo/rpc/ProtoChannelInitializer.java | 50 ---
.../apache/tajo/rpc/RemoteCallException.java | 69 ----
.../org/apache/tajo/rpc/RemoteException.java | 37 --
.../tajo/rpc/RetriesExhaustedException.java | 104 ------
.../org/apache/tajo/rpc/RpcChannelFactory.java | 182 ----------
.../org/apache/tajo/rpc/RpcConnectionPool.java | 194 -----------
.../main/java/org/apache/tajo/rpc/RpcUtils.java | 68 ----
.../org/apache/tajo/rpc/ServerCallable.java | 162 ---------
.../apache/tajo/rpc/TajoServiceException.java | 58 ---
tajo-rpc/src/main/proto/DummyProtos.proto | 47 ---
tajo-rpc/src/main/proto/RpcProtos.proto | 32 --
tajo-rpc/src/main/proto/TestProtocol.proto | 31 --
tajo-rpc/src/main/proto/TestProtos.proto | 35 --
tajo-rpc/src/test/java/log4j.properties | 25 --
.../java/org/apache/tajo/rpc/TestAsyncRpc.java | 339 ------------------
.../org/apache/tajo/rpc/TestBlockingRpc.java | 341 ------------------
.../rpc/test/impl/DummyProtocolAsyncImpl.java | 86 -----
.../test/impl/DummyProtocolBlockingImpl.java | 83 -----
tajo-rpc/tajo-rpc-common/pom.xml | 216 ++++++++++++
.../org/apache/tajo/rpc/NettyServerBase.java | 243 +++++++++++++
.../org/apache/tajo/rpc/RemoteException.java | 37 ++
.../tajo/rpc/RetriesExhaustedException.java | 104 ++++++
.../org/apache/tajo/rpc/RpcChannelFactory.java | 182 ++++++++++
.../org/apache/tajo/rpc/RpcEventListener.java | 62 ++++
.../main/java/org/apache/tajo/rpc/RpcUtils.java | 122 +++++++
tajo-rpc/tajo-rpc-protobuf/pom.xml | 274 +++++++++++++++
.../org/apache/tajo/rpc/AsyncRpcClient.java | 227 ++++++++++++
.../org/apache/tajo/rpc/AsyncRpcServer.java | 148 ++++++++
.../org/apache/tajo/rpc/BlockingRpcClient.java | 273 +++++++++++++++
.../org/apache/tajo/rpc/BlockingRpcServer.java | 147 ++++++++
.../java/org/apache/tajo/rpc/CallFuture.java | 84 +++++
.../apache/tajo/rpc/DefaultRpcController.java | 65 ++++
.../org/apache/tajo/rpc/NettyClientBase.java | 221 ++++++++++++
.../org/apache/tajo/rpc/NettyRpcController.java | 63 ++++
.../java/org/apache/tajo/rpc/NullCallback.java | 38 ++
.../tajo/rpc/ProtoChannelInitializer.java | 50 +++
.../apache/tajo/rpc/RemoteCallException.java | 69 ++++
.../tajo/rpc/RetriesExhaustedException.java | 104 ++++++
.../org/apache/tajo/rpc/RpcConnectionPool.java | 190 ++++++++++
.../org/apache/tajo/rpc/ServerCallable.java | 162 +++++++++
.../apache/tajo/rpc/TajoServiceException.java | 58 +++
.../src/main/proto/DummyProtos.proto | 47 +++
.../src/main/proto/RpcProtos.proto | 32 ++
.../src/main/proto/TestProtocol.proto | 31 ++
.../src/main/proto/TestProtos.proto | 35 ++
.../src/test/java/log4j.properties | 25 ++
.../java/org/apache/tajo/rpc/TestAsyncRpc.java | 345 ++++++++++++++++++
.../org/apache/tajo/rpc/TestBlockingRpc.java | 349 +++++++++++++++++++
.../rpc/test/impl/DummyProtocolAsyncImpl.java | 86 +++++
.../test/impl/DummyProtocolBlockingImpl.java | 83 +++++
tajo-rpc/tajo-ws-rs/pom.xml | 218 ++++++++++++
.../rs/netty/NettyRestChannelInitializer.java | 50 +++
.../ws/rs/netty/NettyRestHandlerContainer.java | 319 +++++++++++++++++
.../NettyRestHandlerContainerProvider.java | 42 +++
.../tajo/ws/rs/netty/NettyRestServer.java | 67 ++++
.../ws/rs/netty/NettyRestServerFactory.java | 89 +++++
.../ws/rs/netty/NettyRestServerListener.java | 72 ++++
.../tajo/ws/rs/netty/gson/GsonFeature.java | 34 ++
.../tajo/ws/rs/netty/gson/GsonReader.java | 52 +++
.../apache/tajo/ws/rs/netty/gson/GsonUtil.java | 32 ++
.../tajo/ws/rs/netty/gson/GsonWriter.java | 59 ++++
.../NettyRestHandlerContainerProviderTest.java | 66 ++++
.../tajo/ws/rs/netty/NettyRestServerTest.java | 137 ++++++++
.../ws/rs/netty/testapp1/TestApplication1.java | 38 ++
.../ws/rs/netty/testapp1/TestResource1.java | 36 ++
.../ws/rs/netty/testapp2/DirectoriesDao.java | 39 +++
.../rs/netty/testapp2/DirectoriesResource.java | 85 +++++
.../tajo/ws/rs/netty/testapp2/Directory.java | 52 +++
.../testapp2/FileManagementApplication.java | 35 ++
.../org/apache/tajo/storage/TestLazyTuple.java | 4 +-
.../tajo/storage/hbase/ColumnMapping.java | 6 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 3 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 3 +-
.../java/org/apache/tajo/storage/CSVFile.java | 21 +-
.../apache/tajo/storage/FileStorageManager.java | 27 +-
.../sequencefile/SequenceFileScanner.java | 3 +-
.../tajo/storage/text/CSVLineDeserializer.java | 14 +-
.../apache/tajo/storage/text/CSVLineSerDe.java | 5 +-
.../tajo/storage/text/CSVLineSerializer.java | 8 +-
.../tajo/storage/text/DelimitedTextFile.java | 2 +-
.../tajo/storage/text/FieldSplitProcessor.java | 8 +-
.../text/MultiBytesFieldSplitProcessor.java | 45 +++
.../tajo/storage/text/TextLineDeserializer.java | 6 +-
.../apache/tajo/storage/text/TextLineSerDe.java | 3 +-
.../apache/tajo/storage/TestSplitProcessor.java | 38 +-
205 files changed, 7516 insertions(+), 4256 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
----------------------------------------------------------------------
diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
index a36e982,8b73756..e320ce9
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
@@@ -18,28 -18,24 +18,31 @@@
package org.apache.tajo.plan.logical;
+ import java.util.Arrays;
+
import com.google.gson.Gson;
import com.google.gson.annotations.Expose;
+
import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.SortSpec;
-import org.apache.tajo.datum.Datum;
+import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
import org.apache.tajo.plan.serder.PlanGsonHelper;
+import org.apache.tajo.util.TUtil;
+
+import java.net.URI;
public class IndexScanNode extends ScanNode {
- @Expose private SortSpec [] sortKeys;
@Expose private Schema keySchema = null;
- @Expose private Datum[] datum = null;
+ @Expose private URI indexPath = null;
+ @Expose private SimplePredicate[] predicates = null;
+
+ public IndexScanNode(int pid) {
+ super(pid);
+ setType(NodeType.INDEX_SCAN);
+ }
public IndexScanNode(int pid, ScanNode scanNode ,
- Schema keySchema , Datum[] datum, SortSpec[] sortKeys ) {
- super(pid);
+ Schema keySchema , SimplePredicate[] predicates, URI indexPath) {
+ this(pid);
init(scanNode.getTableDesc());
setQual(scanNode.getQual());
setInSchema(scanNode.getInSchema());
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index 1ce7019,4cd008a..a257296
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@@ -957,58 -918,17 +957,60 @@@ public class FilterPushDownRule extend
return scanNode;
}
+ private static class Predicate {
+ Column column;
+ Datum value;
+ EvalType evalType;
+
+ public Predicate(EvalType evalType, Column column, Datum value) {
+ this.evalType = evalType;
+ this.column = column;
+ this.value = value;
+ }
+ }
+
+ private static SimplePredicate[] getSimplePredicates(IndexDesc desc, Set<Predicate> predicates) {
+ SimplePredicate[] simplePredicates = new SimplePredicate[predicates.size()];
+ Map<Column, Datum> colToValue = TUtil.newHashMap();
+ for (Predicate predicate : predicates) {
+ colToValue.put(predicate.column, predicate.value);
+ }
+ SortSpec [] keySortSpecs = desc.getKeySortSpecs();
+ for (int i = 0; i < keySortSpecs.length; i++) {
+ simplePredicates[i] = new SimplePredicate(keySortSpecs[i],
+ colToValue.get(keySortSpecs[i].getSortKey()));
+ }
+ return simplePredicates;
+ }
+
+ private static Datum[] extractPredicateValues(List<Predicate> predicates) {
+ Datum[] values = new Datum[predicates.size()];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = predicates.get(i).value;
+ }
+ return values;
+ }
+
+ private static Column[] extractColumns(Set<Predicate> predicates) {
+ Column[] columns = new Column[predicates.size()];
+ int i = 0;
+ for (Predicate p : predicates) {
+ columns[i++] = p.column;
+ }
+ return columns;
+ }
+
private void errorFilterPushDown(LogicalPlan plan, LogicalNode node,
FilterPushDownContext context) throws PlanningException {
- String notMatchedNodeStr = "";
String prefix = "";
+ StringBuilder notMatchedNodeStrBuilder = new StringBuilder();
for (EvalNode notMatchedNode: context.pushingDownFilters) {
- notMatchedNodeStr += prefix + notMatchedNode;
- prefix = ", ";
+ notMatchedNodeStrBuilder.append(prefix).append(notMatchedNode.toString());
+ if (prefix.isEmpty()) {
+ prefix = ", ";
+ }
}
- throw new PlanningException("FilterPushDown failed cause some filters not matched: " + notMatchedNodeStr + "\n" +
+ throw new PlanningException("FilterPushDown failed cause some filters not matched: " + notMatchedNodeStrBuilder.toString() + "\n" +
"Error node: " + node.getPlanString() + "\n" +
plan.toString());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/proto/Plan.proto
----------------------------------------------------------------------