You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/03/05 07:16:02 UTC
[10/10] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Conflicts:
tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/21d6adbd
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/21d6adbd
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/21d6adbd
Branch: refs/heads/index_support
Commit: 21d6adbda61718795fdf59e03f1d67e08378967d
Parents: 1dcc1b0 3982879
Author: Jihoon Son <ji...@apache.org>
Authored: Thu Mar 5 15:15:38 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Thu Mar 5 15:15:38 2015 +0900
----------------------------------------------------------------------
CHANGES | 13 +
pom.xml | 15 +-
tajo-algebra/pom.xml | 2 +-
tajo-catalog/pom.xml | 2 +-
tajo-catalog/tajo-catalog-client/pom.xml | 2 +-
tajo-catalog/tajo-catalog-common/pom.xml | 2 +-
tajo-catalog/tajo-catalog-drivers/pom.xml | 2 +-
.../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 2 +-
tajo-catalog/tajo-catalog-server/pom.xml | 2 +-
tajo-cli/pom.xml | 2 +-
tajo-client/pom.xml | 2 +-
.../org/apache/tajo/client/QueryClientImpl.java | 3 +-
.../apache/tajo/client/SessionConnection.java | 15 +-
tajo-common/pom.xml | 2 +-
.../tajo/util/datetime/DateTimeConstants.java | 31 ++-
.../apache/tajo/datum/TestTimestampDatum.java | 7 +
tajo-core/pom.xml | 6 +-
.../tajo/engine/codegen/EvalCodeGenerator.java | 6 +-
.../engine/codegen/TajoGeneratorAdapter.java | 26 +-
.../tajo/engine/function/builtin/AvgDouble.java | 2 +-
.../tajo/engine/function/builtin/AvgLong.java | 2 +-
.../tajo/engine/function/builtin/CountRows.java | 2 +-
.../function/builtin/CountValueDistinct.java | 2 +-
.../tajo/engine/function/builtin/LastValue.java | 2 +-
.../tajo/engine/function/builtin/Lead.java | 2 +-
.../tajo/engine/function/builtin/Max.java | 2 +-
.../tajo/engine/function/builtin/Min.java | 2 +-
.../tajo/engine/function/builtin/SumDouble.java | 2 +-
.../function/builtin/SumDoubleDistinct.java | 2 +-
.../function/builtin/SumFloatDistinct.java | 2 +-
.../engine/function/builtin/SumIntDistinct.java | 2 +-
.../tajo/engine/function/builtin/SumLong.java | 2 +-
.../function/builtin/SumLongDistinct.java | 2 +-
.../function/datetime/DatePartFromDate.java | 28 +-
.../function/datetime/DatePartFromTime.java | 18 +-
.../datetime/DatePartFromTimestamp.java | 40 +--
.../datetime/DateTimePartFromUnixTimestamp.java | 10 +-
.../engine/function/string/RegexpReplace.java | 2 -
.../tajo/engine/function/window/FirstValue.java | 2 +-
.../apache/tajo/engine/function/window/Lag.java | 2 +-
.../tajo/engine/function/window/Rank.java | 2 +-
.../tajo/engine/function/window/RowNumber.java | 2 +-
.../engine/planner/PhysicalPlannerImpl.java | 24 +-
.../planner/global/ExecutionBlockCursor.java | 8 +
.../engine/planner/global/GlobalPlanner.java | 6 +-
.../global/builder/DistinctGroupbyBuilder.java | 8 +-
.../planner/physical/AggregationExec.java | 7 -
.../engine/planner/physical/BNLJoinExec.java | 2 +-
.../planner/physical/ColPartitionStoreExec.java | 6 +
.../DistinctGroupbyFirstAggregationExec.java | 4 -
.../DistinctGroupbyHashAggregationExec.java | 8 -
.../DistinctGroupbySortAggregationExec.java | 3 -
.../DistinctGroupbyThirdAggregationExec.java | 6 +-
.../planner/physical/ExternalSortExec.java | 2 +-
.../engine/planner/physical/HashJoinExec.java | 2 -
.../physical/HashShuffleFileWriteExec.java | 5 +-
.../physical/RightOuterMergeJoinExec.java | 1 -
.../engine/planner/physical/SeqScanExec.java | 4 +-
.../engine/planner/physical/WindowAggExec.java | 2 -
.../apache/tajo/engine/utils/TupleCacheKey.java | 3 +
.../tajo/engine/utils/TupleCacheScanner.java | 2 +-
.../org/apache/tajo/ha/HdfsServiceTracker.java | 7 +-
.../java/org/apache/tajo/master/QueryInfo.java | 47 ++++
.../apache/tajo/master/TajoContainerProxy.java | 2 +-
.../java/org/apache/tajo/master/TajoMaster.java | 5 +-
.../NonForwardQueryResultSystemScanner.java | 9 +-
.../tajo/master/rm/TajoResourceTracker.java | 4 +-
.../tajo/master/rm/TajoWorkerContainer.java | 22 ++
.../java/org/apache/tajo/master/rm/Worker.java | 37 +++
.../java/org/apache/tajo/querymaster/Query.java | 19 +-
.../apache/tajo/querymaster/QueryMaster.java | 8 +-
.../tajo/querymaster/QueryMasterTask.java | 15 +-
.../apache/tajo/querymaster/Repartitioner.java | 23 +-
.../java/org/apache/tajo/querymaster/Stage.java | 13 +-
.../java/org/apache/tajo/querymaster/Task.java | 12 +
.../apache/tajo/querymaster/TaskAttempt.java | 12 +
.../main/java/org/apache/tajo/util/JSPUtil.java | 3 +
.../tajo/util/history/HistoryCleaner.java | 2 +
.../apache/tajo/util/history/HistoryWriter.java | 3 +
.../reporter/MetricsFileScheduledReporter.java | 5 +-
.../MetricsStreamScheduledReporter.java | 1 -
.../metrics/reporter/TajoMetricsReporter.java | 4 +-
.../reporter/TajoMetricsScheduledReporter.java | 6 +-
.../tajo/webapp/QueryExecutorServlet.java | 29 ++-
.../apache/tajo/webapp/StaticHttpServer.java | 20 +-
.../tajo/worker/ExecutionBlockContext.java | 37 +--
.../worker/ExecutionBlockSharedResource.java | 2 +-
.../java/org/apache/tajo/worker/Fetcher.java | 198 +++++++-------
.../tajo/worker/TajoResourceAllocator.java | 6 +
.../java/org/apache/tajo/worker/TajoWorker.java | 5 +-
.../main/java/org/apache/tajo/worker/Task.java | 12 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 17 +-
.../apache/tajo/worker/TaskRunnerManager.java | 15 +-
.../tajo/worker/WorkerHeartbeatService.java | 1 -
.../apache/tajo/master/TestRepartitioner.java | 5 +-
.../org/apache/tajo/worker/TestFetcher.java | 25 +-
tajo-dist/pom.xml | 2 +-
tajo-docs/src/main/sphinx/index.rst | 1 +
tajo-docs/src/main/sphinx/swift_integration.rst | 110 ++++++++
tajo-jdbc/pom.xml | 2 +-
tajo-maven-plugins/pom.xml | 2 +-
tajo-plan/pom.xml | 2 +-
tajo-project/pom.xml | 30 ++-
tajo-pullserver/pom.xml | 10 +-
.../tajo/pullserver/FadvisedChunkedFile.java | 17 +-
.../tajo/pullserver/FadvisedFileRegion.java | 16 +-
.../tajo/pullserver/FileCloseListener.java | 8 +-
.../HttpDataServerChannelInitializer.java | 58 +++++
.../tajo/pullserver/HttpDataServerHandler.java | 137 +++++-----
.../HttpDataServerPipelineFactory.java | 56 ----
.../tajo/pullserver/PullServerAuxService.java | 229 ++++++++--------
.../tajo/pullserver/TajoPullServerService.java | 259 ++++++++++---------
.../retriever/AdvancedDataRetriever.java | 10 +-
.../pullserver/retriever/DataRetriever.java | 4 +-
.../retriever/DirectoryRetriever.java | 5 +-
tajo-rpc/pom.xml | 12 +-
.../org/apache/tajo/rpc/AsyncRpcClient.java | 106 +++++---
.../org/apache/tajo/rpc/AsyncRpcServer.java | 126 ++++-----
.../org/apache/tajo/rpc/BlockingRpcClient.java | 122 +++++----
.../org/apache/tajo/rpc/BlockingRpcServer.java | 125 +++++----
.../java/org/apache/tajo/rpc/CallFuture.java | 8 +-
.../apache/tajo/rpc/DefaultRpcController.java | 7 +-
.../org/apache/tajo/rpc/NettyClientBase.java | 133 ++++++----
.../org/apache/tajo/rpc/NettyServerBase.java | 82 +++---
.../java/org/apache/tajo/rpc/NullCallback.java | 2 +-
.../tajo/rpc/ProtoChannelInitializer.java | 50 ++++
.../apache/tajo/rpc/ProtoPipelineFactory.java | 50 ----
.../org/apache/tajo/rpc/RpcChannelFactory.java | 160 ++++++++----
.../org/apache/tajo/rpc/RpcConnectionPool.java | 87 +++----
.../org/apache/tajo/rpc/ServerCallable.java | 10 +-
.../java/org/apache/tajo/rpc/TestAsyncRpc.java | 144 ++++++++---
.../org/apache/tajo/rpc/TestBlockingRpc.java | 138 +++++++---
.../rpc/test/impl/DummyProtocolAsyncImpl.java | 3 +-
tajo-storage/pom.xml | 2 +-
tajo-storage/tajo-storage-common/pom.xml | 2 +-
tajo-storage/tajo-storage-hbase/pom.xml | 2 +-
tajo-storage/tajo-storage-hdfs/pom.xml | 14 +-
.../java/org/apache/tajo/HttpFileServer.java | 44 ++--
.../tajo/HttpFileServerChannelInitializer.java | 47 ++++
.../org/apache/tajo/HttpFileServerHandler.java | 109 ++++----
.../tajo/HttpFileServerPipelineFactory.java | 54 ----
tajo-thirdparty/asm/pom.xml | 2 +-
142 files changed, 2146 insertions(+), 1463 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
index 3802682,c2740e5..69fe209
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
@@@ -488,49 -488,7 +491,49 @@@ public class Query implements EventHand
}
}
- private class CreateIndexHook implements QueryHook {
++ private static class CreateIndexHook implements QueryHook {
+
+ @Override
+ public boolean isEligible(QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) {
+ Stage lastStage = query.getStage(finalExecBlockId);
+ return lastStage.getBlock().getPlan().getType() == NodeType.CREATE_INDEX;
+ }
+
+ @Override
+ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception {
+ CatalogService catalog = context.getWorkerContext().getCatalog();
+ Stage lastStage = query.getStage(finalExecBlockId);
+
+ CreateIndexNode createIndexNode = (CreateIndexNode) lastStage.getBlock().getPlan();
+ String databaseName, simpleIndexName, qualifiedIndexName;
+ if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) {
+ String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName());
+ databaseName = splits[0];
+ simpleIndexName = splits[1];
+ qualifiedIndexName = createIndexNode.getIndexName();
+ } else {
+ databaseName = queryContext.getCurrentDatabase();
+ simpleIndexName = createIndexNode.getIndexName();
+ qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName);
+ }
+ ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN);
+ if (scanNode == null) {
+ throw new IOException("Cannot find the table of the relation");
+ }
+ IndexDesc indexDesc = new IndexDesc(databaseName, scanNode.getTableName(),
+ simpleIndexName, createIndexNode.getIndexPath(),
+ createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(),
+ createIndexNode.isUnique(), false, scanNode.getLogicalSchema());
+ if (catalog.createIndex(indexDesc)) {
+ LOG.info("Index " + qualifiedIndexName + " is created for the table " + scanNode.getTableName() + ".");
+ } else {
+ LOG.info("Index creation " + qualifiedIndexName + " is failed.");
+ throw new CatalogException("Cannot create index \"" + qualifiedIndexName + "\".");
+ }
+ }
+ }
+
- private class MaterializedResultHook implements QueryHook {
+ private static class MaterializedResultHook implements QueryHook {
@Override
public boolean isEligible(QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId,
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/21d6adbd/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------